From 4cb54fea811b41ea82daf26e20856d4e7df15979 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Mon, 27 May 2024 20:04:58 +0000 Subject: [PATCH 01/41] add first draft of network utils --- bento_beacon/network/utils.py | 133 ++++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 bento_beacon/network/utils.py diff --git a/bento_beacon/network/utils.py b/bento_beacon/network/utils.py new file mode 100644 index 00000000..1deb4535 --- /dev/null +++ b/bento_beacon/network/utils.py @@ -0,0 +1,133 @@ +from flask import current_app, g +import requests +from urllib.parse import urlsplit, urlunsplit +from json import JSONDecodeError +from .network_config import BEACONS, NETWORK_TIMEOUT +from ..utils.exceptions import APIException, InvalidQuery +from ..utils.beacon_response import beacon_count_response + +DEFAULT_ENDPOINT = "individuals" +OVERVIEW_STATS_QUERY = { + "meta": {"apiVersion": "2.0.0"}, + "query": {"requestParameters": {}, "filters": [], "includeResultsetResponses": "ALL"}, + "bento": {"showSummaryStatistics": True}, +} + + +def network_beacon_call(method, url, payload=None): + try: + if method == "GET": + r = requests.get(url, timeout=NETWORK_TIMEOUT) + else: + r = requests.post(url, json=payload, timeout=NETWORK_TIMEOUT) + beacon_response = r.json() + + except requests.exceptions.RequestException as e: + current_app.logger.error(e) + msg = f"beacon network error calling url {url}: {e}" + raise APIException(message=msg) + + return beacon_response + + +def network_beacon_get(root_url, endpoint=None): + url = root_url if endpoint is None else root_url + "/" + endpoint + return network_beacon_call("GET", url) + + +def network_beacon_post(root_url, payload={}, endpoint=None): + url = root_url if endpoint is None else root_url + "/" + endpoint + return network_beacon_call("POST", url, payload) + + +def init_network_service_registry(): + current_app.logger.info("registering beacons") + network_beacons = {} + for url in BEACONS: + b = network_beacon_get(url, endpoint="overview") + beacon_info = b.get("response") + if not beacon_info: + msg = f"bad response from network beacon {url}" + raise APIException(message=msg) + + beacon_info["api_url"] = url + + # organize overview stats + # TODO (Redmine #2170) modify beacon /overview so we don't have to make two calls here, with different response formats + + # TODO: filters here?? + biosample_and_experiment_stats = ( + network_beacon_post(url, OVERVIEW_STATS_QUERY, DEFAULT_ENDPOINT).get("info", {}).get("bento") + ) + individual_and_variant_stats = beacon_info.get("overview", {}).get("counts") + + overview = { + "individuals": {"count": individual_and_variant_stats.get("individuals")}, + "variants": individual_and_variant_stats.get("variants"), + **biosample_and_experiment_stats, + } + + b_id = beacon_info.get("id") + network_beacons[b_id] = beacon_info + network_beacons[b_id]["overview"] = overview + + current_app.config["NETWORK_BEACONS"] = network_beacons + + +def beacon_network_response(beacon_responses): + num_total_results, bento_stats = sum_network_responses(beacon_responses) + g.response_info["bento"] = bento_stats + g.response_info["network"] = beacon_responses + + # beacon network hardcoded to counts + # to handle all possible granularities, call build_query_response() instead + return beacon_count_response(num_total_results) + + +def sum_network_responses(beacon_responses): + num_total_results = 0 + # could parameterize response, currently hardcoded in bento_public + experiments_count = 0 + biosamples_count = 0 + sampled_tissue_chart = [] + experiment_type_chart = [] + + for response in beacon_responses.values(): + num_total_results += response.get("responseSummary", {}).get("numTotalResults", 0) + stats = response.get("info", {}).get("bento", {}) + experiments_count += stats.get("experiments", {}).get("count") + biosamples_count += stats.get("biosamples", {}).get("count") + sampled_tissue_chart = merge_charts(sampled_tissue_chart, stats.get("biosamples", {}).get("sampled_tissue", [])) + experiment_type_chart = merge_charts( + experiment_type_chart, stats.get("experiments", {}).get("experiment_type", []) + ) + + # count be parameterized but currently hardcoded in bento_public + bento_stats = { + "biosamples": {"count": biosamples_count, "sampled_tissue": sampled_tissue_chart}, + "experiments": {"count": experiments_count, "experiment_type": experiment_type_chart}, + } + + return num_total_results, bento_stats + + +def chart_to_dict(chart): + return {item["label"]: item["value"] for item in chart} + + +def dict_to_chart(d): + return [{"label": key, "value": d[key]} for key in d] + + +def merge_charts(c1, c2): + """ + add data from two categorical charts together + any categories with identical names are merged into a single field with the sum from both charts + """ + merged = chart_to_dict(c1) + for cat in c2: + label = cat["label"] + value = cat["value"] + merged[label] = merged.get(label, 0) + value + + return dict_to_chart(merged) From 4d10ceb5a807c03066c98346d9dadfec842cd71c Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Tue, 28 May 2024 18:18:21 +0000 Subject: [PATCH 02/41] first draft of endpoints for beacon network --- bento_beacon/network/network.py | 88 +++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 bento_beacon/network/network.py diff --git a/bento_beacon/network/network.py b/bento_beacon/network/network.py new file mode 100644 index 00000000..ba60cd6d --- /dev/null +++ b/bento_beacon/network/network.py @@ -0,0 +1,88 @@ +from flask import current_app, g, request, Blueprint +from json import JSONDecodeError +import requests +from urllib.parse import urlsplit, urlunsplit +from ..utils.exceptions import APIException, NotFoundException +from .utils import beacon_network_response, network_beacon_get, network_beacon_post +from .network_config import VALID_ENDPOINTS + +network = Blueprint("network", __name__, url_prefix="/network") + + +# TODOs: +# filtering terms XXXXXXXXXXXXXXXXXXXXXXXXXXX +# front end federating code (get beacon proxy urls, call all of them) +# only need to know network url + ids for each beacon + +# standard beacon info endpoints at the network level: /map, /configuration, etc +# handle GET args + + +@network.route("/") +@network.route("/beacons") +def network_beacons(): + beacons = current_app.config["NETWORK_BEACONS"] + + # all beacons + # filters: copy from ? + # also want overview stats from empty individuals query + # filters?? + return beacons + + +@network.route("/query/", methods=["POST"]) +def dumb_network_query(endpoint): + """ + Beacon network query in a single request and single response. + Returns an aggregate response as well as an array of individual beacon responses. + As slow as the slowest beacon. + """ + beacons = current_app.config["NETWORK_BEACONS"] + if not beacons: + raise APIException(message="beacon network error, network is empty") + + responses = {} + for b in beacons: + url = beacons[b].get("api_url") + try: + r = network_beacon_post( + url, + request.json, + endpoint, + ) + except APIException as e: + r["error"] = {"errorMessage": e.message} + continue + + # discard beacon "meta" response field + # it's the same for all requests, including this network request + r.pop("meta", None) + + responses[b] = r + + if not responses: + raise APIException(message="No response from beacon network") + + return beacon_network_response(responses) + + +@network.route("/beacons/") +@network.route("/beacons//", methods=["GET", "POST"]) +def query(beacon_id, endpoint="overview"): + beacon = current_app.config["NETWORK_BEACONS"].get(beacon_id) + if not beacon: + raise NotFoundException(message=f"no beacon found with id {beacon_id}") + + if endpoint not in VALID_ENDPOINTS: + raise NotFoundException() + + api_url = beacon.get("api_url") + + if request.method == "POST": + payload = request.get_json() + r = network_beacon_post(api_url, payload, endpoint) + else: + # TODO: pass get args + r = network_beacon_get(api_url, endpoint) + + return r From c21d71d247a6c441e7b3d98ff46f0d01ed715311 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Fri, 31 May 2024 11:22:35 -0400 Subject: [PATCH 03/41] don't crash if a network beacon is down --- bento_beacon/network/utils.py | 37 +++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/bento_beacon/network/utils.py b/bento_beacon/network/utils.py index 1deb4535..30b05a0a 100644 --- a/bento_beacon/network/utils.py +++ b/bento_beacon/network/utils.py @@ -22,7 +22,7 @@ def network_beacon_call(method, url, payload=None): r = requests.post(url, json=payload, timeout=NETWORK_TIMEOUT) beacon_response = r.json() - except requests.exceptions.RequestException as e: + except (requests.exceptions.RequestException, JSONDecodeError) as e: current_app.logger.error(e) msg = f"beacon network error calling url {url}: {e}" raise APIException(message=msg) @@ -43,14 +43,23 @@ def network_beacon_post(root_url, payload={}, endpoint=None): def init_network_service_registry(): current_app.logger.info("registering beacons") network_beacons = {} + failed_beacons = [] for url in BEACONS: - b = network_beacon_get(url, endpoint="overview") - beacon_info = b.get("response") + try: + b = network_beacon_get(url, endpoint="overview") + beacon_info = b.get("response") + + except APIException: + failed_beacons.append(url) + current_app.logger.error(f"error contacting network beacon {url}") + continue + if not beacon_info: - msg = f"bad response from network beacon {url}" - raise APIException(message=msg) + failed_beacons.append(url) + current_app.logger.error(f"bad response from network beacon {url}") + continue - beacon_info["api_url"] = url + beacon_info["apiUrl"] = url # organize overview stats # TODO (Redmine #2170) modify beacon /overview so we don't have to make two calls here, with different response formats @@ -71,6 +80,15 @@ def init_network_service_registry(): network_beacons[b_id] = beacon_info network_beacons[b_id]["overview"] = overview + + "s" if len(network_beacons) == 1 else "" + + # make a merged overview? + # what about merged filtering_terms? + current_app.logger.info(f"registered {len(network_beacons)} beacon{'' if len(network_beacons) == 1 else 's'} in network: {', '.join(network_beacons)}") + if failed_beacons: + current_app.logger.error(f"{len(failed_beacons)} network beacon{'' if len(failed_beacons) == 1 else 's'} failed to respond: {', '.join(failed_beacons)}") + current_app.config["NETWORK_BEACONS"] = network_beacons @@ -102,7 +120,6 @@ def sum_network_responses(beacon_responses): experiment_type_chart, stats.get("experiments", {}).get("experiment_type", []) ) - # count be parameterized but currently hardcoded in bento_public bento_stats = { "biosamples": {"count": biosamples_count, "sampled_tissue": sampled_tissue_chart}, "experiments": {"count": experiments_count, "experiment_type": experiment_type_chart}, @@ -116,13 +133,13 @@ def chart_to_dict(chart): def dict_to_chart(d): - return [{"label": key, "value": d[key]} for key in d] + return [{"label": label, "value": value} for label, value in d.items()] def merge_charts(c1, c2): """ - add data from two categorical charts together - any categories with identical names are merged into a single field with the sum from both charts + combine data from two categorical charts + any categories with identical names are merged into a single field with the sum of their values """ merged = chart_to_dict(c1) for cat in c2: From f2b6a48e31df4e9b664ba6f721280972c4b82876 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Tue, 11 Jun 2024 16:26:19 +0000 Subject: [PATCH 04/41] handle filters --- bento_beacon/network/utils.py | 36 ++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/bento_beacon/network/utils.py b/bento_beacon/network/utils.py index 30b05a0a..ff88e57d 100644 --- a/bento_beacon/network/utils.py +++ b/bento_beacon/network/utils.py @@ -80,14 +80,19 @@ def init_network_service_registry(): network_beacons[b_id] = beacon_info network_beacons[b_id]["overview"] = overview - - "s" if len(network_beacons) == 1 else "" + # TODO, katsu calls are inconsistent here + # qs = get_public_search_fields(url) + # network_beacons[b_id]["querySections"] = get_public_search_fields(url) # temp - # make a merged overview? + # make a merged overview? # what about merged filtering_terms? - current_app.logger.info(f"registered {len(network_beacons)} beacon{'' if len(network_beacons) == 1 else 's'} in network: {', '.join(network_beacons)}") + current_app.logger.info( + f"registered {len(network_beacons)} beacon{'' if len(network_beacons) == 1 else 's'} in network: {', '.join(network_beacons)}" + ) if failed_beacons: - current_app.logger.error(f"{len(failed_beacons)} network beacon{'' if len(failed_beacons) == 1 else 's'} failed to respond: {', '.join(failed_beacons)}") + current_app.logger.error( + f"{len(failed_beacons)} network beacon{'' if len(failed_beacons) == 1 else 's'} failed to respond: {', '.join(failed_beacons)}" + ) current_app.config["NETWORK_BEACONS"] = network_beacons @@ -148,3 +153,24 @@ def merge_charts(c1, c2): merged[label] = merged.get(label, 0) + value return dict_to_chart(merged) + + +def get_public_search_fields(beacon_url): + fields_url = public_search_fields_url(beacon_url) + fields = network_beacon_get(fields_url) + return fields + + +def public_search_fields_url(beacon_url): + split_url = urlsplit(beacon_url) + return urlunsplit( + (split_url.scheme, "portal." + split_url.netloc, "/api/metadata/api/public_search_fields", "", "") + ) + + +def filtersUnion(): + pass + + +def filtersIntersection(): + pass From 912b2e2cf45495e4a8cdec76c54ecebda38ea19a Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Tue, 11 Jun 2024 16:27:16 +0000 Subject: [PATCH 05/41] temp network config --- bento_beacon/network/network_config.py | 387 +++++++++++++++++++++++++ 1 file changed, 387 insertions(+) create mode 100644 bento_beacon/network/network_config.py diff --git a/bento_beacon/network/network_config.py b/bento_beacon/network/network_config.py new file mode 100644 index 00000000..9d3001d9 --- /dev/null +++ b/bento_beacon/network/network_config.py @@ -0,0 +1,387 @@ +# XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +# temp file, all of this to be handled elsewhere in final version +# XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + +BEACONS = [ + "https://rsrq.bento.sd4h.ca/api/beacon", + "https://bqc19.bento.sd4h.ca/api/beacon", + # "https://staging.bqc19.bento.sd4h.ca/api/beacon", + "https://staging.bento.sd4h.ca/api/beacon", + "https://ichange.bento.sd4h.ca/api/beacon", + # "https://qa.bento.sd4h.ca/api/beacon/", + # "https://bentov2.local/api/beacon", +] + +NETWORK_TIMEOUT = 30 + +VALID_ENDPOINTS = ["analyses", "biosamples", "cohorts", "datasets", "g_variants", "individuals", "runs", "overview"] + +KATSU_CONFIG_UNION = [ + { + "section_title": "General", + "fields": [ + { + "mapping": "individual/age_numeric", + "title": "Age", + "description": "Age at arrival", + "datatype": "number", + "config": { + "bin_size": 10, + "taper_left": 10, + "taper_right": 100, + "units": "years", + "minimum": 0, + "maximum": 100, + }, + "id": "age", + "options": [ + "< 10", + "[10, 20)", + "[20, 30)", + "[30, 40)", + "[40, 50)", + "[50, 60)", + "[60, 70)", + "[70, 80)", + "[80, 90)", + "[90, 100)", + ], + }, + { + "mapping": "individual/sex", + "title": "Sex", + "description": "Sex at birth", + "datatype": "string", + "config": {"enum": None}, + "id": "sex", + "options": ["MALE", "FEMALE"], + }, + { + "mapping": "individual/extra_properties/date_of_consent", + "title": "Verbal consent date", + "description": "Date of initial verbal consent (participant, legal representative or tutor), yyyy-mm-dd", + "datatype": "date", + "config": {"bin_by": "month"}, + "id": "date_of_consent", + "options": [ + "Jan 2020", + "Feb 2020", + "Mar 2020", + "Apr 2020", + "May 2020", + "Jun 2020", + "Jul 2020", + "Aug 2020", + "Sep 2020", + "Oct 2020", + "Nov 2020", + "Dec 2020", + "Jan 2021", + "Feb 2021", + "Mar 2021", + "Apr 2021", + "May 2021", + "Jun 2021", + "Jul 2021", + "Aug 2021", + "Sep 2021", + "Oct 2021", + "Nov 2021", + "Dec 2021", + "Jan 2022", + "Feb 2022", + "Mar 2022", + "Apr 2022", + "May 2022", + "Jun 2022", + "Jul 2022", + "Aug 2022", + "Sep 2022", + "Oct 2022", + "Nov 2022", + "Dec 2022", + "Jan 2023", + "Feb 2023", + "Mar 2023", + "Apr 2023", + "May 2023", + "Jun 2023", + "Jul 2023", + "Aug 2023", + "Sep 2023", + "Oct 2023", + "Nov 2023", + "Dec 2023", + ], + }, + { + "mapping": "individual/extra_properties/mobility", + "title": "Functional status", + "description": "Mobility", + "datatype": "string", + "config": { + "enum": [ + "I have no problems in walking about", + "I have slight problems in walking about", + "I have moderate problems in walking about", + "I have severe problems in walking about", + "I am unable to walk about", + ] + }, + "id": "mobility", + "options": [ + "I have no problems in walking about", + "I have slight problems in walking about", + "I have moderate problems in walking about", + "I have severe problems in walking about", + "I am unable to walk about", + ], + }, + { + "mapping": "individual/extra_properties/covid_severity", + "title": "Covid severity", + "description": "Covid severity", + "datatype": "string", + "config": {"enum": ["Uninfected", "Mild", "Moderate", "Severe", "Dead"]}, + "id": "covid_severity", + "options": ["Uninfected", "Mild", "Moderate", "Severe", "Dead"], + }, + { + "mapping": "individual/phenopackets/phenotypic_features/pftype/label", + "title": "Phenotypic Features", + "description": "Individual phenotypic features, observed as either present or absent", + "datatype": "string", + "config": {"enum": None}, + "id": "phenotypic_features", + "options": [ + "Loss of appetite", + "Asthma", + "Abdominal pain", + "Headache", + "Cough", + "Cardiac arrest", + "Nausea", + "Pulmonary hypertension", + "Viral pneumonia/pneumonitis", + "HIV Infection", + "Hyperglycemia", + "Patient immunosuppressed", + "Fever", + "Chest pain", + "Pneumothorax", + "Hypertension", + "Dementia", + "Loss of sense of smell", + ], + }, + { + "mapping": "individual/phenopackets/diseases/term/label", + "title": "Diseases", + "description": "Diseases observed as either present or absent", + "datatype": "string", + "config": {"enum": None}, + "id": "diseases", + "options": [ + "viral pneumonia", + "Glioblastoma", + "Polycythemia vera", + "Miller syndrome", + "Cystic fibrosis", + "coronary artery disease, autosomal dominant, 1", + "Hashimoto Thyroiditis", + "Rheumatologic disease", + "Takotsubo cardiomyopathy", + "autoimmune disorder of cardiovascular system", + "COVID-19", + "Marfan syndrome", + "Non-Hodgkin Lymphoma", + "diabetes mellitus", + ], + }, + { + "mapping": "biosample/sampled_tissue/label", + "mapping_for_search_filter": "individual/biosamples/sampled_tissue/label", + "title": "Sampled Tissues", + "description": "Tissue from which the biosample was extracted", + "datatype": "string", + "config": {"enum": None}, + "id": "tissues", + "options": ["Plasma", "blood", "Serum"], + }, + ], + }, + { + "section_title": "Measurements", + "fields": [ + { + "mapping": "individual/extra_properties/lab_test_result_value", + "title": "Lab Test Result", + "description": "Numeric measures from a laboratory test", + "datatype": "number", + "config": {"bins": [200, 300, 500, 1000, 1500, 2000], "minimum": 0, "units": "mg/L"}, + "id": "lab_test_result_value", + "options": [ + "< 200", + "[200, 300)", + "[300, 500)", + "[500, 1000)", + "[1000, 1500)", + "[1500, 2000)", + "≥ 2000", + ], + }, + { + "mapping": "individual/phenopackets/measurements", + "group_by": "assay/id", + "group_by_value": "NCIT:C16358", + "value_mapping": "value/quantity/value", + "title": "BMI", + "description": "Body Mass Index", + "datatype": "number", + "config": {"bins": [18.5, 30], "minimum": 0, "units": "kg/m^2"}, + "id": "bmi", + "options": ["< 18", "[18, 30)", "≥ 30"], + }, + ], + }, + { + "section_title": "Medical Actions", + "fields": [ + { + "mapping": "individual/phenopackets/medical_actions", + "group_by": "procedure/code/label", + "title": "Medical Procedures", + "description": "A clinical procedure performed on a subject", + "datatype": "string", + "config": {"enum": None}, + "id": "medical_procedures", + "options": [ + "Liver Biopsy", + "Magnetic Resonance Imaging", + "Positron Emission Tomography", + "Punch Biopsy", + "X-Ray Imaging", + ], + }, + { + "mapping": "individual/phenopackets/medical_actions", + "group_by": "treatment/agent/label", + "title": "Medical Treatments", + "description": "Treatment with an agent such as a drug", + "datatype": "string", + "config": {"enum": None}, + "id": "medical_treatments", + "options": ["Acetaminophen", "Ibuprofen", "Ondansetron"], + }, + ], + }, + { + "section_title": "Interpretations", + "fields": [ + { + "mapping": "individual/phenopackets/interpretations/diagnosis/genomic_interpretations/interpretation_status", + "title": "Genomic Interpretations", + "description": "Interpretation for an individual variant or gene (CANDIDATE, CONTRIBUTORY, etc)", + "datatype": "string", + "config": {"enum": None}, + "id": "interpretation_status", + "options": ["CONTRIBUTORY"], + }, + { + "mapping": "individual/phenopackets/interpretations/diagnosis/genomic_interpretations/variant_interpretation/acmg_pathogenicity_classification", + "title": "Variant Pathogenicity", + "description": "ACMG Pathogenicity category for a particular variant (BENIGN, PATHOGENIC, etc)", + "datatype": "string", + "config": {"enum": None}, + "id": "acmg_pathogenicity_classification", + "options": ["PATHOGENIC"], + }, + ], + }, + { + "section_title": "Experiments", + "fields": [ + { + "mapping": "experiment/experiment_type", + "mapping_for_search_filter": "individual/biosamples/experiment/experiment_type", + "title": "Experiment Types", + "description": "Types of experiments performed on a sample", + "datatype": "string", + "config": {"enum": None}, + "id": "experiment_type", + "options": [ + "Other", + "Neutralizing antibody titers", + "Metabolite profiling", + "RNA-Seq", + "WGS", + "Proteomic profiling", + ], + }, + { + "mapping": "experiment/study_type", + "mapping_for_search_filter": "individual/biosamples/experiment/study_type", + "title": "Study Types", + "description": "Study type of the experiment (e.g. Genomics, Transcriptomics, etc.)", + "datatype": "string", + "config": {"enum": None}, + "id": "experiment_study_type", + "options": ["Serology", "Genomics", "Other", "Proteomics", "Transcriptomics", "Metabolomics"], + }, + { + "mapping": "experiment/experiment_results/file_format", + "mapping_for_search_filter": "individual/biosamples/experiment/experiment_results/file_format", + "title": "Results File Types", + "description": "File type of experiment results files", + "datatype": "string", + "config": {"enum": None}, + "id": "experiment_results_file_type", + "options": ["XLSX", "CRAM", "VCF", "JPEG", "MP4", "DOCX", "CSV", "MARKDOWN"], + }, + ], + }, +] + +KATSU_CONFIG_INTERSECTION = [ + { + "section_title": "General", + "fields": [ + { + "mapping": "individual/age_numeric", + "title": "Age", + "description": "Age at arrival", + "datatype": "number", + "config": { + "bin_size": 10, + "taper_left": 10, + "taper_right": 100, + "units": "years", + "minimum": 0, + "maximum": 100, + }, + "id": "age", + "options": [ + "< 10", + "[10, 20)", + "[20, 30)", + "[30, 40)", + "[40, 50)", + "[50, 60)", + "[60, 70)", + "[70, 80)", + "[80, 90)", + "[90, 100)", + ], + }, + { + "mapping": "individual/sex", + "title": "Sex", + "description": "Sex at birth", + "datatype": "string", + "config": {"enum": None}, + "id": "sex", + "options": ["MALE", "FEMALE"], + }, + ], + }, +] From 3e458aaea57e9d15370b7c2395a67b1d7ffdb9ab Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Tue, 11 Jun 2024 16:29:46 +0000 Subject: [PATCH 06/41] network fixes --- bento_beacon/app.py | 11 +++++++++++ bento_beacon/network/__init__.py | 0 bento_beacon/network/network.py | 24 ++++++++++++++---------- 3 files changed, 25 insertions(+), 10 deletions(-) create mode 100644 bento_beacon/network/__init__.py diff --git a/bento_beacon/app.py b/bento_beacon/app.py index f6c8f125..c23c3728 100644 --- a/bento_beacon/app.py +++ b/bento_beacon/app.py @@ -9,6 +9,8 @@ from .endpoints.biosamples import biosamples from .endpoints.cohorts import cohorts from .endpoints.datasets import datasets +from .network.network import network +from .network.utils import init_network_service_registry from .utils.exceptions import APIException from werkzeug.exceptions import HTTPException from .authz.middleware import authz_middleware @@ -22,6 +24,10 @@ REQUEST_SPEC_RELATIVE_PATH = "beacon-v2/framework/json/requests/" BEACON_MODELS = ["analyses", "biosamples", "cohorts", "datasets", "individuals", "runs", "variants"] +# temp +# add to config +USE_BEACON_NETWORK = True + app = Flask(__name__) # find path for beacon-v2 spec @@ -44,6 +50,11 @@ app.register_blueprint(info) +if USE_BEACON_NETWORK: + app.register_blueprint(network) + with app.app_context(): + init_network_service_registry() + blueprints = { "biosamples": biosamples, "cohorts": cohorts, diff --git a/bento_beacon/network/__init__.py b/bento_beacon/network/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/bento_beacon/network/network.py b/bento_beacon/network/network.py index ba60cd6d..ef2c91c0 100644 --- a/bento_beacon/network/network.py +++ b/bento_beacon/network/network.py @@ -4,15 +4,15 @@ from urllib.parse import urlsplit, urlunsplit from ..utils.exceptions import APIException, NotFoundException from .utils import beacon_network_response, network_beacon_get, network_beacon_post -from .network_config import VALID_ENDPOINTS +from .network_config import VALID_ENDPOINTS, KATSU_CONFIG_UNION, KATSU_CONFIG_INTERSECTION network = Blueprint("network", __name__, url_prefix="/network") # TODOs: # filtering terms XXXXXXXXXXXXXXXXXXXXXXXXXXX -# front end federating code (get beacon proxy urls, call all of them) -# only need to know network url + ids for each beacon +# /service-info? there's already one at beacon root +# async calls # standard beacon info endpoints at the network level: /map, /configuration, etc # handle GET args @@ -23,11 +23,15 @@ def network_beacons(): beacons = current_app.config["NETWORK_BEACONS"] - # all beacons - # filters: copy from ? - # also want overview stats from empty individuals query - # filters?? - return beacons + # temp, fake + filters_union = KATSU_CONFIG_UNION + filters_intersection = KATSU_CONFIG_INTERSECTION + + return { + "filtersUnion": filters_union, + "filtersIntersection": filters_intersection, + "beacons": list(beacons.values()), + } @network.route("/query/", methods=["POST"]) @@ -43,7 +47,7 @@ def dumb_network_query(endpoint): responses = {} for b in beacons: - url = beacons[b].get("api_url") + url = beacons[b].get("apiUrl") try: r = network_beacon_post( url, @@ -76,7 +80,7 @@ def query(beacon_id, endpoint="overview"): if endpoint not in VALID_ENDPOINTS: raise NotFoundException() - api_url = beacon.get("api_url") + api_url = beacon.get("apiUrl") if request.method == "POST": payload = request.get_json() From b100bef2435dbe1c8b93025ca3c09cf2f57b02f9 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Tue, 11 Jun 2024 16:31:29 +0000 Subject: [PATCH 07/41] lint --- bento_beacon/network/network_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bento_beacon/network/network_config.py b/bento_beacon/network/network_config.py index 9d3001d9..1db3f230 100644 --- a/bento_beacon/network/network_config.py +++ b/bento_beacon/network/network_config.py @@ -1,5 +1,5 @@ # XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX -# temp file, all of this to be handled elsewhere in final version +# temp file, all of this to be handled elsewhere in final version # XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX BEACONS = [ From 69c72214c965db27259c80faac1b396a5e6b72c9 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Tue, 11 Jun 2024 19:15:53 +0000 Subject: [PATCH 08/41] workaround for flask url_prefix issues is to not use it --- bento_beacon/network/network.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bento_beacon/network/network.py b/bento_beacon/network/network.py index ef2c91c0..3c400fb5 100644 --- a/bento_beacon/network/network.py +++ b/bento_beacon/network/network.py @@ -6,7 +6,7 @@ from .utils import beacon_network_response, network_beacon_get, network_beacon_post from .network_config import VALID_ENDPOINTS, KATSU_CONFIG_UNION, KATSU_CONFIG_INTERSECTION -network = Blueprint("network", __name__, url_prefix="/network") +network = Blueprint("network", __name__) # TODOs: @@ -18,8 +18,8 @@ # handle GET args -@network.route("/") -@network.route("/beacons") +@network.route("/network") +@network.route("/network/beacons") def network_beacons(): beacons = current_app.config["NETWORK_BEACONS"] @@ -34,7 +34,7 @@ def network_beacons(): } -@network.route("/query/", methods=["POST"]) +@network.route("/network/query/", methods=["POST"]) def dumb_network_query(endpoint): """ Beacon network query in a single request and single response. @@ -70,8 +70,8 @@ def dumb_network_query(endpoint): return beacon_network_response(responses) -@network.route("/beacons/") -@network.route("/beacons//", methods=["GET", "POST"]) +@network.route("/network/beacons/") +@network.route("/network/beacons//", methods=["GET", "POST"]) def query(beacon_id, endpoint="overview"): beacon = current_app.config["NETWORK_BEACONS"].get(beacon_id) if not beacon: From ae63b282189bba94b14cbd598905f92a28eacdeb Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Mon, 17 Jun 2024 18:21:41 +0000 Subject: [PATCH 09/41] tinkering with querySections --- bento_beacon/network/network_config.py | 2141 ++++++++++++++++++++++-- bento_beacon/network/utils.py | 11 +- 2 files changed, 1963 insertions(+), 189 deletions(-) diff --git a/bento_beacon/network/network_config.py b/bento_beacon/network/network_config.py index 1db3f230..7cc1baed 100644 --- a/bento_beacon/network/network_config.py +++ b/bento_beacon/network/network_config.py @@ -10,30 +10,118 @@ "https://ichange.bento.sd4h.ca/api/beacon", # "https://qa.bento.sd4h.ca/api/beacon/", # "https://bentov2.local/api/beacon", + # "https://renata.bento.sd4h.ca/api/beacon", ] NETWORK_TIMEOUT = 30 VALID_ENDPOINTS = ["analyses", "biosamples", "cohorts", "datasets", "g_variants", "individuals", "runs", "overview"] -KATSU_CONFIG_UNION = [ + + +KATSU_CONFIG_INTERSECTION = [ { - "section_title": "General", + "section_title": "Common Filters", "fields": [ { - "mapping": "individual/age_numeric", - "title": "Age", - "description": "Age at arrival", - "datatype": "number", "config": { "bin_size": 10, + "maximum": 100, + "minimum": 0, "taper_left": 10, "taper_right": 100, - "units": "years", - "minimum": 0, + "units": "years" + }, + "datatype": "number", + "description": "Age at arrival", + "id": "age", + "mapping": "individual/age_numeric", + "options": [ + "[30, 40)", + "[40, 50)", + "[50, 60)", + "[60, 70)", + "[70, 80)", + "[80, 90)" + ], + "title": "Age" + }, + { + "config": { + "enum": None + }, + "datatype": "string", + "description": "Sex at birth", + "id": "sex", + "mapping": "individual/sex", + "options": [ + "MALE", + "FEMALE" + ], + "title": "Sex" + } + ] + } +] + + + + + + +KATSU_CONFIG_UNION = [ + { + "section_title": "All Filters", + "fields": [ + { + "config": { + "enum": None + }, + "datatype": "string", + "description": "Phenotypic features of the individual", + "id": "phenotypic_features_type", + "mapping": "individual/phenopackets/phenotypic_features/pftype/label", + "options": [ + "Asthma", + "Allergy", + "Rhinosinusitis", + "Overlap Syndrome", + "Tabagism", + "Asthma subject type", + "Phenotype", + "Loss of appetite", + "Abdominal pain", + "Headache", + "Cough", + "Cardiac arrest", + "Viral pneumonia/pneumonitis", + "Pulmonary hypertension", + "Nausea", + "HIV Infection", + "Hyperglycemia", + "Patient immunosuppressed", + "Fever", + "Chest pain", + "Pneumothorax", + "Dementia", + "Hypertension", + "Loss of sense of smell" + ], + "title": "Phenotypic Features" + }, + { + "config": { + "bin_size": 10, "maximum": 100, + "minimum": 0, + "taper_left": 10, + "taper_right": 100, + "units": "years" }, + "datatype": "number", + "description": "Age at arrival", "id": "age", + "mapping": "individual/age_numeric", "options": [ "< 10", "[10, 20)", @@ -45,24 +133,269 @@ "[70, 80)", "[80, 90)", "[90, 100)", + "< 18", + "[18, 30)", + "≥ 90" ], + "title": "Age" }, { - "mapping": "individual/sex", - "title": "Sex", - "description": "Sex at birth", + "config": { + "enum": None + }, "datatype": "string", - "config": {"enum": None}, + "description": "Sex at birth", "id": "sex", - "options": ["MALE", "FEMALE"], + "mapping": "individual/sex", + "options": [ + "MALE", + "FEMALE", + "UNKNOWN_SEX" + ], + "title": "Sex" }, { - "mapping": "individual/extra_properties/date_of_consent", - "title": "Verbal consent date", - "description": "Date of initial verbal consent (participant, legal representative or tutor), yyyy-mm-dd", + "config": { + "enum": None + }, + "datatype": "string", + "description": "Measurements performed", + "group_by": "assay/label", + "id": "measurement_types", + "mapping": "individual/phenopackets/measurements", + "options": [ + "Body Mass Index", + "Cp20 Measurement", + "Eosinophils Percentage", + "Histological Eosinophils", + "Histological Neutrophils", + "Immunoglobulin E", + "Neutrophils Percentage" + ], + "title": "Measurement types" + }, + { + "config": { + "bin_size": "10", + "maximum": 50, + "minimum": 0, + "taper_left": 0, + "taper_right": 50, + "units": "kg/m^2" + }, + "datatype": "number", + "description": "Measurements performed", + "group_by": "assay/id", + "group_by_value": "NCIT:C16358", + "id": "measurement_bmi", + "mapping": "individual/phenopackets/measurements", + "options": [ + "[0, 10)", + "[10, 20)", + "[20, 30)", + "[30, 40)", + "[40, 50)", + "< 18", + "[18, 30)", + "≥ 30" + ], + "title": "BMI", + "value_mapping": "value/quantity/value" + }, + { + "config": { + "enum": None + }, + "datatype": "string", + "description": "Visits where measurements were performed", + "group_by": "extra_properties/visit_index", + "id": "measurement_visits", + "mapping": "individual/phenopackets/measurements", + "options": [ + "1", + "10", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "18", + "19", + "2", + "20", + "21", + "22", + "23", + "24", + "25", + "26", + "27", + "28", + "29", + "3", + "30", + "31", + "32", + "33", + "4", + "5", + "6", + "7", + "8", + "9" + ], + "title": "Recurring visits" + }, + { + "config": { + "bin_size": 50, + "maximum": 1000, + "minimum": 0, + "taper_left": 50, + "taper_right": 500, + "units": "years" + }, + "datatype": "number", + "description": "Measurements performed x", + "group_by": "assay/label", + "group_by_value": "Immunoglobulin E", + "id": "measurement_immnoglobulin_e", + "mapping": "individual/phenopackets/measurements", + "options": [ + "< 50", + "[50, 100)", + "[100, 150)", + "[150, 200)", + "[200, 250)", + "[250, 300)", + "[300, 350)", + "[350, 400)", + "[400, 450)", + "[450, 500)", + "≥ 500" + ], + "title": "Immunoglobulin E", + "value_mapping": "value/quantity/value" + }, + { + "config": { + "bin_size": 10, + "maximum": 1000, + "minimum": 0, + "taper_left": 10, + "taper_right": 100, + "units": "years" + }, + "datatype": "number", + "description": "Measurements performed Cp20", + "group_by": "assay/label", + "group_by_value": "Cp20 Measurement", + "id": "measurement_Cp20", + "mapping": "individual/phenopackets/measurements", + "options": [ + "< 10", + "[10, 20)", + "[20, 30)", + "[30, 40)", + "[40, 50)", + "[50, 60)", + "[60, 70)", + "[70, 80)", + "[80, 90)", + "[90, 100)", + "≥ 100" + ], + "title": "Cp20", + "value_mapping": "value/quantity/value" + }, + { + "config": { + "enum": None + }, + "datatype": "string", + "description": "Tissue from which the specimen was collected", + "id": "sampled_tissue", + "mapping": "individual/phenopackets/biosamples/sampled_tissue/label", + "options": [ + "Sputum-DTT plug", + "Cells from nasal brush-with reagent", + "Bronchoalveolar lavage supernatant aliquot", + "Sputum slide", + "Whole blood RNA", + "Blood cells", + "EDTA arterial plasma aliquot", + "Blood cells pellet", + "Nasal lavage or nasosorption supernatant", + "Urine aliquot", + "Sputum cells pellet", + "Nasal mucosa biopsy", + "DTT sputum supernatant aliquot", + "Cellules mononuclées isolées de sang périphérique (PBMC)", + "Buffy coat aliquot", + "Serum aliquot", + "Stools", + "Bronchial biopsies RNA", + "Plasma aliquot", + "Cells from nasal brush-dry", + "Buffycoat DNA", + "Nasal polyp biopsy", + "Macrophages RNA from bronchoalveolar lavage", + "Nasal secretion swab", + "EDTA whole blood aliquot", + "Sputum cells RNA", + "Bronchoalveolar lavage slide", + "Bronchial biopsy", + "Sputum supernatant aliquot without DTT", + "Venous serum aliquot", + "Optic Pathway", + "Parietal Lobe", + "Hypothalamus", + "Fronto-Temporal Lobe", + "Frontal Lobe", + "Cerebellum", + "Fronto-Parietal Lobe", + "Temporo-Occipital Lobe", + "Temporo-Parietal Lobe", + "Intraventricular", + "Fronto-Temporo-Insular", + "Unknown", + "Diencephalon", + "Brainstem", + "Cell Line", + "Supratentorial", + "Pineal", + "Thalamus", + "Ventricle", + "Hemisphere", + "Temporal Lobe", + "Ventricle (Lateral)", + "Suprasellar", + "Occipital Lobe", + "Mandible", + "Medulla Oblongata", + "Ventricle (Fourth)", + "Cerebellar Vermis", + "Ventricle (Third)", + "Cortex", + "Spinal Cord", + "Posterior Fossa", + "Optic Chiasm", + "Cerebello-Pontine Angle", + "Pons", + "Parieto-Occipital Lobe" + ], + "title": "Biosample Tissue Location" + }, + { + "config": { + "bin_by": "month" + }, "datatype": "date", - "config": {"bin_by": "month"}, + "description": "Date of initial verbal consent (participant, legal representative or tutor), yyyy-mm-dd", "id": "date_of_consent", + "mapping": "individual/extra_properties/date_of_consent", "options": [ "Jan 2020", "Feb 2020", @@ -111,115 +444,183 @@ "Sep 2023", "Oct 2023", "Nov 2023", - "Dec 2023", + "Dec 2023" ], + "title": "Verbal consent date" }, { - "mapping": "individual/extra_properties/mobility", - "title": "Functional status", - "description": "Mobility", - "datatype": "string", "config": { "enum": [ "I have no problems in walking about", "I have slight problems in walking about", "I have moderate problems in walking about", "I have severe problems in walking about", - "I am unable to walk about", + "I am unable to walk about" ] }, + "datatype": "string", + "description": "Mobility", "id": "mobility", + "mapping": "individual/extra_properties/mobility", "options": [ "I have no problems in walking about", "I have slight problems in walking about", "I have moderate problems in walking about", "I have severe problems in walking about", - "I am unable to walk about", + "I am unable to walk about" ], + "title": "Functional status" }, { - "mapping": "individual/extra_properties/covid_severity", - "title": "Covid severity", - "description": "Covid severity", + "config": { + "enum": [ + "Uninfected", + "Mild", + "Moderate", + "Severe", + "Dead" + ] + }, "datatype": "string", - "config": {"enum": ["Uninfected", "Mild", "Moderate", "Severe", "Dead"]}, + "description": "Covid severity", "id": "covid_severity", - "options": ["Uninfected", "Mild", "Moderate", "Severe", "Dead"], - }, - { - "mapping": "individual/phenopackets/phenotypic_features/pftype/label", - "title": "Phenotypic Features", - "description": "Individual phenotypic features, observed as either present or absent", - "datatype": "string", - "config": {"enum": None}, - "id": "phenotypic_features", + "mapping": "individual/extra_properties/covid_severity", "options": [ - "Loss of appetite", - "Asthma", - "Abdominal pain", - "Headache", - "Cough", - "Cardiac arrest", - "Nausea", - "Pulmonary hypertension", - "Viral pneumonia/pneumonitis", - "HIV Infection", - "Hyperglycemia", - "Patient immunosuppressed", - "Fever", - "Chest pain", - "Pneumothorax", - "Hypertension", - "Dementia", - "Loss of sense of smell", + "Uninfected", + "Mild", + "Moderate", + "Severe", + "Dead" ], + "title": "Covid severity" }, { - "mapping": "individual/phenopackets/diseases/term/label", - "title": "Diseases", - "description": "Diseases observed as either present or absent", + "config": { + "enum": None + }, "datatype": "string", - "config": {"enum": None}, + "description": "Diseases observed as either present or absent", "id": "diseases", + "mapping": "individual/phenopackets/diseases/term/label", "options": [ "viral pneumonia", - "Glioblastoma", "Polycythemia vera", - "Miller syndrome", - "Cystic fibrosis", - "coronary artery disease, autosomal dominant, 1", + "Non-Hodgkin Lymphoma", "Hashimoto Thyroiditis", + "diabetes mellitus", + "Glioblastoma", + "coronary artery disease, autosomal dominant, 1", + "COVID-19", "Rheumatologic disease", + "Marfan syndrome", + "Miller syndrome", + "Cystic fibrosis", "Takotsubo cardiomyopathy", "autoimmune disorder of cardiovascular system", - "COVID-19", - "Marfan syndrome", - "Non-Hodgkin Lymphoma", - "diabetes mellitus", + "Negative", + "Medulloblastoma", + "Testicular Cancer", + "Sarcoma", + "DIPG", + "Ovarian Tumor", + "Control", + "Neurofibromatosis", + "Desmoplastic infantile ganglioglioma (DIG)", + "GBM", + "HGG", + "Desmoplastic Infantile Astrocytoma (DIA)", + "Melanoma", + "Acute Lymphocytic Leukemia (ALL)", + "Ependymoma-PFB", + "Germinoma", + "Normal Brain", + "Schwannoma", + "Giant Cell Lesion (central)", + "Anaplastic Astrocytoma (AA)", + "Odontogenic Myxoma", + "Glioneuronal Tumor", + "Oligodendroglioma GrII", + "Craniopharyngioma", + "Ganglioglioma", + "Anaplastic Ependymoma", + "Brain Tumor", + "Neurofibroma", + "ATRT", + "Pilocytic Astrocytoma", + "Ependymoma-PFA", + "Angiosarcoma", + "DNET", + "Cemento Ossifying Fibroma", + "Meningioma", + "Diffuse Astrocytoma", + "Oral Granular Cell Tumor (OGCT)", + "Ewing's Sarcoma", + "HNSCC", + "Neuroblastoma", + "Acute Myeloid Leukemia (AML)", + "Parathyroid Tumor", + "Diffuse midline glioma, H3 K27M-mutant", + "Giant Cell Tumor (GCT)", + "Ependymoma - myxopapillary", + "Rhabdomyosarcoma", + "Glioma", + "Epithelioid Hemangioendothelioma", + "Endometrial Sarcoma", + "Giant Cell Lesion (peripheral)", + "Low Grade Glioma (LGG)", + "Astrocytoma", + "ETMR", + "Subcutaneous Panniculitis-like T-cell lymphoma", + "Giant Cell lesion", + "PNET", + "Anaplastic Oligoastrocytoma", + "Osteosarcoma", + "Pilomyxoid Astrocytoma (PMA)", + "Pleomorphic Xanthoastrocytoma (PXA)", + "Immunodeficiency", + "Cortical Dysplasia", + "Oligoastrocytoma", + "Anaplastic Oligodendroglioma", + "Ependymoma", + "Choroid Plexus Carcinoma", + "Oral Squamous Cell Carcinoma (OSCC)", + "Chondroblastoma" ], + "title": "Diseases" }, { - "mapping": "biosample/sampled_tissue/label", - "mapping_for_search_filter": "individual/biosamples/sampled_tissue/label", - "title": "Sampled Tissues", - "description": "Tissue from which the biosample was extracted", + "config": { + "enum": None + }, "datatype": "string", - "config": {"enum": None}, + "description": "Tissue from which the biosample was extracted", "id": "tissues", - "options": ["Plasma", "blood", "Serum"], + "mapping": "biosample/sampled_tissue/label", + "mapping_for_search_filter": "individual/biosamples/sampled_tissue/label", + "options": [ + "Plasma", + "blood", + "Serum" + ], + "title": "Sampled Tissues" }, - ], - }, - { - "section_title": "Measurements", - "fields": [ { - "mapping": "individual/extra_properties/lab_test_result_value", - "title": "Lab Test Result", - "description": "Numeric measures from a laboratory test", + "config": { + "bins": [ + 200, + 300, + 500, + 1000, + 1500, + 2000 + ], + "minimum": 0, + "units": "mg/L" + }, "datatype": "number", - "config": {"bins": [200, 300, 500, 1000, 1500, 2000], "minimum": 0, "units": "mg/L"}, + "description": "Numeric measures from a laboratory test", "id": "lab_test_result_value", + "mapping": "individual/extra_properties/lab_test_result_value", "options": [ "< 200", "[200, 300)", @@ -227,88 +628,79 @@ "[500, 1000)", "[1000, 1500)", "[1500, 2000)", - "≥ 2000", + "≥ 2000" ], + "title": "Lab Test Result" }, { - "mapping": "individual/phenopackets/measurements", - "group_by": "assay/id", - "group_by_value": "NCIT:C16358", - "value_mapping": "value/quantity/value", - "title": "BMI", - "description": "Body Mass Index", - "datatype": "number", - "config": {"bins": [18.5, 30], "minimum": 0, "units": "kg/m^2"}, - "id": "bmi", - "options": ["< 18", "[18, 30)", "≥ 30"], - }, - ], - }, - { - "section_title": "Medical Actions", - "fields": [ - { - "mapping": "individual/phenopackets/medical_actions", - "group_by": "procedure/code/label", - "title": "Medical Procedures", - "description": "A clinical procedure performed on a subject", + "config": { + "enum": None + }, "datatype": "string", - "config": {"enum": None}, + "description": "A clinical procedure performed on a subject", + "group_by": "procedure/code/label", "id": "medical_procedures", + "mapping": "individual/phenopackets/medical_actions", "options": [ "Liver Biopsy", "Magnetic Resonance Imaging", "Positron Emission Tomography", "Punch Biopsy", - "X-Ray Imaging", + "X-Ray Imaging" ], + "title": "Medical Procedures" }, { - "mapping": "individual/phenopackets/medical_actions", - "group_by": "treatment/agent/label", - "title": "Medical Treatments", - "description": "Treatment with an agent such as a drug", + "config": { + "enum": None + }, "datatype": "string", - "config": {"enum": None}, + "description": "Treatment with an agent such as a drug", + "group_by": "treatment/agent/label", "id": "medical_treatments", - "options": ["Acetaminophen", "Ibuprofen", "Ondansetron"], + "mapping": "individual/phenopackets/medical_actions", + "options": [ + "Acetaminophen", + "Ibuprofen", + "NCIT:C1119" + ], + "title": "Medical Treatments" }, - ], - }, - { - "section_title": "Interpretations", - "fields": [ { - "mapping": "individual/phenopackets/interpretations/diagnosis/genomic_interpretations/interpretation_status", - "title": "Genomic Interpretations", - "description": "Interpretation for an individual variant or gene (CANDIDATE, CONTRIBUTORY, etc)", + "config": { + "enum": None + }, "datatype": "string", - "config": {"enum": None}, + "description": "Interpretation for an individual variant or gene (CANDIDATE, CONTRIBUTORY, etc)", "id": "interpretation_status", - "options": ["CONTRIBUTORY"], + "mapping": "individual/phenopackets/interpretations/diagnosis/genomic_interpretations/interpretation_status", + "options": [ + "CONTRIBUTORY" + ], + "title": "Genomic Interpretations" }, { - "mapping": "individual/phenopackets/interpretations/diagnosis/genomic_interpretations/variant_interpretation/acmg_pathogenicity_classification", - "title": "Variant Pathogenicity", - "description": "ACMG Pathogenicity category for a particular variant (BENIGN, PATHOGENIC, etc)", + "config": { + "enum": None + }, "datatype": "string", - "config": {"enum": None}, + "description": "ACMG Pathogenicity category for a particular variant (BENIGN, PATHOGENIC, etc)", "id": "acmg_pathogenicity_classification", - "options": ["PATHOGENIC"], + "mapping": "individual/phenopackets/interpretations/diagnosis/genomic_interpretations/variant_interpretation/acmg_pathogenicity_classification", + "options": [ + "PATHOGENIC" + ], + "title": "Variant Pathogenicity" }, - ], - }, - { - "section_title": "Experiments", - "fields": [ { - "mapping": "experiment/experiment_type", - "mapping_for_search_filter": "individual/biosamples/experiment/experiment_type", - "title": "Experiment Types", - "description": "Types of experiments performed on a sample", + "config": { + "enum": None + }, "datatype": "string", - "config": {"enum": None}, + "description": "Types of experiments performed on a sample", "id": "experiment_type", + "mapping": "experiment/experiment_type", + "mapping_for_search_filter": "individual/biosamples/experiment/experiment_type", "options": [ "Other", "Neutralizing antibody titers", @@ -316,72 +708,1451 @@ "RNA-Seq", "WGS", "Proteomic profiling", + "WES" ], + "title": "Experiment Types" }, { - "mapping": "experiment/study_type", - "mapping_for_search_filter": "individual/biosamples/experiment/study_type", - "title": "Study Types", - "description": "Study type of the experiment (e.g. Genomics, Transcriptomics, etc.)", + "config": { + "enum": None + }, "datatype": "string", - "config": {"enum": None}, + "description": "Study type of the experiment (e.g. Genomics, Transcriptomics, etc.)", "id": "experiment_study_type", - "options": ["Serology", "Genomics", "Other", "Proteomics", "Transcriptomics", "Metabolomics"], + "mapping": "experiment/study_type", + "mapping_for_search_filter": "individual/biosamples/experiment/study_type", + "options": [ + "Serology", + "Other", + "Genomics", + "Proteomics", + "Transcriptomics", + "Metabolomics" + ], + "title": "Study Types" }, { + "config": { + "enum": None + }, + "datatype": "string", + "description": "File type of experiment results files", + "id": "experiment_results_file_type", "mapping": "experiment/experiment_results/file_format", "mapping_for_search_filter": "individual/biosamples/experiment/experiment_results/file_format", - "title": "Results File Types", - "description": "File type of experiment results files", + "options": [ + "PDF", + "XLSX", + "JPEG", + "CRAM", + "VCF", + "MP4", + "DOCX", + "CSV", + "MARKDOWN" + ], + "title": "Results File Types" + }, + { + "config": { + "bin_by": "month" + }, + "datatype": "date", + "description": "Date of initial verbal consent (participant, legal representative or tutor)", + "id": "dconsverbpa", + "mapping": "individual/extra_properties/dconsverbpa", + "options": [ + "Mar 2020", + "Apr 2020", + "May 2020", + "Jun 2020", + "Jul 2020", + "Aug 2020", + "Sep 2020", + "Oct 2020", + "Nov 2020", + "Dec 2020", + "Jan 2021", + "Feb 2021", + "Mar 2021", + "Apr 2021", + "May 2021", + "Jun 2021", + "Jul 2021", + "Aug 2021", + "Sep 2021", + "Oct 2021", + "Nov 2021", + "Dec 2021", + "Jan 2022", + "Feb 2022", + "Mar 2022", + "Apr 2022", + "May 2022", + "Jun 2022", + "Jul 2022", + "Aug 2022", + "Sep 2022", + "Oct 2022", + "Nov 2022", + "Dec 2022", + "Jan 2023", + "Feb 2023", + "Mar 2023", + "Apr 2023", + "May 2023", + "Jun 2023", + "Jul 2023", + "Aug 2023" + ], + "title": "Verbal consent date" + }, + { + "config": { + "enum": [ + "Hospitalized", + "Outpatient" + ] + }, "datatype": "string", - "config": {"enum": None}, - "id": "experiment_results_file_type", - "options": ["XLSX", "CRAM", "VCF", "JPEG", "MP4", "DOCX", "CSV", "MARKDOWN"], + "description": "Has the participant been hospitalized or is the participant seen as an outpatient?", + "id": "type_partic", + "mapping": "individual/extra_properties/type_partic", + "options": [ + "Hospitalized", + "Outpatient" + ], + "title": "Hospitalization" }, - ], - }, -] - -KATSU_CONFIG_INTERSECTION = [ - { - "section_title": "General", - "fields": [ { - "mapping": "individual/age_numeric", - "title": "Age", - "description": "Age at arrival", + "config": { + "enum": [ + "Non-smoker", + "Smoker", + "Former smoker", + "Passive smoker", + "Not specified" + ] + }, + "datatype": "string", + "description": "Smoking status", + "id": "smoking", + "mapping": "individual/extra_properties/smoking", + "options": [ + "Non-smoker", + "Smoker", + "Former smoker", + "Passive smoker", + "Not specified" + ], + "title": "Smoking" + }, + { + "config": { + "enum": [ + "Positive", + "Negative", + "Indeterminate" + ] + }, + "datatype": "string", + "description": "Final COVID status according to the PCR test", + "id": "covidstatus", + "mapping": "individual/extra_properties/covidstatus", + "options": [ + "Positive", + "Negative", + "Indeterminate" + ], + "title": "COVID status" + }, + { + "config": { + "enum": [ + "Alive", + "Deceased" + ] + }, + "datatype": "string", + "description": "Vital status at discharge", + "id": "death_dc", + "mapping": "individual/extra_properties/death_dc", + "options": [ + "Alive", + "Deceased" + ], + "title": "Vital status" + }, + { + "config": { + "bins": [ + 20, + 25, + 27, + 30, + 35, + 40 + ], + "units": "kg/m^2" + }, "datatype": "number", + "description": "BMI", + "id": "bmi", + "mapping": "individual/extra_properties/bmi", + "options": [ + "< 20", + "[20, 25)", + "[25, 27)", + "[27, 30)", + "[30, 35)", + "[35, 40)", + "≥ 40" + ], + "title": "BMI" + }, + { "config": { - "bin_size": 10, - "taper_left": 10, - "taper_right": 100, - "units": "years", - "minimum": 0, - "maximum": 100, + "enum": [ + "Yes", + "No" + ] }, - "id": "age", + "datatype": "string", + "description": "Intensive Care Unit admission?", + "id": "icu", + "mapping": "individual/extra_properties/icu", "options": [ - "< 10", - "[10, 20)", - "[20, 30)", - "[30, 40)", - "[40, 50)", - "[50, 60)", - "[60, 70)", - "[70, 80)", - "[80, 90)", - "[90, 100)", + "Yes", + "No" ], + "title": "ICU" }, { - "mapping": "individual/sex", - "title": "Sex", - "description": "Sex at birth", + "config": { + "enum": [ + "Hospitalized adult", + "Adult outpatient", + "Pediatric", + "Pregnant woman" + ] + }, "datatype": "string", - "config": {"enum": None}, - "id": "sex", - "options": ["MALE", "FEMALE"], + "description": "To which category the participant belongs?", + "id": "core_cohorte", + "mapping": "individual/extra_properties/core_cohorte", + "options": [ + "Hospitalized adult", + "Adult outpatient", + "Pediatric", + "Pregnant woman" + ], + "title": "Participant category" }, - ], - }, -] + { + "config": { + "enum": [ + "At home", + "In residence for the elderly (RPA)", + "Nursing home (CHSLD)", + "In intermediate and family-type resources", + "In rooming house", + "Homeless" + ] + }, + "datatype": "string", + "description": "Living where?", + "id": "livewhere", + "mapping": "individual/extra_properties/livewhere", + "options": [ + "At home", + "In residence for the elderly (RPA)", + "Nursing home (CHSLD)", + "In intermediate and family-type resources", + "In rooming house", + "Homeless" + ], + "title": "Domicile" + }, + { + "config": { + "enum": [ + "Yes", + "No" + ] + }, + "datatype": "string", + "description": "Has the participant been vaccinated?", + "id": "vaccinate", + "mapping": "individual/extra_properties/vaccinate", + "options": [ + "Yes", + "No" + ], + "title": "Vaccinal status" + }, + { + "config": { + "enum": [ + "1", + "2", + "3", + "4" + ] + }, + "datatype": "string", + "description": "Number of doses received", + "id": "vaccin_dosenum", + "mapping": "individual/extra_properties/vaccin_dosenum", + "options": [ + "1", + "2", + "3", + "4" + ], + "title": "Vaccine dose" + }, + { + "config": { + "enum": [ + "I have no problem washing or dressing myself", + "I have slight problem washing or dressing myself", + "I have moderate problems washing or dressing myself", + "I have severe problems washing or dressing myself", + "I am unable to wash or dress myself" + ] + }, + "datatype": "string", + "description": "Self-care", + "id": "selfcare", + "mapping": "individual/extra_properties/selfcare", + "options": [ + "I have no problem washing or dressing myself", + "I have slight problem washing or dressing myself", + "I have moderate problems washing or dressing myself", + "I have severe problems washing or dressing myself", + "I am unable to wash or dress myself" + ], + "title": "Self-care" + }, + { + "config": { + "enum": [ + "Yes", + "No" + ] + }, + "datatype": "string", + "description": "Other chronic cardiac disease?", + "id": "phx_cardiac", + "mapping": "individual/extra_properties/phx_cardiac", + "options": [ + "Yes", + "No" + ], + "title": "Cardiac history" + }, + { + "config": { + "enum": [ + "Yes", + "No" + ] + }, + "datatype": "string", + "description": "Prior transient ischemic attack (TIA)?", + "id": "phx_tia", + "mapping": "individual/extra_properties/phx_tia", + "options": [ + "Yes", + "No" + ], + "title": "TIA" + }, + { + "config": { + "enum": [ + "Yes", + "No" + ] + }, + "datatype": "string", + "description": "Asthma?", + "id": "phx_asthma", + "mapping": "individual/extra_properties/phx_asthma", + "options": [ + "Yes", + "No" + ], + "title": "Asthma" + }, + { + "config": { + "enum": [ + "Yes", + "No" + ] + }, + "datatype": "string", + "description": "Prior stroke?", + "id": "phx_cva", + "mapping": "individual/extra_properties/phx_cva", + "options": [ + "Yes", + "No" + ], + "title": "Stroke" + }, + { + "config": { + "enum": [ + "Yes", + "No" + ] + }, + "datatype": "string", + "description": "Diabetes?", + "id": "phx_diabetes", + "mapping": "individual/extra_properties/phx_diabetes", + "options": [ + "Yes", + "No" + ], + "title": "Diabetes" + }, + { + "config": { + "enum": [ + "Yes", + "No" + ] + }, + "datatype": "string", + "description": "Malignant neoplasm?", + "id": "phx_cancer", + "mapping": "individual/extra_properties/phx_cancer", + "options": [ + "Yes", + "No" + ], + "title": "Cancer" + }, + { + "config": { + "enum": [ + "Yes", + "No" + ] + }, + "datatype": "string", + "description": "Dementia?", + "id": "phx_dementia", + "mapping": "individual/extra_properties/phx_dementia", + "options": [ + "Yes", + "No" + ], + "title": "Dementia" + }, + { + "config": { + "enum": [ + "Yes", + "No" + ] + }, + "datatype": "string", + "description": "Atrial fibrillation or flutter?", + "id": "phx_afib", + "mapping": "individual/extra_properties/phx_afib", + "options": [ + "Yes", + "No" + ], + "title": "Atrial fibrillation" + }, + { + "config": { + "enum": [ + "Yes", + "No" + ] + }, + "datatype": "string", + "description": "HIV or AIDS?", + "id": "phx_hiv", + "mapping": "individual/extra_properties/phx_hiv", + "options": [ + "Yes", + "No" + ], + "title": "HIV" + }, + { + "config": { + "enum": [ + "Yes", + "No" + ] + }, + "datatype": "string", + "description": "Arterial Hypertension?", + "id": "phx_htn", + "mapping": "individual/extra_properties/phx_htn", + "options": [ + "Yes", + "No" + ], + "title": "Arterial Hypertension" + }, + { + "config": { + "enum": [ + "Yes", + "No" + ] + }, + "datatype": "string", + "description": "Immunosuppressed state?", + "id": "phx_imm", + "mapping": "individual/extra_properties/phx_imm", + "options": [ + "Yes", + "No" + ], + "title": "Immunosupression" + }, + { + "config": { + "enum": [ + "Yes", + "No" + ] + }, + "datatype": "string", + "description": "Prior myocardial infarction?", + "id": "phx_mi", + "mapping": "individual/extra_properties/phx_mi", + "options": [ + "Yes", + "No" + ], + "title": "Myocardial infarction" + }, + { + "config": { + "enum": [ + "Yes", + "No" + ] + }, + "datatype": "string", + "description": "Heart failure?", + "id": "phx_chf", + "mapping": "individual/extra_properties/phx_chf", + "options": [ + "Yes", + "No" + ], + "title": "Heart failure" + }, + { + "config": { + "enum": [ + "Yes", + "No" + ] + }, + "datatype": "string", + "description": "Coronary artery disease?", + "id": "phx_cad", + "mapping": "individual/extra_properties/phx_cad", + "options": [ + "Yes", + "No" + ], + "title": "Coronary artery disease" + }, + { + "config": { + "enum": [ + "Yes", + "No" + ] + }, + "datatype": "string", + "description": "Liver disease?", + "id": "phx_liver", + "mapping": "individual/extra_properties/phx_liver", + "options": [ + "Yes", + "No" + ], + "title": "Liver disease" + }, + { + "config": { + "enum": [ + "Yes", + "No" + ] + }, + "datatype": "string", + "description": "Chronic obstructive pulmonary disease (emphysema, chronic bronchitis)?", + "id": "phx_copd", + "mapping": "individual/extra_properties/phx_copd", + "options": [ + "Yes", + "No" + ], + "title": "COPD" + }, + { + "config": { + "enum": [ + "Yes", + "No" + ] + }, + "datatype": "string", + "description": "Psychiatric disease?", + "id": "phx_psych", + "mapping": "individual/extra_properties/phx_psych", + "options": [ + "Yes", + "No" + ], + "title": "Psychiatric disease" + }, + { + "config": { + "enum": [ + "Yes", + "No" + ] + }, + "datatype": "string", + "description": "Chronic kidney disease?", + "id": "phx_ckd", + "mapping": "individual/extra_properties/phx_ckd", + "options": [ + "Yes", + "No" + ], + "title": "Chronic kidney disease" + }, + { + "config": { + "enum": [ + "Yes", + "No" + ] + }, + "datatype": "string", + "description": "Dialysis?", + "id": "phx_dialysis", + "mapping": "individual/extra_properties/phx_dialysis", + "options": [ + "Yes", + "No" + ], + "title": "Dialysis" + }, + { + "config": { + "enum": [ + "Yes", + "No" + ] + }, + "datatype": "string", + "description": "Rheumatologic disease?", + "id": "phx_rheum", + "mapping": "individual/extra_properties/phx_rheum", + "options": [ + "Yes", + "No" + ], + "title": "Rheumatologic disease" + }, + { + "config": { + "bins": [ + 5, + 10, + 50, + 100 + ], + "units": "mg/L" + }, + "datatype": "number", + "description": "C-reactive protein (CRP)", + "id": "lab_crp", + "mapping": "individual/extra_properties/lab_crp", + "options": [ + "< 5", + "[5, 10)", + "[10, 50)", + "[50, 100)", + "≥ 100" + ], + "title": "CRP" + }, + { + "config": { + "enum": [ + "Yes", + "No" + ] + }, + "datatype": "string", + "description": "Chest X-ray?", + "id": "cxr", + "mapping": "individual/extra_properties/cxr", + "options": [ + "Yes", + "No" + ], + "title": "Chest X-ray" + }, + { + "config": { + "bins": [ + 4, + 10, + 15 + ], + "units": "×10^9/L" + }, + "datatype": "number", + "description": "Total White Blood Cell (WBC) count", + "id": "lab_wbc", + "mapping": "individual/extra_properties/lab_wbc", + "options": [ + "< 4", + "[4, 10)", + "[10, 15)", + "≥ 15" + ], + "title": "WBC" + }, + { + "config": { + "bins": [ + 70, + 90, + 110, + 130, + 150 + ], + "units": "g/L" + }, + "datatype": "number", + "description": "Haemoglobin", + "id": "lab_hg", + "mapping": "individual/extra_properties/lab_hg", + "options": [ + "< 70", + "[70, 90)", + "[90, 110)", + "[110, 130)", + "[130, 150)", + "≥ 150" + ], + "title": "Haemoglobin" + }, + { + "config": { + "bins": [ + 50, + 90, + 120, + 200, + 300 + ], + "units": "μmol/L" + }, + "datatype": "number", + "description": "Creatinine", + "id": "lab_cr", + "mapping": "individual/extra_properties/lab_cr", + "options": [ + "< 50", + "[50, 90)", + "[90, 120)", + "[120, 200)", + "[200, 300)", + "≥ 300" + ], + "title": "Creatinine" + }, + { + "config": { + "bins": [ + 200, + 300, + 500, + 1000, + 1500, + 2000 + ], + "units": "ng/mL" + }, + "datatype": "number", + "description": "NT-proBNP", + "id": "lab_ntprobnp", + "mapping": "individual/extra_properties/lab_ntprobnp", + "options": [ + "< 200", + "[200, 300)", + "[300, 500)", + "[500, 1000)", + "[1000, 1500)", + "[1500, 2000)", + "≥ 2000" + ], + "title": "NT-proBNP" + }, + { + "config": { + "bins": [ + 200, + 500, + 1000, + 1500, + 2000 + ], + "units": "U/L" + }, + "datatype": "number", + "description": "Lactate Dehydrogenase", + "id": "lab_ldh", + "mapping": "individual/extra_properties/lab_ldh", + "options": [ + "< 200", + "[200, 500)", + "[500, 1000)", + "[1000, 1500)", + "[1500, 2000)", + "≥ 2000" + ], + "title": "LDH" + }, + { + "config": { + "bins": [ + 500, + 1000, + 2000, + 5000 + ], + "units": "μg/L" + }, + "datatype": "number", + "description": "D-Dimer", + "id": "lab_ddimer", + "mapping": "individual/extra_properties/lab_ddimer", + "options": [ + "< 500", + "[500, 1000)", + "[1000, 2000)", + "[2000, 5000)", + "≥ 5000" + ], + "title": "D-Dimer" + }, + { + "config": { + "enum": [ + "Yes", + "No" + ] + }, + "datatype": "string", + "description": "Did or does the patient receive ventilatory support?", + "id": "repsupport_yesno", + "mapping": "individual/extra_properties/repsupport_yesno", + "options": [ + "Yes", + "No" + ], + "title": "Respiratory support" + }, + { + "config": { + "enum": [ + "Yes", + "No" + ] + }, + "datatype": "string", + "description": "Have you been diagnosed with a new or recurrent case of COVID since your last follow-up?", + "id": "newcovid", + "mapping": "individual/extra_properties/newcovid", + "options": [ + "Yes", + "No" + ], + "title": "Reinfection" + }, + { + "config": { + "enum": [ + "Yes", + "No" + ] + }, + "datatype": "string", + "description": "Does the participant report persistent symptoms related to SARS-CoV-2 infection?", + "id": "sx_report", + "mapping": "individual/extra_properties/sx_report", + "options": [ + "Yes", + "No" + ], + "title": "Persisting symptoms" + }, + { + "config": { + "enum": [ + "Yes", + "No", + "Not available" + ] + }, + "datatype": "string", + "description": "Systemic corticosteroid?", + "id": "rx_roid", + "mapping": "individual/extra_properties/rx_roid", + "options": [ + "Yes", + "No", + "Not available" + ], + "title": "Systemic corticosteroid" + }, + { + "config": { + "enum": [ + "Yes", + "No", + "Not available" + ] + }, + "datatype": "string", + "description": "Antocoagulants?", + "id": "rx_aco", + "mapping": "individual/extra_properties/rx_aco", + "options": [ + "Yes", + "No", + "Not available" + ], + "title": "Antocoagulants" + }, + { + "config": { + "enum": [ + "Worse", + "Same", + "Better", + "Indeterminate" + ] + }, + "datatype": "string", + "description": "Ability to self-care at discharge versus pre-COVID", + "id": "selfcare_dc", + "mapping": "individual/extra_properties/selfcare_dc", + "options": [ + "Worse", + "Same", + "Better", + "Indeterminate" + ], + "title": "Self-care post covid" + }, + { + "config": { + "enum": [ + "Yes", + "No" + ] + }, + "datatype": "string", + "description": "Acute kidney injury?", + "id": "c_aki", + "mapping": "individual/extra_properties/c_aki", + "options": [ + "Yes", + "No" + ], + "title": "Acute kidney injury" + }, + { + "config": { + "enum": [ + "Yes", + "No" + ] + }, + "datatype": "string", + "description": "Acute Respiratory Distress Syndrome (ARDS)?", + "id": "c_ards", + "mapping": "individual/extra_properties/c_ards", + "options": [ + "Yes", + "No" + ], + "title": "ARDS" + }, + { + "config": { + "enum": [ + "Yes", + "No" + ] + }, + "datatype": "string", + "description": "Deep vein thrombosis (DVT)?", + "id": "c_dvt", + "mapping": "individual/extra_properties/c_dvt", + "options": [ + "Yes", + "No" + ], + "title": "Deep vein thrombosis" + }, + { + "config": { + "enum": [ + "Yes", + "No" + ] + }, + "datatype": "string", + "description": "Stroke?", + "id": "c_stroke", + "mapping": "individual/extra_properties/c_stroke", + "options": [ + "Yes", + "No" + ], + "title": "Stroke" + }, + { + "config": { + "enum": [ + "Yes", + "No" + ] + }, + "datatype": "string", + "description": "Pulmonary embolism?", + "id": "c_pe", + "mapping": "individual/extra_properties/c_pe", + "options": [ + "Yes", + "No" + ], + "title": "Pulmonary embolism" + }, + { + "config": { + "enum": None + }, + "datatype": "string", + "description": "The type of molecule that was extracted from the biological material.", + "id": "molecule", + "mapping": "experiment/molecule", + "mapping_for_search_filter": "individual/phenopackets/biosamples/experiment/molecule", + "options": [ + "genomic DNA" + ], + "title": "Molecules Used" + } + ] + } +] + + + + + + + + + + + + + + + +# KATSU_CONFIG_UNION = [ +# { +# "section_title": "General", +# "fields": [ +# { +# "mapping": "individual/age_numeric", +# "title": "Age", +# "description": "Age at arrival", +# "datatype": "number", +# "config": { +# "bin_size": 10, +# "taper_left": 10, +# "taper_right": 100, +# "units": "years", +# "minimum": 0, +# "maximum": 100, +# }, +# "id": "age", +# "options": [ +# "< 10", +# "[10, 20)", +# "[20, 30)", +# "[30, 40)", +# "[40, 50)", +# "[50, 60)", +# "[60, 70)", +# "[70, 80)", +# "[80, 90)", +# "[90, 100)", +# ], +# }, +# { +# "mapping": "individual/sex", +# "title": "Sex", +# "description": "Sex at birth", +# "datatype": "string", +# "config": {"enum": None}, +# "id": "sex", +# "options": ["MALE", "FEMALE"], +# }, +# { +# "mapping": "individual/extra_properties/date_of_consent", +# "title": "Verbal consent date", +# "description": "Date of initial verbal consent (participant, legal representative or tutor), yyyy-mm-dd", +# "datatype": "date", +# "config": {"bin_by": "month"}, +# "id": "date_of_consent", +# "options": [ +# "Jan 2020", +# "Feb 2020", +# "Mar 2020", +# "Apr 2020", +# "May 2020", +# "Jun 2020", +# "Jul 2020", +# "Aug 2020", +# "Sep 2020", +# "Oct 2020", +# "Nov 2020", +# "Dec 2020", +# "Jan 2021", +# "Feb 2021", +# "Mar 2021", +# "Apr 2021", +# "May 2021", +# "Jun 2021", +# "Jul 2021", +# "Aug 2021", +# "Sep 2021", +# "Oct 2021", +# "Nov 2021", +# "Dec 2021", +# "Jan 2022", +# "Feb 2022", +# "Mar 2022", +# "Apr 2022", +# "May 2022", +# "Jun 2022", +# "Jul 2022", +# "Aug 2022", +# "Sep 2022", +# "Oct 2022", +# "Nov 2022", +# "Dec 2022", +# "Jan 2023", +# "Feb 2023", +# "Mar 2023", +# "Apr 2023", +# "May 2023", +# "Jun 2023", +# "Jul 2023", +# "Aug 2023", +# "Sep 2023", +# "Oct 2023", +# "Nov 2023", +# "Dec 2023", +# ], +# }, +# { +# "mapping": "individual/extra_properties/mobility", +# "title": "Functional status", +# "description": "Mobility", +# "datatype": "string", +# "config": { +# "enum": [ +# "I have no problems in walking about", +# "I have slight problems in walking about", +# "I have moderate problems in walking about", +# "I have severe problems in walking about", +# "I am unable to walk about", +# ] +# }, +# "id": "mobility", +# "options": [ +# "I have no problems in walking about", +# "I have slight problems in walking about", +# "I have moderate problems in walking about", +# "I have severe problems in walking about", +# "I am unable to walk about", +# ], +# }, +# { +# "mapping": "individual/extra_properties/covid_severity", +# "title": "Covid severity", +# "description": "Covid severity", +# "datatype": "string", +# "config": {"enum": ["Uninfected", "Mild", "Moderate", "Severe", "Dead"]}, +# "id": "covid_severity", +# "options": ["Uninfected", "Mild", "Moderate", "Severe", "Dead"], +# }, +# { +# "mapping": "individual/phenopackets/phenotypic_features/pftype/label", +# "title": "Phenotypic Features", +# "description": "Individual phenotypic features, observed as either present or absent", +# "datatype": "string", +# "config": {"enum": None}, +# "id": "phenotypic_features", +# "options": [ +# "Loss of appetite", +# "Asthma", +# "Abdominal pain", +# "Headache", +# "Cough", +# "Cardiac arrest", +# "Nausea", +# "Pulmonary hypertension", +# "Viral pneumonia/pneumonitis", +# "HIV Infection", +# "Hyperglycemia", +# "Patient immunosuppressed", +# "Fever", +# "Chest pain", +# "Pneumothorax", +# "Hypertension", +# "Dementia", +# "Loss of sense of smell", +# ], +# }, +# { +# "mapping": "individual/phenopackets/diseases/term/label", +# "title": "Diseases", +# "description": "Diseases observed as either present or absent", +# "datatype": "string", +# "config": {"enum": None}, +# "id": "diseases", +# "options": [ +# "viral pneumonia", +# "Glioblastoma", +# "Polycythemia vera", +# "Miller syndrome", +# "Cystic fibrosis", +# "coronary artery disease, autosomal dominant, 1", +# "Hashimoto Thyroiditis", +# "Rheumatologic disease", +# "Takotsubo cardiomyopathy", +# "autoimmune disorder of cardiovascular system", +# "COVID-19", +# "Marfan syndrome", +# "Non-Hodgkin Lymphoma", +# "diabetes mellitus", +# ], +# }, +# { +# "mapping": "biosample/sampled_tissue/label", +# "mapping_for_search_filter": "individual/biosamples/sampled_tissue/label", +# "title": "Sampled Tissues", +# "description": "Tissue from which the biosample was extracted", +# "datatype": "string", +# "config": {"enum": None}, +# "id": "tissues", +# "options": ["Plasma", "blood", "Serum"], +# }, +# ], +# }, +# { +# "section_title": "Measurements", +# "fields": [ +# { +# "mapping": "individual/extra_properties/lab_test_result_value", +# "title": "Lab Test Result", +# "description": "Numeric measures from a laboratory test", +# "datatype": "number", +# "config": {"bins": [200, 300, 500, 1000, 1500, 2000], "minimum": 0, "units": "mg/L"}, +# "id": "lab_test_result_value", +# "options": [ +# "< 200", +# "[200, 300)", +# "[300, 500)", +# "[500, 1000)", +# "[1000, 1500)", +# "[1500, 2000)", +# "≥ 2000", +# ], +# }, +# { +# "mapping": "individual/phenopackets/measurements", +# "group_by": "assay/id", +# "group_by_value": "NCIT:C16358", +# "value_mapping": "value/quantity/value", +# "title": "BMI", +# "description": "Body Mass Index", +# "datatype": "number", +# "config": {"bins": [18.5, 30], "minimum": 0, "units": "kg/m^2"}, +# "id": "bmi", +# "options": ["< 18", "[18, 30)", "≥ 30"], +# }, +# ], +# }, +# { +# "section_title": "Medical Actions", +# "fields": [ +# { +# "mapping": "individual/phenopackets/medical_actions", +# "group_by": "procedure/code/label", +# "title": "Medical Procedures", +# "description": "A clinical procedure performed on a subject", +# "datatype": "string", +# "config": {"enum": None}, +# "id": "medical_procedures", +# "options": [ +# "Liver Biopsy", +# "Magnetic Resonance Imaging", +# "Positron Emission Tomography", +# "Punch Biopsy", +# "X-Ray Imaging", +# ], +# }, +# { +# "mapping": "individual/phenopackets/medical_actions", +# "group_by": "treatment/agent/label", +# "title": "Medical Treatments", +# "description": "Treatment with an agent such as a drug", +# "datatype": "string", +# "config": {"enum": None}, +# "id": "medical_treatments", +# "options": ["Acetaminophen", "Ibuprofen", "Ondansetron"], +# }, +# ], +# }, +# { +# "section_title": "Interpretations", +# "fields": [ +# { +# "mapping": "individual/phenopackets/interpretations/diagnosis/genomic_interpretations/interpretation_status", +# "title": "Genomic Interpretations", +# "description": "Interpretation for an individual variant or gene (CANDIDATE, CONTRIBUTORY, etc)", +# "datatype": "string", +# "config": {"enum": None}, +# "id": "interpretation_status", +# "options": ["CONTRIBUTORY"], +# }, +# { +# "mapping": "individual/phenopackets/interpretations/diagnosis/genomic_interpretations/variant_interpretation/acmg_pathogenicity_classification", +# "title": "Variant Pathogenicity", +# "description": "ACMG Pathogenicity category for a particular variant (BENIGN, PATHOGENIC, etc)", +# "datatype": "string", +# "config": {"enum": None}, +# "id": "acmg_pathogenicity_classification", +# "options": ["PATHOGENIC"], +# }, +# ], +# }, +# { +# "section_title": "Experiments", +# "fields": [ +# { +# "mapping": "experiment/experiment_type", +# "mapping_for_search_filter": "individual/biosamples/experiment/experiment_type", +# "title": "Experiment Types", +# "description": "Types of experiments performed on a sample", +# "datatype": "string", +# "config": {"enum": None}, +# "id": "experiment_type", +# "options": [ +# "Other", +# "Neutralizing antibody titers", +# "Metabolite profiling", +# "RNA-Seq", +# "WGS", +# "Proteomic profiling", +# ], +# }, +# { +# "mapping": "experiment/study_type", +# "mapping_for_search_filter": "individual/biosamples/experiment/study_type", +# "title": "Study Types", +# "description": "Study type of the experiment (e.g. Genomics, Transcriptomics, etc.)", +# "datatype": "string", +# "config": {"enum": None}, +# "id": "experiment_study_type", +# "options": ["Serology", "Genomics", "Other", "Proteomics", "Transcriptomics", "Metabolomics"], +# }, +# { +# "mapping": "experiment/experiment_results/file_format", +# "mapping_for_search_filter": "individual/biosamples/experiment/experiment_results/file_format", +# "title": "Results File Types", +# "description": "File type of experiment results files", +# "datatype": "string", +# "config": {"enum": None}, +# "id": "experiment_results_file_type", +# "options": ["XLSX", "CRAM", "VCF", "JPEG", "MP4", "DOCX", "CSV", "MARKDOWN"], +# }, +# ], +# }, +# ] + +# KATSU_CONFIG_INTERSECTION = [ +# { +# "section_title": "General", +# "fields": [ +# { +# "mapping": "individual/age_numeric", +# "title": "Age", +# "description": "Age at arrival", +# "datatype": "number", +# "config": { +# "bin_size": 10, +# "taper_left": 10, +# "taper_right": 100, +# "units": "years", +# "minimum": 0, +# "maximum": 100, +# }, +# "id": "age", +# "options": [ +# "< 10", +# "[10, 20)", +# "[20, 30)", +# "[30, 40)", +# "[40, 50)", +# "[50, 60)", +# "[60, 70)", +# "[70, 80)", +# "[80, 90)", +# "[90, 100)", +# ], +# }, +# { +# "mapping": "individual/sex", +# "title": "Sex", +# "description": "Sex at birth", +# "datatype": "string", +# "config": {"enum": None}, +# "id": "sex", +# "options": ["MALE", "FEMALE"], +# }, +# ], +# }, +# ] diff --git a/bento_beacon/network/utils.py b/bento_beacon/network/utils.py index ff88e57d..c67803b0 100644 --- a/bento_beacon/network/utils.py +++ b/bento_beacon/network/utils.py @@ -6,6 +6,7 @@ from ..utils.exceptions import APIException, InvalidQuery from ..utils.beacon_response import beacon_count_response +PUBLIC_SEARCH_FIELDS_PATH = "/api/metadata/api/public_search_fields" DEFAULT_ENDPOINT = "individuals" OVERVIEW_STATS_QUERY = { "meta": {"apiVersion": "2.0.0"}, @@ -80,9 +81,9 @@ def init_network_service_registry(): network_beacons[b_id] = beacon_info network_beacons[b_id]["overview"] = overview - # TODO, katsu calls are inconsistent here - # qs = get_public_search_fields(url) - # network_beacons[b_id]["querySections"] = get_public_search_fields(url) # temp + # TODO, katsu calls are inconsistent here (v15 katsu does not respond) + # TODO (longer): serve beacon spec filtering terms instead of bento public querySections + network_beacons[b_id]["querySections"] = get_public_search_fields(url).get("sections", []) # temp # make a merged overview? # what about merged filtering_terms? @@ -157,6 +158,8 @@ def merge_charts(c1, c2): def get_public_search_fields(beacon_url): fields_url = public_search_fields_url(beacon_url) + current_app.logger.info(f"trying public fields url {fields_url}") + fields = network_beacon_get(fields_url) return fields @@ -164,7 +167,7 @@ def get_public_search_fields(beacon_url): def public_search_fields_url(beacon_url): split_url = urlsplit(beacon_url) return urlunsplit( - (split_url.scheme, "portal." + split_url.netloc, "/api/metadata/api/public_search_fields", "", "") + (split_url.scheme, "portal." + split_url.netloc, PUBLIC_SEARCH_FIELDS_PATH, "", "") ) From 378a3e84c6747a4979a6ba5a11bad08c350c0fbd Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Mon, 22 Jul 2024 19:03:03 +0000 Subject: [PATCH 10/41] add katsu rule changes --- bento_beacon/config_files/config.py | 1 + bento_beacon/utils/katsu_utils.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/bento_beacon/config_files/config.py b/bento_beacon/config_files/config.py index baf6f3eb..8a6cf171 100644 --- a/bento_beacon/config_files/config.py +++ b/bento_beacon/config_files/config.py @@ -149,6 +149,7 @@ class Config: KATSU_SEARCH_OVERVIEW = "/api/search_overview" KATSU_PRIVATE_OVERVIEW = "/api/overview" KATSU_PUBLIC_OVERVIEW = "/api/public_overview" + KATSU_PUBLIC_RULES = "/api/public_rules" KATSU_TIMEOUT = int(os.environ.get("BEACON_KATSU_TIMEOUT", 180)) MAP_EXTRA_PROPERTIES_TO_INFO = os.environ.get("MAP_EXTRA_PROPERTIES_TO_INFO", True) diff --git a/bento_beacon/utils/katsu_utils.py b/bento_beacon/utils/katsu_utils.py index 341cc71a..a1294ba2 100644 --- a/bento_beacon/utils/katsu_utils.py +++ b/bento_beacon/utils/katsu_utils.py @@ -278,9 +278,10 @@ def overview_statistics(): def katsu_censorship_settings() -> tuple[int | None, int | None]: - overview = katsu_get(current_app.config["KATSU_PUBLIC_OVERVIEW"]) - max_filters = overview.get("max_query_parameters") - count_threshold = overview.get("count_threshold") + # TODO: should be project-dataset scoped + rules = katsu_get(current_app.config["KATSU_PUBLIC_RULES"]) + max_filters = rules.get("max_query_parameters") + count_threshold = rules.get("count_threshold") # return even if None return max_filters, count_threshold From 8b320d7e745ea440c014a7ed453e6f6a2fba4be2 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Mon, 12 Aug 2024 14:29:38 +0000 Subject: [PATCH 11/41] tweak network timeouts --- bento_beacon/network/network_config.py | 8 ++++++-- bento_beacon/network/utils.py | 6 +++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/bento_beacon/network/network_config.py b/bento_beacon/network/network_config.py index 7cc1baed..9a0213dd 100644 --- a/bento_beacon/network/network_config.py +++ b/bento_beacon/network/network_config.py @@ -10,10 +10,14 @@ "https://ichange.bento.sd4h.ca/api/beacon", # "https://qa.bento.sd4h.ca/api/beacon/", # "https://bentov2.local/api/beacon", - # "https://renata.bento.sd4h.ca/api/beacon", + "https://renata.bento.sd4h.ca/api/beacon", + "https://signature.bento.sd4h.ca/api/beacon", + # "https://bentov2.local/api/beacon" ] -NETWORK_TIMEOUT = 30 +NETWORK_INIT_TIMEOUT = 30 +NETWORK_QUERY_WITH_VARIANTS_TIMEOUT = 2 * 60 +NETWORK_QUERY_WITHOUT_VARIANTS_TIMEOUT = 30 VALID_ENDPOINTS = ["analyses", "biosamples", "cohorts", "datasets", "g_variants", "individuals", "runs", "overview"] diff --git a/bento_beacon/network/utils.py b/bento_beacon/network/utils.py index c67803b0..4f94ddac 100644 --- a/bento_beacon/network/utils.py +++ b/bento_beacon/network/utils.py @@ -2,7 +2,7 @@ import requests from urllib.parse import urlsplit, urlunsplit from json import JSONDecodeError -from .network_config import BEACONS, NETWORK_TIMEOUT +from .network_config import BEACONS, NETWORK_INIT_TIMEOUT from ..utils.exceptions import APIException, InvalidQuery from ..utils.beacon_response import beacon_count_response @@ -18,9 +18,9 @@ def network_beacon_call(method, url, payload=None): try: if method == "GET": - r = requests.get(url, timeout=NETWORK_TIMEOUT) + r = requests.get(url, timeout=NETWORK_INIT_TIMEOUT) else: - r = requests.post(url, json=payload, timeout=NETWORK_TIMEOUT) + r = requests.post(url, json=payload, timeout=NETWORK_INIT_TIMEOUT) beacon_response = r.json() except (requests.exceptions.RequestException, JSONDecodeError) as e: From aace127775c96c88296053731fb8381a049f02bf Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Mon, 12 Aug 2024 17:41:41 +0000 Subject: [PATCH 12/41] lint --- bento_beacon/network/network_config.py | 1057 ++++++------------------ bento_beacon/network/utils.py | 4 +- 2 files changed, 233 insertions(+), 828 deletions(-) diff --git a/bento_beacon/network/network_config.py b/bento_beacon/network/network_config.py index 9a0213dd..b56d5b5c 100644 --- a/bento_beacon/network/network_config.py +++ b/bento_beacon/network/network_config.py @@ -22,7 +22,6 @@ VALID_ENDPOINTS = ["analyses", "biosamples", "cohorts", "datasets", "g_variants", "individuals", "runs", "overview"] - KATSU_CONFIG_INTERSECTION = [ { "section_title": "Common Filters", @@ -34,53 +33,35 @@ "minimum": 0, "taper_left": 10, "taper_right": 100, - "units": "years" + "units": "years", }, "datatype": "number", "description": "Age at arrival", "id": "age", "mapping": "individual/age_numeric", - "options": [ - "[30, 40)", - "[40, 50)", - "[50, 60)", - "[60, 70)", - "[70, 80)", - "[80, 90)" - ], - "title": "Age" + "options": ["[30, 40)", "[40, 50)", "[50, 60)", "[60, 70)", "[70, 80)", "[80, 90)"], + "title": "Age", }, { - "config": { - "enum": None - }, + "config": {"enum": None}, "datatype": "string", "description": "Sex at birth", "id": "sex", "mapping": "individual/sex", - "options": [ - "MALE", - "FEMALE" - ], - "title": "Sex" - } - ] + "options": ["MALE", "FEMALE"], + "title": "Sex", + }, + ], } ] - - - - KATSU_CONFIG_UNION = [ { "section_title": "All Filters", "fields": [ { - "config": { - "enum": None - }, + "config": {"enum": None}, "datatype": "string", "description": "Phenotypic features of the individual", "id": "phenotypic_features_type", @@ -109,9 +90,9 @@ "Pneumothorax", "Dementia", "Hypertension", - "Loss of sense of smell" + "Loss of sense of smell", ], - "title": "Phenotypic Features" + "title": "Phenotypic Features", }, { "config": { @@ -120,7 +101,7 @@ "minimum": 0, "taper_left": 10, "taper_right": 100, - "units": "years" + "units": "years", }, "datatype": "number", "description": "Age at arrival", @@ -139,29 +120,21 @@ "[90, 100)", "< 18", "[18, 30)", - "≥ 90" + "≥ 90", ], - "title": "Age" + "title": "Age", }, { - "config": { - "enum": None - }, + "config": {"enum": None}, "datatype": "string", "description": "Sex at birth", "id": "sex", "mapping": "individual/sex", - "options": [ - "MALE", - "FEMALE", - "UNKNOWN_SEX" - ], - "title": "Sex" + "options": ["MALE", "FEMALE", "UNKNOWN_SEX"], + "title": "Sex", }, { - "config": { - "enum": None - }, + "config": {"enum": None}, "datatype": "string", "description": "Measurements performed", "group_by": "assay/label", @@ -174,9 +147,9 @@ "Histological Eosinophils", "Histological Neutrophils", "Immunoglobulin E", - "Neutrophils Percentage" + "Neutrophils Percentage", ], - "title": "Measurement types" + "title": "Measurement types", }, { "config": { @@ -185,7 +158,7 @@ "minimum": 0, "taper_left": 0, "taper_right": 50, - "units": "kg/m^2" + "units": "kg/m^2", }, "datatype": "number", "description": "Measurements performed", @@ -193,23 +166,12 @@ "group_by_value": "NCIT:C16358", "id": "measurement_bmi", "mapping": "individual/phenopackets/measurements", - "options": [ - "[0, 10)", - "[10, 20)", - "[20, 30)", - "[30, 40)", - "[40, 50)", - "< 18", - "[18, 30)", - "≥ 30" - ], + "options": ["[0, 10)", "[10, 20)", "[20, 30)", "[30, 40)", "[40, 50)", "< 18", "[18, 30)", "≥ 30"], "title": "BMI", - "value_mapping": "value/quantity/value" + "value_mapping": "value/quantity/value", }, { - "config": { - "enum": None - }, + "config": {"enum": None}, "datatype": "string", "description": "Visits where measurements were performed", "group_by": "extra_properties/visit_index", @@ -248,9 +210,9 @@ "6", "7", "8", - "9" + "9", ], - "title": "Recurring visits" + "title": "Recurring visits", }, { "config": { @@ -259,7 +221,7 @@ "minimum": 0, "taper_left": 50, "taper_right": 500, - "units": "years" + "units": "years", }, "datatype": "number", "description": "Measurements performed x", @@ -278,10 +240,10 @@ "[350, 400)", "[400, 450)", "[450, 500)", - "≥ 500" + "≥ 500", ], "title": "Immunoglobulin E", - "value_mapping": "value/quantity/value" + "value_mapping": "value/quantity/value", }, { "config": { @@ -290,7 +252,7 @@ "minimum": 0, "taper_left": 10, "taper_right": 100, - "units": "years" + "units": "years", }, "datatype": "number", "description": "Measurements performed Cp20", @@ -309,15 +271,13 @@ "[70, 80)", "[80, 90)", "[90, 100)", - "≥ 100" + "≥ 100", ], "title": "Cp20", - "value_mapping": "value/quantity/value" + "value_mapping": "value/quantity/value", }, { - "config": { - "enum": None - }, + "config": {"enum": None}, "datatype": "string", "description": "Tissue from which the specimen was collected", "id": "sampled_tissue", @@ -388,14 +348,12 @@ "Optic Chiasm", "Cerebello-Pontine Angle", "Pons", - "Parieto-Occipital Lobe" + "Parieto-Occipital Lobe", ], - "title": "Biosample Tissue Location" + "title": "Biosample Tissue Location", }, { - "config": { - "bin_by": "month" - }, + "config": {"bin_by": "month"}, "datatype": "date", "description": "Date of initial verbal consent (participant, legal representative or tutor), yyyy-mm-dd", "id": "date_of_consent", @@ -448,9 +406,9 @@ "Sep 2023", "Oct 2023", "Nov 2023", - "Dec 2023" + "Dec 2023", ], - "title": "Verbal consent date" + "title": "Verbal consent date", }, { "config": { @@ -459,7 +417,7 @@ "I have slight problems in walking about", "I have moderate problems in walking about", "I have severe problems in walking about", - "I am unable to walk about" + "I am unable to walk about", ] }, "datatype": "string", @@ -471,37 +429,21 @@ "I have slight problems in walking about", "I have moderate problems in walking about", "I have severe problems in walking about", - "I am unable to walk about" + "I am unable to walk about", ], - "title": "Functional status" + "title": "Functional status", }, { - "config": { - "enum": [ - "Uninfected", - "Mild", - "Moderate", - "Severe", - "Dead" - ] - }, + "config": {"enum": ["Uninfected", "Mild", "Moderate", "Severe", "Dead"]}, "datatype": "string", "description": "Covid severity", "id": "covid_severity", "mapping": "individual/extra_properties/covid_severity", - "options": [ - "Uninfected", - "Mild", - "Moderate", - "Severe", - "Dead" - ], - "title": "Covid severity" + "options": ["Uninfected", "Mild", "Moderate", "Severe", "Dead"], + "title": "Covid severity", }, { - "config": { - "enum": None - }, + "config": {"enum": None}, "datatype": "string", "description": "Diseases observed as either present or absent", "id": "diseases", @@ -588,39 +530,22 @@ "Ependymoma", "Choroid Plexus Carcinoma", "Oral Squamous Cell Carcinoma (OSCC)", - "Chondroblastoma" + "Chondroblastoma", ], - "title": "Diseases" + "title": "Diseases", }, { - "config": { - "enum": None - }, + "config": {"enum": None}, "datatype": "string", "description": "Tissue from which the biosample was extracted", "id": "tissues", "mapping": "biosample/sampled_tissue/label", "mapping_for_search_filter": "individual/biosamples/sampled_tissue/label", - "options": [ - "Plasma", - "blood", - "Serum" - ], - "title": "Sampled Tissues" + "options": ["Plasma", "blood", "Serum"], + "title": "Sampled Tissues", }, { - "config": { - "bins": [ - 200, - 300, - 500, - 1000, - 1500, - 2000 - ], - "minimum": 0, - "units": "mg/L" - }, + "config": {"bins": [200, 300, 500, 1000, 1500, 2000], "minimum": 0, "units": "mg/L"}, "datatype": "number", "description": "Numeric measures from a laboratory test", "id": "lab_test_result_value", @@ -632,14 +557,12 @@ "[500, 1000)", "[1000, 1500)", "[1500, 2000)", - "≥ 2000" + "≥ 2000", ], - "title": "Lab Test Result" + "title": "Lab Test Result", }, { - "config": { - "enum": None - }, + "config": {"enum": None}, "datatype": "string", "description": "A clinical procedure performed on a subject", "group_by": "procedure/code/label", @@ -650,56 +573,40 @@ "Magnetic Resonance Imaging", "Positron Emission Tomography", "Punch Biopsy", - "X-Ray Imaging" + "X-Ray Imaging", ], - "title": "Medical Procedures" + "title": "Medical Procedures", }, { - "config": { - "enum": None - }, + "config": {"enum": None}, "datatype": "string", "description": "Treatment with an agent such as a drug", "group_by": "treatment/agent/label", "id": "medical_treatments", "mapping": "individual/phenopackets/medical_actions", - "options": [ - "Acetaminophen", - "Ibuprofen", - "NCIT:C1119" - ], - "title": "Medical Treatments" + "options": ["Acetaminophen", "Ibuprofen", "NCIT:C1119"], + "title": "Medical Treatments", }, { - "config": { - "enum": None - }, + "config": {"enum": None}, "datatype": "string", "description": "Interpretation for an individual variant or gene (CANDIDATE, CONTRIBUTORY, etc)", "id": "interpretation_status", "mapping": "individual/phenopackets/interpretations/diagnosis/genomic_interpretations/interpretation_status", - "options": [ - "CONTRIBUTORY" - ], - "title": "Genomic Interpretations" + "options": ["CONTRIBUTORY"], + "title": "Genomic Interpretations", }, { - "config": { - "enum": None - }, + "config": {"enum": None}, "datatype": "string", "description": "ACMG Pathogenicity category for a particular variant (BENIGN, PATHOGENIC, etc)", "id": "acmg_pathogenicity_classification", "mapping": "individual/phenopackets/interpretations/diagnosis/genomic_interpretations/variant_interpretation/acmg_pathogenicity_classification", - "options": [ - "PATHOGENIC" - ], - "title": "Variant Pathogenicity" + "options": ["PATHOGENIC"], + "title": "Variant Pathogenicity", }, { - "config": { - "enum": None - }, + "config": {"enum": None}, "datatype": "string", "description": "Types of experiments performed on a sample", "id": "experiment_type", @@ -712,55 +619,32 @@ "RNA-Seq", "WGS", "Proteomic profiling", - "WES" + "WES", ], - "title": "Experiment Types" + "title": "Experiment Types", }, { - "config": { - "enum": None - }, + "config": {"enum": None}, "datatype": "string", "description": "Study type of the experiment (e.g. Genomics, Transcriptomics, etc.)", "id": "experiment_study_type", "mapping": "experiment/study_type", "mapping_for_search_filter": "individual/biosamples/experiment/study_type", - "options": [ - "Serology", - "Other", - "Genomics", - "Proteomics", - "Transcriptomics", - "Metabolomics" - ], - "title": "Study Types" + "options": ["Serology", "Other", "Genomics", "Proteomics", "Transcriptomics", "Metabolomics"], + "title": "Study Types", }, { - "config": { - "enum": None - }, + "config": {"enum": None}, "datatype": "string", "description": "File type of experiment results files", "id": "experiment_results_file_type", "mapping": "experiment/experiment_results/file_format", "mapping_for_search_filter": "individual/biosamples/experiment/experiment_results/file_format", - "options": [ - "PDF", - "XLSX", - "JPEG", - "CRAM", - "VCF", - "MP4", - "DOCX", - "CSV", - "MARKDOWN" - ], - "title": "Results File Types" + "options": ["PDF", "XLSX", "JPEG", "CRAM", "VCF", "MP4", "DOCX", "CSV", "MARKDOWN"], + "title": "Results File Types", }, { - "config": { - "bin_by": "month" - }, + "config": {"bin_by": "month"}, "datatype": "date", "description": "Date of initial verbal consent (participant, legal representative or tutor)", "id": "dconsverbpa", @@ -807,150 +691,72 @@ "May 2023", "Jun 2023", "Jul 2023", - "Aug 2023" + "Aug 2023", ], - "title": "Verbal consent date" + "title": "Verbal consent date", }, { - "config": { - "enum": [ - "Hospitalized", - "Outpatient" - ] - }, + "config": {"enum": ["Hospitalized", "Outpatient"]}, "datatype": "string", "description": "Has the participant been hospitalized or is the participant seen as an outpatient?", "id": "type_partic", "mapping": "individual/extra_properties/type_partic", - "options": [ - "Hospitalized", - "Outpatient" - ], - "title": "Hospitalization" + "options": ["Hospitalized", "Outpatient"], + "title": "Hospitalization", }, { - "config": { - "enum": [ - "Non-smoker", - "Smoker", - "Former smoker", - "Passive smoker", - "Not specified" - ] - }, + "config": {"enum": ["Non-smoker", "Smoker", "Former smoker", "Passive smoker", "Not specified"]}, "datatype": "string", "description": "Smoking status", "id": "smoking", "mapping": "individual/extra_properties/smoking", - "options": [ - "Non-smoker", - "Smoker", - "Former smoker", - "Passive smoker", - "Not specified" - ], - "title": "Smoking" + "options": ["Non-smoker", "Smoker", "Former smoker", "Passive smoker", "Not specified"], + "title": "Smoking", }, { - "config": { - "enum": [ - "Positive", - "Negative", - "Indeterminate" - ] - }, + "config": {"enum": ["Positive", "Negative", "Indeterminate"]}, "datatype": "string", "description": "Final COVID status according to the PCR test", "id": "covidstatus", "mapping": "individual/extra_properties/covidstatus", - "options": [ - "Positive", - "Negative", - "Indeterminate" - ], - "title": "COVID status" + "options": ["Positive", "Negative", "Indeterminate"], + "title": "COVID status", }, { - "config": { - "enum": [ - "Alive", - "Deceased" - ] - }, + "config": {"enum": ["Alive", "Deceased"]}, "datatype": "string", "description": "Vital status at discharge", "id": "death_dc", "mapping": "individual/extra_properties/death_dc", - "options": [ - "Alive", - "Deceased" - ], - "title": "Vital status" + "options": ["Alive", "Deceased"], + "title": "Vital status", }, { - "config": { - "bins": [ - 20, - 25, - 27, - 30, - 35, - 40 - ], - "units": "kg/m^2" - }, + "config": {"bins": [20, 25, 27, 30, 35, 40], "units": "kg/m^2"}, "datatype": "number", "description": "BMI", "id": "bmi", "mapping": "individual/extra_properties/bmi", - "options": [ - "< 20", - "[20, 25)", - "[25, 27)", - "[27, 30)", - "[30, 35)", - "[35, 40)", - "≥ 40" - ], - "title": "BMI" + "options": ["< 20", "[20, 25)", "[25, 27)", "[27, 30)", "[30, 35)", "[35, 40)", "≥ 40"], + "title": "BMI", }, { - "config": { - "enum": [ - "Yes", - "No" - ] - }, + "config": {"enum": ["Yes", "No"]}, "datatype": "string", "description": "Intensive Care Unit admission?", "id": "icu", "mapping": "individual/extra_properties/icu", - "options": [ - "Yes", - "No" - ], - "title": "ICU" + "options": ["Yes", "No"], + "title": "ICU", }, { - "config": { - "enum": [ - "Hospitalized adult", - "Adult outpatient", - "Pediatric", - "Pregnant woman" - ] - }, + "config": {"enum": ["Hospitalized adult", "Adult outpatient", "Pediatric", "Pregnant woman"]}, "datatype": "string", "description": "To which category the participant belongs?", "id": "core_cohorte", "mapping": "individual/extra_properties/core_cohorte", - "options": [ - "Hospitalized adult", - "Adult outpatient", - "Pediatric", - "Pregnant woman" - ], - "title": "Participant category" + "options": ["Hospitalized adult", "Adult outpatient", "Pediatric", "Pregnant woman"], + "title": "Participant category", }, { "config": { @@ -960,7 +766,7 @@ "Nursing home (CHSLD)", "In intermediate and family-type resources", "In rooming house", - "Homeless" + "Homeless", ] }, "datatype": "string", @@ -973,47 +779,27 @@ "Nursing home (CHSLD)", "In intermediate and family-type resources", "In rooming house", - "Homeless" + "Homeless", ], - "title": "Domicile" + "title": "Domicile", }, { - "config": { - "enum": [ - "Yes", - "No" - ] - }, + "config": {"enum": ["Yes", "No"]}, "datatype": "string", "description": "Has the participant been vaccinated?", "id": "vaccinate", "mapping": "individual/extra_properties/vaccinate", - "options": [ - "Yes", - "No" - ], - "title": "Vaccinal status" + "options": ["Yes", "No"], + "title": "Vaccinal status", }, { - "config": { - "enum": [ - "1", - "2", - "3", - "4" - ] - }, + "config": {"enum": ["1", "2", "3", "4"]}, "datatype": "string", "description": "Number of doses received", "id": "vaccin_dosenum", "mapping": "individual/extra_properties/vaccin_dosenum", - "options": [ - "1", - "2", - "3", - "4" - ], - "title": "Vaccine dose" + "options": ["1", "2", "3", "4"], + "title": "Vaccine dose", }, { "config": { @@ -1022,7 +808,7 @@ "I have slight problem washing or dressing myself", "I have moderate problems washing or dressing myself", "I have severe problems washing or dressing myself", - "I am unable to wash or dress myself" + "I am unable to wash or dress myself", ] }, "datatype": "string", @@ -1034,473 +820,237 @@ "I have slight problem washing or dressing myself", "I have moderate problems washing or dressing myself", "I have severe problems washing or dressing myself", - "I am unable to wash or dress myself" + "I am unable to wash or dress myself", ], - "title": "Self-care" + "title": "Self-care", }, { - "config": { - "enum": [ - "Yes", - "No" - ] - }, + "config": {"enum": ["Yes", "No"]}, "datatype": "string", "description": "Other chronic cardiac disease?", "id": "phx_cardiac", "mapping": "individual/extra_properties/phx_cardiac", - "options": [ - "Yes", - "No" - ], - "title": "Cardiac history" + "options": ["Yes", "No"], + "title": "Cardiac history", }, { - "config": { - "enum": [ - "Yes", - "No" - ] - }, + "config": {"enum": ["Yes", "No"]}, "datatype": "string", "description": "Prior transient ischemic attack (TIA)?", "id": "phx_tia", "mapping": "individual/extra_properties/phx_tia", - "options": [ - "Yes", - "No" - ], - "title": "TIA" + "options": ["Yes", "No"], + "title": "TIA", }, { - "config": { - "enum": [ - "Yes", - "No" - ] - }, + "config": {"enum": ["Yes", "No"]}, "datatype": "string", "description": "Asthma?", "id": "phx_asthma", "mapping": "individual/extra_properties/phx_asthma", - "options": [ - "Yes", - "No" - ], - "title": "Asthma" + "options": ["Yes", "No"], + "title": "Asthma", }, { - "config": { - "enum": [ - "Yes", - "No" - ] - }, + "config": {"enum": ["Yes", "No"]}, "datatype": "string", "description": "Prior stroke?", "id": "phx_cva", "mapping": "individual/extra_properties/phx_cva", - "options": [ - "Yes", - "No" - ], - "title": "Stroke" + "options": ["Yes", "No"], + "title": "Stroke", }, { - "config": { - "enum": [ - "Yes", - "No" - ] - }, + "config": {"enum": ["Yes", "No"]}, "datatype": "string", "description": "Diabetes?", "id": "phx_diabetes", "mapping": "individual/extra_properties/phx_diabetes", - "options": [ - "Yes", - "No" - ], - "title": "Diabetes" + "options": ["Yes", "No"], + "title": "Diabetes", }, { - "config": { - "enum": [ - "Yes", - "No" - ] - }, + "config": {"enum": ["Yes", "No"]}, "datatype": "string", "description": "Malignant neoplasm?", "id": "phx_cancer", "mapping": "individual/extra_properties/phx_cancer", - "options": [ - "Yes", - "No" - ], - "title": "Cancer" + "options": ["Yes", "No"], + "title": "Cancer", }, { - "config": { - "enum": [ - "Yes", - "No" - ] - }, + "config": {"enum": ["Yes", "No"]}, "datatype": "string", "description": "Dementia?", "id": "phx_dementia", "mapping": "individual/extra_properties/phx_dementia", - "options": [ - "Yes", - "No" - ], - "title": "Dementia" + "options": ["Yes", "No"], + "title": "Dementia", }, { - "config": { - "enum": [ - "Yes", - "No" - ] - }, + "config": {"enum": ["Yes", "No"]}, "datatype": "string", "description": "Atrial fibrillation or flutter?", "id": "phx_afib", "mapping": "individual/extra_properties/phx_afib", - "options": [ - "Yes", - "No" - ], - "title": "Atrial fibrillation" + "options": ["Yes", "No"], + "title": "Atrial fibrillation", }, { - "config": { - "enum": [ - "Yes", - "No" - ] - }, + "config": {"enum": ["Yes", "No"]}, "datatype": "string", "description": "HIV or AIDS?", "id": "phx_hiv", "mapping": "individual/extra_properties/phx_hiv", - "options": [ - "Yes", - "No" - ], - "title": "HIV" + "options": ["Yes", "No"], + "title": "HIV", }, { - "config": { - "enum": [ - "Yes", - "No" - ] - }, + "config": {"enum": ["Yes", "No"]}, "datatype": "string", "description": "Arterial Hypertension?", "id": "phx_htn", "mapping": "individual/extra_properties/phx_htn", - "options": [ - "Yes", - "No" - ], - "title": "Arterial Hypertension" + "options": ["Yes", "No"], + "title": "Arterial Hypertension", }, { - "config": { - "enum": [ - "Yes", - "No" - ] - }, + "config": {"enum": ["Yes", "No"]}, "datatype": "string", "description": "Immunosuppressed state?", "id": "phx_imm", "mapping": "individual/extra_properties/phx_imm", - "options": [ - "Yes", - "No" - ], - "title": "Immunosupression" + "options": ["Yes", "No"], + "title": "Immunosupression", }, { - "config": { - "enum": [ - "Yes", - "No" - ] - }, + "config": {"enum": ["Yes", "No"]}, "datatype": "string", "description": "Prior myocardial infarction?", "id": "phx_mi", "mapping": "individual/extra_properties/phx_mi", - "options": [ - "Yes", - "No" - ], - "title": "Myocardial infarction" + "options": ["Yes", "No"], + "title": "Myocardial infarction", }, { - "config": { - "enum": [ - "Yes", - "No" - ] - }, + "config": {"enum": ["Yes", "No"]}, "datatype": "string", "description": "Heart failure?", "id": "phx_chf", "mapping": "individual/extra_properties/phx_chf", - "options": [ - "Yes", - "No" - ], - "title": "Heart failure" + "options": ["Yes", "No"], + "title": "Heart failure", }, { - "config": { - "enum": [ - "Yes", - "No" - ] - }, + "config": {"enum": ["Yes", "No"]}, "datatype": "string", "description": "Coronary artery disease?", "id": "phx_cad", "mapping": "individual/extra_properties/phx_cad", - "options": [ - "Yes", - "No" - ], - "title": "Coronary artery disease" + "options": ["Yes", "No"], + "title": "Coronary artery disease", }, { - "config": { - "enum": [ - "Yes", - "No" - ] - }, + "config": {"enum": ["Yes", "No"]}, "datatype": "string", "description": "Liver disease?", "id": "phx_liver", "mapping": "individual/extra_properties/phx_liver", - "options": [ - "Yes", - "No" - ], - "title": "Liver disease" + "options": ["Yes", "No"], + "title": "Liver disease", }, { - "config": { - "enum": [ - "Yes", - "No" - ] - }, + "config": {"enum": ["Yes", "No"]}, "datatype": "string", "description": "Chronic obstructive pulmonary disease (emphysema, chronic bronchitis)?", "id": "phx_copd", "mapping": "individual/extra_properties/phx_copd", - "options": [ - "Yes", - "No" - ], - "title": "COPD" + "options": ["Yes", "No"], + "title": "COPD", }, { - "config": { - "enum": [ - "Yes", - "No" - ] - }, + "config": {"enum": ["Yes", "No"]}, "datatype": "string", "description": "Psychiatric disease?", "id": "phx_psych", "mapping": "individual/extra_properties/phx_psych", - "options": [ - "Yes", - "No" - ], - "title": "Psychiatric disease" + "options": ["Yes", "No"], + "title": "Psychiatric disease", }, { - "config": { - "enum": [ - "Yes", - "No" - ] - }, + "config": {"enum": ["Yes", "No"]}, "datatype": "string", "description": "Chronic kidney disease?", "id": "phx_ckd", "mapping": "individual/extra_properties/phx_ckd", - "options": [ - "Yes", - "No" - ], - "title": "Chronic kidney disease" + "options": ["Yes", "No"], + "title": "Chronic kidney disease", }, { - "config": { - "enum": [ - "Yes", - "No" - ] - }, + "config": {"enum": ["Yes", "No"]}, "datatype": "string", "description": "Dialysis?", "id": "phx_dialysis", "mapping": "individual/extra_properties/phx_dialysis", - "options": [ - "Yes", - "No" - ], - "title": "Dialysis" + "options": ["Yes", "No"], + "title": "Dialysis", }, { - "config": { - "enum": [ - "Yes", - "No" - ] - }, + "config": {"enum": ["Yes", "No"]}, "datatype": "string", "description": "Rheumatologic disease?", "id": "phx_rheum", "mapping": "individual/extra_properties/phx_rheum", - "options": [ - "Yes", - "No" - ], - "title": "Rheumatologic disease" + "options": ["Yes", "No"], + "title": "Rheumatologic disease", }, { - "config": { - "bins": [ - 5, - 10, - 50, - 100 - ], - "units": "mg/L" - }, + "config": {"bins": [5, 10, 50, 100], "units": "mg/L"}, "datatype": "number", "description": "C-reactive protein (CRP)", "id": "lab_crp", "mapping": "individual/extra_properties/lab_crp", - "options": [ - "< 5", - "[5, 10)", - "[10, 50)", - "[50, 100)", - "≥ 100" - ], - "title": "CRP" + "options": ["< 5", "[5, 10)", "[10, 50)", "[50, 100)", "≥ 100"], + "title": "CRP", }, { - "config": { - "enum": [ - "Yes", - "No" - ] - }, + "config": {"enum": ["Yes", "No"]}, "datatype": "string", "description": "Chest X-ray?", "id": "cxr", "mapping": "individual/extra_properties/cxr", - "options": [ - "Yes", - "No" - ], - "title": "Chest X-ray" + "options": ["Yes", "No"], + "title": "Chest X-ray", }, { - "config": { - "bins": [ - 4, - 10, - 15 - ], - "units": "×10^9/L" - }, + "config": {"bins": [4, 10, 15], "units": "×10^9/L"}, "datatype": "number", "description": "Total White Blood Cell (WBC) count", "id": "lab_wbc", "mapping": "individual/extra_properties/lab_wbc", - "options": [ - "< 4", - "[4, 10)", - "[10, 15)", - "≥ 15" - ], - "title": "WBC" + "options": ["< 4", "[4, 10)", "[10, 15)", "≥ 15"], + "title": "WBC", }, { - "config": { - "bins": [ - 70, - 90, - 110, - 130, - 150 - ], - "units": "g/L" - }, + "config": {"bins": [70, 90, 110, 130, 150], "units": "g/L"}, "datatype": "number", "description": "Haemoglobin", "id": "lab_hg", "mapping": "individual/extra_properties/lab_hg", - "options": [ - "< 70", - "[70, 90)", - "[90, 110)", - "[110, 130)", - "[130, 150)", - "≥ 150" - ], - "title": "Haemoglobin" + "options": ["< 70", "[70, 90)", "[90, 110)", "[110, 130)", "[130, 150)", "≥ 150"], + "title": "Haemoglobin", }, { - "config": { - "bins": [ - 50, - 90, - 120, - 200, - 300 - ], - "units": "μmol/L" - }, + "config": {"bins": [50, 90, 120, 200, 300], "units": "μmol/L"}, "datatype": "number", "description": "Creatinine", "id": "lab_cr", "mapping": "individual/extra_properties/lab_cr", - "options": [ - "< 50", - "[50, 90)", - "[90, 120)", - "[120, 200)", - "[200, 300)", - "≥ 300" - ], - "title": "Creatinine" + "options": ["< 50", "[50, 90)", "[90, 120)", "[120, 200)", "[200, 300)", "≥ 300"], + "title": "Creatinine", }, { - "config": { - "bins": [ - 200, - 300, - 500, - 1000, - 1500, - 2000 - ], - "units": "ng/mL" - }, + "config": {"bins": [200, 300, 500, 1000, 1500, 2000], "units": "ng/mL"}, "datatype": "number", "description": "NT-proBNP", "id": "lab_ntprobnp", @@ -1512,285 +1062,142 @@ "[500, 1000)", "[1000, 1500)", "[1500, 2000)", - "≥ 2000" + "≥ 2000", ], - "title": "NT-proBNP" + "title": "NT-proBNP", }, { - "config": { - "bins": [ - 200, - 500, - 1000, - 1500, - 2000 - ], - "units": "U/L" - }, + "config": {"bins": [200, 500, 1000, 1500, 2000], "units": "U/L"}, "datatype": "number", "description": "Lactate Dehydrogenase", "id": "lab_ldh", "mapping": "individual/extra_properties/lab_ldh", - "options": [ - "< 200", - "[200, 500)", - "[500, 1000)", - "[1000, 1500)", - "[1500, 2000)", - "≥ 2000" - ], - "title": "LDH" + "options": ["< 200", "[200, 500)", "[500, 1000)", "[1000, 1500)", "[1500, 2000)", "≥ 2000"], + "title": "LDH", }, { - "config": { - "bins": [ - 500, - 1000, - 2000, - 5000 - ], - "units": "μg/L" - }, + "config": {"bins": [500, 1000, 2000, 5000], "units": "μg/L"}, "datatype": "number", "description": "D-Dimer", "id": "lab_ddimer", "mapping": "individual/extra_properties/lab_ddimer", - "options": [ - "< 500", - "[500, 1000)", - "[1000, 2000)", - "[2000, 5000)", - "≥ 5000" - ], - "title": "D-Dimer" + "options": ["< 500", "[500, 1000)", "[1000, 2000)", "[2000, 5000)", "≥ 5000"], + "title": "D-Dimer", }, { - "config": { - "enum": [ - "Yes", - "No" - ] - }, + "config": {"enum": ["Yes", "No"]}, "datatype": "string", "description": "Did or does the patient receive ventilatory support?", "id": "repsupport_yesno", "mapping": "individual/extra_properties/repsupport_yesno", - "options": [ - "Yes", - "No" - ], - "title": "Respiratory support" + "options": ["Yes", "No"], + "title": "Respiratory support", }, { - "config": { - "enum": [ - "Yes", - "No" - ] - }, + "config": {"enum": ["Yes", "No"]}, "datatype": "string", "description": "Have you been diagnosed with a new or recurrent case of COVID since your last follow-up?", "id": "newcovid", "mapping": "individual/extra_properties/newcovid", - "options": [ - "Yes", - "No" - ], - "title": "Reinfection" + "options": ["Yes", "No"], + "title": "Reinfection", }, { - "config": { - "enum": [ - "Yes", - "No" - ] - }, + "config": {"enum": ["Yes", "No"]}, "datatype": "string", "description": "Does the participant report persistent symptoms related to SARS-CoV-2 infection?", "id": "sx_report", "mapping": "individual/extra_properties/sx_report", - "options": [ - "Yes", - "No" - ], - "title": "Persisting symptoms" + "options": ["Yes", "No"], + "title": "Persisting symptoms", }, { - "config": { - "enum": [ - "Yes", - "No", - "Not available" - ] - }, + "config": {"enum": ["Yes", "No", "Not available"]}, "datatype": "string", "description": "Systemic corticosteroid?", "id": "rx_roid", "mapping": "individual/extra_properties/rx_roid", - "options": [ - "Yes", - "No", - "Not available" - ], - "title": "Systemic corticosteroid" + "options": ["Yes", "No", "Not available"], + "title": "Systemic corticosteroid", }, { - "config": { - "enum": [ - "Yes", - "No", - "Not available" - ] - }, + "config": {"enum": ["Yes", "No", "Not available"]}, "datatype": "string", "description": "Antocoagulants?", "id": "rx_aco", "mapping": "individual/extra_properties/rx_aco", - "options": [ - "Yes", - "No", - "Not available" - ], - "title": "Antocoagulants" + "options": ["Yes", "No", "Not available"], + "title": "Antocoagulants", }, { - "config": { - "enum": [ - "Worse", - "Same", - "Better", - "Indeterminate" - ] - }, + "config": {"enum": ["Worse", "Same", "Better", "Indeterminate"]}, "datatype": "string", "description": "Ability to self-care at discharge versus pre-COVID", "id": "selfcare_dc", "mapping": "individual/extra_properties/selfcare_dc", - "options": [ - "Worse", - "Same", - "Better", - "Indeterminate" - ], - "title": "Self-care post covid" + "options": ["Worse", "Same", "Better", "Indeterminate"], + "title": "Self-care post covid", }, { - "config": { - "enum": [ - "Yes", - "No" - ] - }, + "config": {"enum": ["Yes", "No"]}, "datatype": "string", "description": "Acute kidney injury?", "id": "c_aki", "mapping": "individual/extra_properties/c_aki", - "options": [ - "Yes", - "No" - ], - "title": "Acute kidney injury" + "options": ["Yes", "No"], + "title": "Acute kidney injury", }, { - "config": { - "enum": [ - "Yes", - "No" - ] - }, + "config": {"enum": ["Yes", "No"]}, "datatype": "string", "description": "Acute Respiratory Distress Syndrome (ARDS)?", "id": "c_ards", "mapping": "individual/extra_properties/c_ards", - "options": [ - "Yes", - "No" - ], - "title": "ARDS" + "options": ["Yes", "No"], + "title": "ARDS", }, { - "config": { - "enum": [ - "Yes", - "No" - ] - }, + "config": {"enum": ["Yes", "No"]}, "datatype": "string", "description": "Deep vein thrombosis (DVT)?", "id": "c_dvt", "mapping": "individual/extra_properties/c_dvt", - "options": [ - "Yes", - "No" - ], - "title": "Deep vein thrombosis" + "options": ["Yes", "No"], + "title": "Deep vein thrombosis", }, { - "config": { - "enum": [ - "Yes", - "No" - ] - }, + "config": {"enum": ["Yes", "No"]}, "datatype": "string", "description": "Stroke?", "id": "c_stroke", "mapping": "individual/extra_properties/c_stroke", - "options": [ - "Yes", - "No" - ], - "title": "Stroke" + "options": ["Yes", "No"], + "title": "Stroke", }, { - "config": { - "enum": [ - "Yes", - "No" - ] - }, + "config": {"enum": ["Yes", "No"]}, "datatype": "string", "description": "Pulmonary embolism?", "id": "c_pe", "mapping": "individual/extra_properties/c_pe", - "options": [ - "Yes", - "No" - ], - "title": "Pulmonary embolism" + "options": ["Yes", "No"], + "title": "Pulmonary embolism", }, { - "config": { - "enum": None - }, + "config": {"enum": None}, "datatype": "string", "description": "The type of molecule that was extracted from the biological material.", "id": "molecule", "mapping": "experiment/molecule", "mapping_for_search_filter": "individual/phenopackets/biosamples/experiment/molecule", - "options": [ - "genomic DNA" - ], - "title": "Molecules Used" - } - ] + "options": ["genomic DNA"], + "title": "Molecules Used", + }, + ], } ] - - - - - - - - - - - - - # KATSU_CONFIG_UNION = [ # { # "section_title": "General", diff --git a/bento_beacon/network/utils.py b/bento_beacon/network/utils.py index 4f94ddac..b0128ff1 100644 --- a/bento_beacon/network/utils.py +++ b/bento_beacon/network/utils.py @@ -166,9 +166,7 @@ def get_public_search_fields(beacon_url): def public_search_fields_url(beacon_url): split_url = urlsplit(beacon_url) - return urlunsplit( - (split_url.scheme, "portal." + split_url.netloc, PUBLIC_SEARCH_FIELDS_PATH, "", "") - ) + return urlunsplit((split_url.scheme, "portal." + split_url.netloc, PUBLIC_SEARCH_FIELDS_PATH, "", "")) def filtersUnion(): From b4a3877b6044ced07166d1917ae321b1a921b92d Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Mon, 12 Aug 2024 17:42:39 +0000 Subject: [PATCH 13/41] temp network test --- bento_beacon/network/network_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bento_beacon/network/network_config.py b/bento_beacon/network/network_config.py index b56d5b5c..38297381 100644 --- a/bento_beacon/network/network_config.py +++ b/bento_beacon/network/network_config.py @@ -8,7 +8,7 @@ # "https://staging.bqc19.bento.sd4h.ca/api/beacon", "https://staging.bento.sd4h.ca/api/beacon", "https://ichange.bento.sd4h.ca/api/beacon", - # "https://qa.bento.sd4h.ca/api/beacon/", + "https://qa.bento.sd4h.ca/api/beacon/", # "https://bentov2.local/api/beacon", "https://renata.bento.sd4h.ca/api/beacon", "https://signature.bento.sd4h.ca/api/beacon", From 8b91bbeb45d2f7c4d9bac0a5a694e904a8a4c447 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Thu, 15 Aug 2024 13:14:08 +0000 Subject: [PATCH 14/41] temp bento public query handling --- bento_beacon/network/bento_public_query.py | 89 ++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 bento_beacon/network/bento_public_query.py diff --git a/bento_beacon/network/bento_public_query.py b/bento_beacon/network/bento_public_query.py new file mode 100644 index 00000000..05a664d3 --- /dev/null +++ b/bento_beacon/network/bento_public_query.py @@ -0,0 +1,89 @@ +import copy + +# TEMP FILE +# +# handling for bento public query terms, currently how beacon UI handles search options to present to the user +# to be replaced by beacon spec filtering_terms in a future version +# best approach here is not yet clear: +# - intersection of all query terms is too small +# - union of all query terms loses any organization into categories, which varies across instances +# +# we may prefer to make network query terms configurable rather than generating them automatically + + +def flatten(nested_list): + return [item for nested_items in nested_list for item in nested_items] + + +def fields_dict(searchFields): + """ + Given a list of bento_public search fields, one for each instance, + return a dictionary of search fields keyed to phenopackets mapping, with an array of all fields for that mapping + """ + # create a single array of all search fields for all instances, removing nesting + copy = searchFields[:] + + all_fields = [] + for sf in copy: + for f in sf: + all_fields.extend(f["fields"]) + + # make a dict of entries, keyed to phenopackets mapping + group_by, etc, keeping duplicate values + all_fields_by_mapping = {} + for f in all_fields: + field_key = f["mapping"] + f.get("group_by", "") + f.get("group_by_value", "") + f.get("value_mapping", "") + all_fields_by_mapping[field_key] = all_fields_by_mapping.get(field_key, []) + [f] + + return all_fields_by_mapping + + +def options_union(options_list): + # remove duplicates but keep any ordering + return list(dict.fromkeys(flatten(options_list[:]))) + + +def options_intersection(options_list): + num_instances = len(options_list) + flat_options = flatten(options_list[:]) + # only keep options that are present in all instances, preserving order + counter = {} + for option in flat_options: + counter[option] = counter.get(option, 0) + 1 + + intersection = [key for key in counter if counter[key] == num_instances] + return intersection + + +# any filters that exist in all beacons +# bins should be joined also, although some ordering may disappear +# still unclear if this is an useful feature or not +# shortcomings here can be addressed by keeping our configs consistent where possible +def fields_union(searchFields): + fields = fields_dict(searchFields) + + # create one entry for each mapping + union_fields = [] + for f in fields.values(): + entry = copy.deepcopy(f[0]) # arbitrarily get name, description, etc from first entry + entry["options"] = options_union([e["options"] for e in f]) + union_fields.append(entry) + + return union_fields + + +def fields_intersection(searchFields): + num_instances = len(searchFields) + fields = fields_dict(searchFields) + + # remove any fields not in all entries + intersection_dict = {mapping: entries for mapping, entries in fields.items() if len(entries) == num_instances} + + # create one entry for each mapping + intersection_fields = [] + for f in intersection_dict.values(): + entry = {} + entry = copy.deepcopy(f[0]) # arbitrarily get name, description, etc from first entry + entry["options"] = options_intersection([e["options"] for e in f]) + intersection_fields.append(entry) + + return intersection_fields From 25e4e363245d32b906c74fb4f5762517a3964954 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Thu, 15 Aug 2024 15:13:07 +0000 Subject: [PATCH 15/41] headers edge case --- bento_beacon/authz/headers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bento_beacon/authz/headers.py b/bento_beacon/authz/headers.py index 05ba24c3..9ec32084 100644 --- a/bento_beacon/authz/headers.py +++ b/bento_beacon/authz/headers.py @@ -2,6 +2,9 @@ def auth_header_getter(r: Request) -> dict[str, str]: + if not request: + # edge case for beacon network init, calling katsu / gohan with no request context + return None token = r.headers.get("authorization") return {"Authorization": token} if token else {} From 3426ab99444b81b98f2ad0cf940cc76bd04a626f Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Thu, 15 Aug 2024 15:15:14 +0000 Subject: [PATCH 16/41] config additions for network --- bento_beacon/network/network_config.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/bento_beacon/network/network_config.py b/bento_beacon/network/network_config.py index 38297381..738be983 100644 --- a/bento_beacon/network/network_config.py +++ b/bento_beacon/network/network_config.py @@ -2,25 +2,10 @@ # temp file, all of this to be handled elsewhere in final version # XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX -BEACONS = [ - "https://rsrq.bento.sd4h.ca/api/beacon", - "https://bqc19.bento.sd4h.ca/api/beacon", - # "https://staging.bqc19.bento.sd4h.ca/api/beacon", - "https://staging.bento.sd4h.ca/api/beacon", - "https://ichange.bento.sd4h.ca/api/beacon", - "https://qa.bento.sd4h.ca/api/beacon/", - # "https://bentov2.local/api/beacon", - "https://renata.bento.sd4h.ca/api/beacon", - "https://signature.bento.sd4h.ca/api/beacon", - # "https://bentov2.local/api/beacon" -] - NETWORK_INIT_TIMEOUT = 30 NETWORK_QUERY_WITH_VARIANTS_TIMEOUT = 2 * 60 NETWORK_QUERY_WITHOUT_VARIANTS_TIMEOUT = 30 -VALID_ENDPOINTS = ["analyses", "biosamples", "cohorts", "datasets", "g_variants", "individuals", "runs", "overview"] - KATSU_CONFIG_INTERSECTION = [ { From 709826115e49b487f715e5dd86905ccbe3d26f1f Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Thu, 15 Aug 2024 15:16:59 +0000 Subject: [PATCH 17/41] network config --- bento_beacon/config_files/config.py | 37 +++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/bento_beacon/config_files/config.py b/bento_beacon/config_files/config.py index a228b866..1950e443 100644 --- a/bento_beacon/config_files/config.py +++ b/bento_beacon/config_files/config.py @@ -1,5 +1,6 @@ import json import os +from ..constants import GRANULARITY_COUNT, GRANULARITY_RECORD GA4GH_BEACON_REPO_URL = "https://raw.githubusercontent.com/ga4gh-beacon/beacon-v2" @@ -12,12 +13,13 @@ class Config: # default when no requested granularity, as well as max granularity for anonymous users DEFAULT_GRANULARITY = { - "individuals": "count", - "variants": "count", - "biosamples": "count", - "cohorts": "record", - "datasets": "record", - "info": "record", + "individuals": GRANULARITY_COUNT, + "variants": GRANULARITY_COUNT, + "biosamples": GRANULARITY_COUNT, + "cohorts": GRANULARITY_RECORD, + "datasets": GRANULARITY_RECORD, + "info": GRANULARITY_RECORD, + "network": GRANULARITY_COUNT, } DEFAULT_PAGINATION_PAGE_SIZE = 10 @@ -154,7 +156,7 @@ class Config: MAP_EXTRA_PROPERTIES_TO_INFO = os.environ.get("MAP_EXTRA_PROPERTIES_TO_INFO", True) - PHENOPACKETS_SCHEMA_REFERENCE = {"entityType": "individual", "schema": "phenopackets v1"} + PHENOPACKETS_SCHEMA_REFERENCE = {"entityType": "individual", "schema": "phenopackets v2"} MAX_RETRIES_FOR_CENSORSHIP_PARAMS = 2 # ------------------- @@ -206,3 +208,24 @@ def retrieve_config_json(filename): BEACON_COHORT = retrieve_config_json("beacon_cohort.json") BEACON_CONFIG = retrieve_config_json("beacon_config.json") + + # ------------------- + # network + + NETWORK_CONFIG = retrieve_config_json("network_config.json") + + NETWORK_URLS = NETWORK_CONFIG.get("beacons", []) + NETWORK_INIT_TIMEOUT_SECONDS = NETWORK_CONFIG.get("network_init_timeout_seconds", 30) + NETWORK_QUERY_WITH_VARIANTS_TIMEOUT_SECONDS = NETWORK_CONFIG.get("network_query_with_variants_timeout_seconds", 120) + NETWORK_QUERY_WITHOUT_VARIANTS_TIMEOUT_SECONDS = NETWORK_CONFIG.get( + "network_query_without_variants_timeout_seconds", 30 + ) + NETWORK_VALID_QUERY_ENDPOINTS = [ + "analyses", + "biosamples", + "cohorts", + "datasets", + "g_variants", + "individuals", + "runs", + ] From 76bab6b813f2cf7a06d4ec0ad24802f7e2b9ed64 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Thu, 15 Aug 2024 15:18:20 +0000 Subject: [PATCH 18/41] delete temp hardcoded stuff --- bento_beacon/network/network_config.py | 1554 ------------------------ 1 file changed, 1554 deletions(-) delete mode 100644 bento_beacon/network/network_config.py diff --git a/bento_beacon/network/network_config.py b/bento_beacon/network/network_config.py deleted file mode 100644 index 738be983..00000000 --- a/bento_beacon/network/network_config.py +++ /dev/null @@ -1,1554 +0,0 @@ -# XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX -# temp file, all of this to be handled elsewhere in final version -# XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX - -NETWORK_INIT_TIMEOUT = 30 -NETWORK_QUERY_WITH_VARIANTS_TIMEOUT = 2 * 60 -NETWORK_QUERY_WITHOUT_VARIANTS_TIMEOUT = 30 - - -KATSU_CONFIG_INTERSECTION = [ - { - "section_title": "Common Filters", - "fields": [ - { - "config": { - "bin_size": 10, - "maximum": 100, - "minimum": 0, - "taper_left": 10, - "taper_right": 100, - "units": "years", - }, - "datatype": "number", - "description": "Age at arrival", - "id": "age", - "mapping": "individual/age_numeric", - "options": ["[30, 40)", "[40, 50)", "[50, 60)", "[60, 70)", "[70, 80)", "[80, 90)"], - "title": "Age", - }, - { - "config": {"enum": None}, - "datatype": "string", - "description": "Sex at birth", - "id": "sex", - "mapping": "individual/sex", - "options": ["MALE", "FEMALE"], - "title": "Sex", - }, - ], - } -] - - -KATSU_CONFIG_UNION = [ - { - "section_title": "All Filters", - "fields": [ - { - "config": {"enum": None}, - "datatype": "string", - "description": "Phenotypic features of the individual", - "id": "phenotypic_features_type", - "mapping": "individual/phenopackets/phenotypic_features/pftype/label", - "options": [ - "Asthma", - "Allergy", - "Rhinosinusitis", - "Overlap Syndrome", - "Tabagism", - "Asthma subject type", - "Phenotype", - "Loss of appetite", - "Abdominal pain", - "Headache", - "Cough", - "Cardiac arrest", - "Viral pneumonia/pneumonitis", - "Pulmonary hypertension", - "Nausea", - "HIV Infection", - "Hyperglycemia", - "Patient immunosuppressed", - "Fever", - "Chest pain", - "Pneumothorax", - "Dementia", - "Hypertension", - "Loss of sense of smell", - ], - "title": "Phenotypic Features", - }, - { - "config": { - "bin_size": 10, - "maximum": 100, - "minimum": 0, - "taper_left": 10, - "taper_right": 100, - "units": "years", - }, - "datatype": "number", - "description": "Age at arrival", - "id": "age", - "mapping": "individual/age_numeric", - "options": [ - "< 10", - "[10, 20)", - "[20, 30)", - "[30, 40)", - "[40, 50)", - "[50, 60)", - "[60, 70)", - "[70, 80)", - "[80, 90)", - "[90, 100)", - "< 18", - "[18, 30)", - "≥ 90", - ], - "title": "Age", - }, - { - "config": {"enum": None}, - "datatype": "string", - "description": "Sex at birth", - "id": "sex", - "mapping": "individual/sex", - "options": ["MALE", "FEMALE", "UNKNOWN_SEX"], - "title": "Sex", - }, - { - "config": {"enum": None}, - "datatype": "string", - "description": "Measurements performed", - "group_by": "assay/label", - "id": "measurement_types", - "mapping": "individual/phenopackets/measurements", - "options": [ - "Body Mass Index", - "Cp20 Measurement", - "Eosinophils Percentage", - "Histological Eosinophils", - "Histological Neutrophils", - "Immunoglobulin E", - "Neutrophils Percentage", - ], - "title": "Measurement types", - }, - { - "config": { - "bin_size": "10", - "maximum": 50, - "minimum": 0, - "taper_left": 0, - "taper_right": 50, - "units": "kg/m^2", - }, - "datatype": "number", - "description": "Measurements performed", - "group_by": "assay/id", - "group_by_value": "NCIT:C16358", - "id": "measurement_bmi", - "mapping": "individual/phenopackets/measurements", - "options": ["[0, 10)", "[10, 20)", "[20, 30)", "[30, 40)", "[40, 50)", "< 18", "[18, 30)", "≥ 30"], - "title": "BMI", - "value_mapping": "value/quantity/value", - }, - { - "config": {"enum": None}, - "datatype": "string", - "description": "Visits where measurements were performed", - "group_by": "extra_properties/visit_index", - "id": "measurement_visits", - "mapping": "individual/phenopackets/measurements", - "options": [ - "1", - "10", - "11", - "12", - "13", - "14", - "15", - "16", - "17", - "18", - "19", - "2", - "20", - "21", - "22", - "23", - "24", - "25", - "26", - "27", - "28", - "29", - "3", - "30", - "31", - "32", - "33", - "4", - "5", - "6", - "7", - "8", - "9", - ], - "title": "Recurring visits", - }, - { - "config": { - "bin_size": 50, - "maximum": 1000, - "minimum": 0, - "taper_left": 50, - "taper_right": 500, - "units": "years", - }, - "datatype": "number", - "description": "Measurements performed x", - "group_by": "assay/label", - "group_by_value": "Immunoglobulin E", - "id": "measurement_immnoglobulin_e", - "mapping": "individual/phenopackets/measurements", - "options": [ - "< 50", - "[50, 100)", - "[100, 150)", - "[150, 200)", - "[200, 250)", - "[250, 300)", - "[300, 350)", - "[350, 400)", - "[400, 450)", - "[450, 500)", - "≥ 500", - ], - "title": "Immunoglobulin E", - "value_mapping": "value/quantity/value", - }, - { - "config": { - "bin_size": 10, - "maximum": 1000, - "minimum": 0, - "taper_left": 10, - "taper_right": 100, - "units": "years", - }, - "datatype": "number", - "description": "Measurements performed Cp20", - "group_by": "assay/label", - "group_by_value": "Cp20 Measurement", - "id": "measurement_Cp20", - "mapping": "individual/phenopackets/measurements", - "options": [ - "< 10", - "[10, 20)", - "[20, 30)", - "[30, 40)", - "[40, 50)", - "[50, 60)", - "[60, 70)", - "[70, 80)", - "[80, 90)", - "[90, 100)", - "≥ 100", - ], - "title": "Cp20", - "value_mapping": "value/quantity/value", - }, - { - "config": {"enum": None}, - "datatype": "string", - "description": "Tissue from which the specimen was collected", - "id": "sampled_tissue", - "mapping": "individual/phenopackets/biosamples/sampled_tissue/label", - "options": [ - "Sputum-DTT plug", - "Cells from nasal brush-with reagent", - "Bronchoalveolar lavage supernatant aliquot", - "Sputum slide", - "Whole blood RNA", - "Blood cells", - "EDTA arterial plasma aliquot", - "Blood cells pellet", - "Nasal lavage or nasosorption supernatant", - "Urine aliquot", - "Sputum cells pellet", - "Nasal mucosa biopsy", - "DTT sputum supernatant aliquot", - "Cellules mononuclées isolées de sang périphérique (PBMC)", - "Buffy coat aliquot", - "Serum aliquot", - "Stools", - "Bronchial biopsies RNA", - "Plasma aliquot", - "Cells from nasal brush-dry", - "Buffycoat DNA", - "Nasal polyp biopsy", - "Macrophages RNA from bronchoalveolar lavage", - "Nasal secretion swab", - "EDTA whole blood aliquot", - "Sputum cells RNA", - "Bronchoalveolar lavage slide", - "Bronchial biopsy", - "Sputum supernatant aliquot without DTT", - "Venous serum aliquot", - "Optic Pathway", - "Parietal Lobe", - "Hypothalamus", - "Fronto-Temporal Lobe", - "Frontal Lobe", - "Cerebellum", - "Fronto-Parietal Lobe", - "Temporo-Occipital Lobe", - "Temporo-Parietal Lobe", - "Intraventricular", - "Fronto-Temporo-Insular", - "Unknown", - "Diencephalon", - "Brainstem", - "Cell Line", - "Supratentorial", - "Pineal", - "Thalamus", - "Ventricle", - "Hemisphere", - "Temporal Lobe", - "Ventricle (Lateral)", - "Suprasellar", - "Occipital Lobe", - "Mandible", - "Medulla Oblongata", - "Ventricle (Fourth)", - "Cerebellar Vermis", - "Ventricle (Third)", - "Cortex", - "Spinal Cord", - "Posterior Fossa", - "Optic Chiasm", - "Cerebello-Pontine Angle", - "Pons", - "Parieto-Occipital Lobe", - ], - "title": "Biosample Tissue Location", - }, - { - "config": {"bin_by": "month"}, - "datatype": "date", - "description": "Date of initial verbal consent (participant, legal representative or tutor), yyyy-mm-dd", - "id": "date_of_consent", - "mapping": "individual/extra_properties/date_of_consent", - "options": [ - "Jan 2020", - "Feb 2020", - "Mar 2020", - "Apr 2020", - "May 2020", - "Jun 2020", - "Jul 2020", - "Aug 2020", - "Sep 2020", - "Oct 2020", - "Nov 2020", - "Dec 2020", - "Jan 2021", - "Feb 2021", - "Mar 2021", - "Apr 2021", - "May 2021", - "Jun 2021", - "Jul 2021", - "Aug 2021", - "Sep 2021", - "Oct 2021", - "Nov 2021", - "Dec 2021", - "Jan 2022", - "Feb 2022", - "Mar 2022", - "Apr 2022", - "May 2022", - "Jun 2022", - "Jul 2022", - "Aug 2022", - "Sep 2022", - "Oct 2022", - "Nov 2022", - "Dec 2022", - "Jan 2023", - "Feb 2023", - "Mar 2023", - "Apr 2023", - "May 2023", - "Jun 2023", - "Jul 2023", - "Aug 2023", - "Sep 2023", - "Oct 2023", - "Nov 2023", - "Dec 2023", - ], - "title": "Verbal consent date", - }, - { - "config": { - "enum": [ - "I have no problems in walking about", - "I have slight problems in walking about", - "I have moderate problems in walking about", - "I have severe problems in walking about", - "I am unable to walk about", - ] - }, - "datatype": "string", - "description": "Mobility", - "id": "mobility", - "mapping": "individual/extra_properties/mobility", - "options": [ - "I have no problems in walking about", - "I have slight problems in walking about", - "I have moderate problems in walking about", - "I have severe problems in walking about", - "I am unable to walk about", - ], - "title": "Functional status", - }, - { - "config": {"enum": ["Uninfected", "Mild", "Moderate", "Severe", "Dead"]}, - "datatype": "string", - "description": "Covid severity", - "id": "covid_severity", - "mapping": "individual/extra_properties/covid_severity", - "options": ["Uninfected", "Mild", "Moderate", "Severe", "Dead"], - "title": "Covid severity", - }, - { - "config": {"enum": None}, - "datatype": "string", - "description": "Diseases observed as either present or absent", - "id": "diseases", - "mapping": "individual/phenopackets/diseases/term/label", - "options": [ - "viral pneumonia", - "Polycythemia vera", - "Non-Hodgkin Lymphoma", - "Hashimoto Thyroiditis", - "diabetes mellitus", - "Glioblastoma", - "coronary artery disease, autosomal dominant, 1", - "COVID-19", - "Rheumatologic disease", - "Marfan syndrome", - "Miller syndrome", - "Cystic fibrosis", - "Takotsubo cardiomyopathy", - "autoimmune disorder of cardiovascular system", - "Negative", - "Medulloblastoma", - "Testicular Cancer", - "Sarcoma", - "DIPG", - "Ovarian Tumor", - "Control", - "Neurofibromatosis", - "Desmoplastic infantile ganglioglioma (DIG)", - "GBM", - "HGG", - "Desmoplastic Infantile Astrocytoma (DIA)", - "Melanoma", - "Acute Lymphocytic Leukemia (ALL)", - "Ependymoma-PFB", - "Germinoma", - "Normal Brain", - "Schwannoma", - "Giant Cell Lesion (central)", - "Anaplastic Astrocytoma (AA)", - "Odontogenic Myxoma", - "Glioneuronal Tumor", - "Oligodendroglioma GrII", - "Craniopharyngioma", - "Ganglioglioma", - "Anaplastic Ependymoma", - "Brain Tumor", - "Neurofibroma", - "ATRT", - "Pilocytic Astrocytoma", - "Ependymoma-PFA", - "Angiosarcoma", - "DNET", - "Cemento Ossifying Fibroma", - "Meningioma", - "Diffuse Astrocytoma", - "Oral Granular Cell Tumor (OGCT)", - "Ewing's Sarcoma", - "HNSCC", - "Neuroblastoma", - "Acute Myeloid Leukemia (AML)", - "Parathyroid Tumor", - "Diffuse midline glioma, H3 K27M-mutant", - "Giant Cell Tumor (GCT)", - "Ependymoma - myxopapillary", - "Rhabdomyosarcoma", - "Glioma", - "Epithelioid Hemangioendothelioma", - "Endometrial Sarcoma", - "Giant Cell Lesion (peripheral)", - "Low Grade Glioma (LGG)", - "Astrocytoma", - "ETMR", - "Subcutaneous Panniculitis-like T-cell lymphoma", - "Giant Cell lesion", - "PNET", - "Anaplastic Oligoastrocytoma", - "Osteosarcoma", - "Pilomyxoid Astrocytoma (PMA)", - "Pleomorphic Xanthoastrocytoma (PXA)", - "Immunodeficiency", - "Cortical Dysplasia", - "Oligoastrocytoma", - "Anaplastic Oligodendroglioma", - "Ependymoma", - "Choroid Plexus Carcinoma", - "Oral Squamous Cell Carcinoma (OSCC)", - "Chondroblastoma", - ], - "title": "Diseases", - }, - { - "config": {"enum": None}, - "datatype": "string", - "description": "Tissue from which the biosample was extracted", - "id": "tissues", - "mapping": "biosample/sampled_tissue/label", - "mapping_for_search_filter": "individual/biosamples/sampled_tissue/label", - "options": ["Plasma", "blood", "Serum"], - "title": "Sampled Tissues", - }, - { - "config": {"bins": [200, 300, 500, 1000, 1500, 2000], "minimum": 0, "units": "mg/L"}, - "datatype": "number", - "description": "Numeric measures from a laboratory test", - "id": "lab_test_result_value", - "mapping": "individual/extra_properties/lab_test_result_value", - "options": [ - "< 200", - "[200, 300)", - "[300, 500)", - "[500, 1000)", - "[1000, 1500)", - "[1500, 2000)", - "≥ 2000", - ], - "title": "Lab Test Result", - }, - { - "config": {"enum": None}, - "datatype": "string", - "description": "A clinical procedure performed on a subject", - "group_by": "procedure/code/label", - "id": "medical_procedures", - "mapping": "individual/phenopackets/medical_actions", - "options": [ - "Liver Biopsy", - "Magnetic Resonance Imaging", - "Positron Emission Tomography", - "Punch Biopsy", - "X-Ray Imaging", - ], - "title": "Medical Procedures", - }, - { - "config": {"enum": None}, - "datatype": "string", - "description": "Treatment with an agent such as a drug", - "group_by": "treatment/agent/label", - "id": "medical_treatments", - "mapping": "individual/phenopackets/medical_actions", - "options": ["Acetaminophen", "Ibuprofen", "NCIT:C1119"], - "title": "Medical Treatments", - }, - { - "config": {"enum": None}, - "datatype": "string", - "description": "Interpretation for an individual variant or gene (CANDIDATE, CONTRIBUTORY, etc)", - "id": "interpretation_status", - "mapping": "individual/phenopackets/interpretations/diagnosis/genomic_interpretations/interpretation_status", - "options": ["CONTRIBUTORY"], - "title": "Genomic Interpretations", - }, - { - "config": {"enum": None}, - "datatype": "string", - "description": "ACMG Pathogenicity category for a particular variant (BENIGN, PATHOGENIC, etc)", - "id": "acmg_pathogenicity_classification", - "mapping": "individual/phenopackets/interpretations/diagnosis/genomic_interpretations/variant_interpretation/acmg_pathogenicity_classification", - "options": ["PATHOGENIC"], - "title": "Variant Pathogenicity", - }, - { - "config": {"enum": None}, - "datatype": "string", - "description": "Types of experiments performed on a sample", - "id": "experiment_type", - "mapping": "experiment/experiment_type", - "mapping_for_search_filter": "individual/biosamples/experiment/experiment_type", - "options": [ - "Other", - "Neutralizing antibody titers", - "Metabolite profiling", - "RNA-Seq", - "WGS", - "Proteomic profiling", - "WES", - ], - "title": "Experiment Types", - }, - { - "config": {"enum": None}, - "datatype": "string", - "description": "Study type of the experiment (e.g. Genomics, Transcriptomics, etc.)", - "id": "experiment_study_type", - "mapping": "experiment/study_type", - "mapping_for_search_filter": "individual/biosamples/experiment/study_type", - "options": ["Serology", "Other", "Genomics", "Proteomics", "Transcriptomics", "Metabolomics"], - "title": "Study Types", - }, - { - "config": {"enum": None}, - "datatype": "string", - "description": "File type of experiment results files", - "id": "experiment_results_file_type", - "mapping": "experiment/experiment_results/file_format", - "mapping_for_search_filter": "individual/biosamples/experiment/experiment_results/file_format", - "options": ["PDF", "XLSX", "JPEG", "CRAM", "VCF", "MP4", "DOCX", "CSV", "MARKDOWN"], - "title": "Results File Types", - }, - { - "config": {"bin_by": "month"}, - "datatype": "date", - "description": "Date of initial verbal consent (participant, legal representative or tutor)", - "id": "dconsverbpa", - "mapping": "individual/extra_properties/dconsverbpa", - "options": [ - "Mar 2020", - "Apr 2020", - "May 2020", - "Jun 2020", - "Jul 2020", - "Aug 2020", - "Sep 2020", - "Oct 2020", - "Nov 2020", - "Dec 2020", - "Jan 2021", - "Feb 2021", - "Mar 2021", - "Apr 2021", - "May 2021", - "Jun 2021", - "Jul 2021", - "Aug 2021", - "Sep 2021", - "Oct 2021", - "Nov 2021", - "Dec 2021", - "Jan 2022", - "Feb 2022", - "Mar 2022", - "Apr 2022", - "May 2022", - "Jun 2022", - "Jul 2022", - "Aug 2022", - "Sep 2022", - "Oct 2022", - "Nov 2022", - "Dec 2022", - "Jan 2023", - "Feb 2023", - "Mar 2023", - "Apr 2023", - "May 2023", - "Jun 2023", - "Jul 2023", - "Aug 2023", - ], - "title": "Verbal consent date", - }, - { - "config": {"enum": ["Hospitalized", "Outpatient"]}, - "datatype": "string", - "description": "Has the participant been hospitalized or is the participant seen as an outpatient?", - "id": "type_partic", - "mapping": "individual/extra_properties/type_partic", - "options": ["Hospitalized", "Outpatient"], - "title": "Hospitalization", - }, - { - "config": {"enum": ["Non-smoker", "Smoker", "Former smoker", "Passive smoker", "Not specified"]}, - "datatype": "string", - "description": "Smoking status", - "id": "smoking", - "mapping": "individual/extra_properties/smoking", - "options": ["Non-smoker", "Smoker", "Former smoker", "Passive smoker", "Not specified"], - "title": "Smoking", - }, - { - "config": {"enum": ["Positive", "Negative", "Indeterminate"]}, - "datatype": "string", - "description": "Final COVID status according to the PCR test", - "id": "covidstatus", - "mapping": "individual/extra_properties/covidstatus", - "options": ["Positive", "Negative", "Indeterminate"], - "title": "COVID status", - }, - { - "config": {"enum": ["Alive", "Deceased"]}, - "datatype": "string", - "description": "Vital status at discharge", - "id": "death_dc", - "mapping": "individual/extra_properties/death_dc", - "options": ["Alive", "Deceased"], - "title": "Vital status", - }, - { - "config": {"bins": [20, 25, 27, 30, 35, 40], "units": "kg/m^2"}, - "datatype": "number", - "description": "BMI", - "id": "bmi", - "mapping": "individual/extra_properties/bmi", - "options": ["< 20", "[20, 25)", "[25, 27)", "[27, 30)", "[30, 35)", "[35, 40)", "≥ 40"], - "title": "BMI", - }, - { - "config": {"enum": ["Yes", "No"]}, - "datatype": "string", - "description": "Intensive Care Unit admission?", - "id": "icu", - "mapping": "individual/extra_properties/icu", - "options": ["Yes", "No"], - "title": "ICU", - }, - { - "config": {"enum": ["Hospitalized adult", "Adult outpatient", "Pediatric", "Pregnant woman"]}, - "datatype": "string", - "description": "To which category the participant belongs?", - "id": "core_cohorte", - "mapping": "individual/extra_properties/core_cohorte", - "options": ["Hospitalized adult", "Adult outpatient", "Pediatric", "Pregnant woman"], - "title": "Participant category", - }, - { - "config": { - "enum": [ - "At home", - "In residence for the elderly (RPA)", - "Nursing home (CHSLD)", - "In intermediate and family-type resources", - "In rooming house", - "Homeless", - ] - }, - "datatype": "string", - "description": "Living where?", - "id": "livewhere", - "mapping": "individual/extra_properties/livewhere", - "options": [ - "At home", - "In residence for the elderly (RPA)", - "Nursing home (CHSLD)", - "In intermediate and family-type resources", - "In rooming house", - "Homeless", - ], - "title": "Domicile", - }, - { - "config": {"enum": ["Yes", "No"]}, - "datatype": "string", - "description": "Has the participant been vaccinated?", - "id": "vaccinate", - "mapping": "individual/extra_properties/vaccinate", - "options": ["Yes", "No"], - "title": "Vaccinal status", - }, - { - "config": {"enum": ["1", "2", "3", "4"]}, - "datatype": "string", - "description": "Number of doses received", - "id": "vaccin_dosenum", - "mapping": "individual/extra_properties/vaccin_dosenum", - "options": ["1", "2", "3", "4"], - "title": "Vaccine dose", - }, - { - "config": { - "enum": [ - "I have no problem washing or dressing myself", - "I have slight problem washing or dressing myself", - "I have moderate problems washing or dressing myself", - "I have severe problems washing or dressing myself", - "I am unable to wash or dress myself", - ] - }, - "datatype": "string", - "description": "Self-care", - "id": "selfcare", - "mapping": "individual/extra_properties/selfcare", - "options": [ - "I have no problem washing or dressing myself", - "I have slight problem washing or dressing myself", - "I have moderate problems washing or dressing myself", - "I have severe problems washing or dressing myself", - "I am unable to wash or dress myself", - ], - "title": "Self-care", - }, - { - "config": {"enum": ["Yes", "No"]}, - "datatype": "string", - "description": "Other chronic cardiac disease?", - "id": "phx_cardiac", - "mapping": "individual/extra_properties/phx_cardiac", - "options": ["Yes", "No"], - "title": "Cardiac history", - }, - { - "config": {"enum": ["Yes", "No"]}, - "datatype": "string", - "description": "Prior transient ischemic attack (TIA)?", - "id": "phx_tia", - "mapping": "individual/extra_properties/phx_tia", - "options": ["Yes", "No"], - "title": "TIA", - }, - { - "config": {"enum": ["Yes", "No"]}, - "datatype": "string", - "description": "Asthma?", - "id": "phx_asthma", - "mapping": "individual/extra_properties/phx_asthma", - "options": ["Yes", "No"], - "title": "Asthma", - }, - { - "config": {"enum": ["Yes", "No"]}, - "datatype": "string", - "description": "Prior stroke?", - "id": "phx_cva", - "mapping": "individual/extra_properties/phx_cva", - "options": ["Yes", "No"], - "title": "Stroke", - }, - { - "config": {"enum": ["Yes", "No"]}, - "datatype": "string", - "description": "Diabetes?", - "id": "phx_diabetes", - "mapping": "individual/extra_properties/phx_diabetes", - "options": ["Yes", "No"], - "title": "Diabetes", - }, - { - "config": {"enum": ["Yes", "No"]}, - "datatype": "string", - "description": "Malignant neoplasm?", - "id": "phx_cancer", - "mapping": "individual/extra_properties/phx_cancer", - "options": ["Yes", "No"], - "title": "Cancer", - }, - { - "config": {"enum": ["Yes", "No"]}, - "datatype": "string", - "description": "Dementia?", - "id": "phx_dementia", - "mapping": "individual/extra_properties/phx_dementia", - "options": ["Yes", "No"], - "title": "Dementia", - }, - { - "config": {"enum": ["Yes", "No"]}, - "datatype": "string", - "description": "Atrial fibrillation or flutter?", - "id": "phx_afib", - "mapping": "individual/extra_properties/phx_afib", - "options": ["Yes", "No"], - "title": "Atrial fibrillation", - }, - { - "config": {"enum": ["Yes", "No"]}, - "datatype": "string", - "description": "HIV or AIDS?", - "id": "phx_hiv", - "mapping": "individual/extra_properties/phx_hiv", - "options": ["Yes", "No"], - "title": "HIV", - }, - { - "config": {"enum": ["Yes", "No"]}, - "datatype": "string", - "description": "Arterial Hypertension?", - "id": "phx_htn", - "mapping": "individual/extra_properties/phx_htn", - "options": ["Yes", "No"], - "title": "Arterial Hypertension", - }, - { - "config": {"enum": ["Yes", "No"]}, - "datatype": "string", - "description": "Immunosuppressed state?", - "id": "phx_imm", - "mapping": "individual/extra_properties/phx_imm", - "options": ["Yes", "No"], - "title": "Immunosupression", - }, - { - "config": {"enum": ["Yes", "No"]}, - "datatype": "string", - "description": "Prior myocardial infarction?", - "id": "phx_mi", - "mapping": "individual/extra_properties/phx_mi", - "options": ["Yes", "No"], - "title": "Myocardial infarction", - }, - { - "config": {"enum": ["Yes", "No"]}, - "datatype": "string", - "description": "Heart failure?", - "id": "phx_chf", - "mapping": "individual/extra_properties/phx_chf", - "options": ["Yes", "No"], - "title": "Heart failure", - }, - { - "config": {"enum": ["Yes", "No"]}, - "datatype": "string", - "description": "Coronary artery disease?", - "id": "phx_cad", - "mapping": "individual/extra_properties/phx_cad", - "options": ["Yes", "No"], - "title": "Coronary artery disease", - }, - { - "config": {"enum": ["Yes", "No"]}, - "datatype": "string", - "description": "Liver disease?", - "id": "phx_liver", - "mapping": "individual/extra_properties/phx_liver", - "options": ["Yes", "No"], - "title": "Liver disease", - }, - { - "config": {"enum": ["Yes", "No"]}, - "datatype": "string", - "description": "Chronic obstructive pulmonary disease (emphysema, chronic bronchitis)?", - "id": "phx_copd", - "mapping": "individual/extra_properties/phx_copd", - "options": ["Yes", "No"], - "title": "COPD", - }, - { - "config": {"enum": ["Yes", "No"]}, - "datatype": "string", - "description": "Psychiatric disease?", - "id": "phx_psych", - "mapping": "individual/extra_properties/phx_psych", - "options": ["Yes", "No"], - "title": "Psychiatric disease", - }, - { - "config": {"enum": ["Yes", "No"]}, - "datatype": "string", - "description": "Chronic kidney disease?", - "id": "phx_ckd", - "mapping": "individual/extra_properties/phx_ckd", - "options": ["Yes", "No"], - "title": "Chronic kidney disease", - }, - { - "config": {"enum": ["Yes", "No"]}, - "datatype": "string", - "description": "Dialysis?", - "id": "phx_dialysis", - "mapping": "individual/extra_properties/phx_dialysis", - "options": ["Yes", "No"], - "title": "Dialysis", - }, - { - "config": {"enum": ["Yes", "No"]}, - "datatype": "string", - "description": "Rheumatologic disease?", - "id": "phx_rheum", - "mapping": "individual/extra_properties/phx_rheum", - "options": ["Yes", "No"], - "title": "Rheumatologic disease", - }, - { - "config": {"bins": [5, 10, 50, 100], "units": "mg/L"}, - "datatype": "number", - "description": "C-reactive protein (CRP)", - "id": "lab_crp", - "mapping": "individual/extra_properties/lab_crp", - "options": ["< 5", "[5, 10)", "[10, 50)", "[50, 100)", "≥ 100"], - "title": "CRP", - }, - { - "config": {"enum": ["Yes", "No"]}, - "datatype": "string", - "description": "Chest X-ray?", - "id": "cxr", - "mapping": "individual/extra_properties/cxr", - "options": ["Yes", "No"], - "title": "Chest X-ray", - }, - { - "config": {"bins": [4, 10, 15], "units": "×10^9/L"}, - "datatype": "number", - "description": "Total White Blood Cell (WBC) count", - "id": "lab_wbc", - "mapping": "individual/extra_properties/lab_wbc", - "options": ["< 4", "[4, 10)", "[10, 15)", "≥ 15"], - "title": "WBC", - }, - { - "config": {"bins": [70, 90, 110, 130, 150], "units": "g/L"}, - "datatype": "number", - "description": "Haemoglobin", - "id": "lab_hg", - "mapping": "individual/extra_properties/lab_hg", - "options": ["< 70", "[70, 90)", "[90, 110)", "[110, 130)", "[130, 150)", "≥ 150"], - "title": "Haemoglobin", - }, - { - "config": {"bins": [50, 90, 120, 200, 300], "units": "μmol/L"}, - "datatype": "number", - "description": "Creatinine", - "id": "lab_cr", - "mapping": "individual/extra_properties/lab_cr", - "options": ["< 50", "[50, 90)", "[90, 120)", "[120, 200)", "[200, 300)", "≥ 300"], - "title": "Creatinine", - }, - { - "config": {"bins": [200, 300, 500, 1000, 1500, 2000], "units": "ng/mL"}, - "datatype": "number", - "description": "NT-proBNP", - "id": "lab_ntprobnp", - "mapping": "individual/extra_properties/lab_ntprobnp", - "options": [ - "< 200", - "[200, 300)", - "[300, 500)", - "[500, 1000)", - "[1000, 1500)", - "[1500, 2000)", - "≥ 2000", - ], - "title": "NT-proBNP", - }, - { - "config": {"bins": [200, 500, 1000, 1500, 2000], "units": "U/L"}, - "datatype": "number", - "description": "Lactate Dehydrogenase", - "id": "lab_ldh", - "mapping": "individual/extra_properties/lab_ldh", - "options": ["< 200", "[200, 500)", "[500, 1000)", "[1000, 1500)", "[1500, 2000)", "≥ 2000"], - "title": "LDH", - }, - { - "config": {"bins": [500, 1000, 2000, 5000], "units": "μg/L"}, - "datatype": "number", - "description": "D-Dimer", - "id": "lab_ddimer", - "mapping": "individual/extra_properties/lab_ddimer", - "options": ["< 500", "[500, 1000)", "[1000, 2000)", "[2000, 5000)", "≥ 5000"], - "title": "D-Dimer", - }, - { - "config": {"enum": ["Yes", "No"]}, - "datatype": "string", - "description": "Did or does the patient receive ventilatory support?", - "id": "repsupport_yesno", - "mapping": "individual/extra_properties/repsupport_yesno", - "options": ["Yes", "No"], - "title": "Respiratory support", - }, - { - "config": {"enum": ["Yes", "No"]}, - "datatype": "string", - "description": "Have you been diagnosed with a new or recurrent case of COVID since your last follow-up?", - "id": "newcovid", - "mapping": "individual/extra_properties/newcovid", - "options": ["Yes", "No"], - "title": "Reinfection", - }, - { - "config": {"enum": ["Yes", "No"]}, - "datatype": "string", - "description": "Does the participant report persistent symptoms related to SARS-CoV-2 infection?", - "id": "sx_report", - "mapping": "individual/extra_properties/sx_report", - "options": ["Yes", "No"], - "title": "Persisting symptoms", - }, - { - "config": {"enum": ["Yes", "No", "Not available"]}, - "datatype": "string", - "description": "Systemic corticosteroid?", - "id": "rx_roid", - "mapping": "individual/extra_properties/rx_roid", - "options": ["Yes", "No", "Not available"], - "title": "Systemic corticosteroid", - }, - { - "config": {"enum": ["Yes", "No", "Not available"]}, - "datatype": "string", - "description": "Antocoagulants?", - "id": "rx_aco", - "mapping": "individual/extra_properties/rx_aco", - "options": ["Yes", "No", "Not available"], - "title": "Antocoagulants", - }, - { - "config": {"enum": ["Worse", "Same", "Better", "Indeterminate"]}, - "datatype": "string", - "description": "Ability to self-care at discharge versus pre-COVID", - "id": "selfcare_dc", - "mapping": "individual/extra_properties/selfcare_dc", - "options": ["Worse", "Same", "Better", "Indeterminate"], - "title": "Self-care post covid", - }, - { - "config": {"enum": ["Yes", "No"]}, - "datatype": "string", - "description": "Acute kidney injury?", - "id": "c_aki", - "mapping": "individual/extra_properties/c_aki", - "options": ["Yes", "No"], - "title": "Acute kidney injury", - }, - { - "config": {"enum": ["Yes", "No"]}, - "datatype": "string", - "description": "Acute Respiratory Distress Syndrome (ARDS)?", - "id": "c_ards", - "mapping": "individual/extra_properties/c_ards", - "options": ["Yes", "No"], - "title": "ARDS", - }, - { - "config": {"enum": ["Yes", "No"]}, - "datatype": "string", - "description": "Deep vein thrombosis (DVT)?", - "id": "c_dvt", - "mapping": "individual/extra_properties/c_dvt", - "options": ["Yes", "No"], - "title": "Deep vein thrombosis", - }, - { - "config": {"enum": ["Yes", "No"]}, - "datatype": "string", - "description": "Stroke?", - "id": "c_stroke", - "mapping": "individual/extra_properties/c_stroke", - "options": ["Yes", "No"], - "title": "Stroke", - }, - { - "config": {"enum": ["Yes", "No"]}, - "datatype": "string", - "description": "Pulmonary embolism?", - "id": "c_pe", - "mapping": "individual/extra_properties/c_pe", - "options": ["Yes", "No"], - "title": "Pulmonary embolism", - }, - { - "config": {"enum": None}, - "datatype": "string", - "description": "The type of molecule that was extracted from the biological material.", - "id": "molecule", - "mapping": "experiment/molecule", - "mapping_for_search_filter": "individual/phenopackets/biosamples/experiment/molecule", - "options": ["genomic DNA"], - "title": "Molecules Used", - }, - ], - } -] - - -# KATSU_CONFIG_UNION = [ -# { -# "section_title": "General", -# "fields": [ -# { -# "mapping": "individual/age_numeric", -# "title": "Age", -# "description": "Age at arrival", -# "datatype": "number", -# "config": { -# "bin_size": 10, -# "taper_left": 10, -# "taper_right": 100, -# "units": "years", -# "minimum": 0, -# "maximum": 100, -# }, -# "id": "age", -# "options": [ -# "< 10", -# "[10, 20)", -# "[20, 30)", -# "[30, 40)", -# "[40, 50)", -# "[50, 60)", -# "[60, 70)", -# "[70, 80)", -# "[80, 90)", -# "[90, 100)", -# ], -# }, -# { -# "mapping": "individual/sex", -# "title": "Sex", -# "description": "Sex at birth", -# "datatype": "string", -# "config": {"enum": None}, -# "id": "sex", -# "options": ["MALE", "FEMALE"], -# }, -# { -# "mapping": "individual/extra_properties/date_of_consent", -# "title": "Verbal consent date", -# "description": "Date of initial verbal consent (participant, legal representative or tutor), yyyy-mm-dd", -# "datatype": "date", -# "config": {"bin_by": "month"}, -# "id": "date_of_consent", -# "options": [ -# "Jan 2020", -# "Feb 2020", -# "Mar 2020", -# "Apr 2020", -# "May 2020", -# "Jun 2020", -# "Jul 2020", -# "Aug 2020", -# "Sep 2020", -# "Oct 2020", -# "Nov 2020", -# "Dec 2020", -# "Jan 2021", -# "Feb 2021", -# "Mar 2021", -# "Apr 2021", -# "May 2021", -# "Jun 2021", -# "Jul 2021", -# "Aug 2021", -# "Sep 2021", -# "Oct 2021", -# "Nov 2021", -# "Dec 2021", -# "Jan 2022", -# "Feb 2022", -# "Mar 2022", -# "Apr 2022", -# "May 2022", -# "Jun 2022", -# "Jul 2022", -# "Aug 2022", -# "Sep 2022", -# "Oct 2022", -# "Nov 2022", -# "Dec 2022", -# "Jan 2023", -# "Feb 2023", -# "Mar 2023", -# "Apr 2023", -# "May 2023", -# "Jun 2023", -# "Jul 2023", -# "Aug 2023", -# "Sep 2023", -# "Oct 2023", -# "Nov 2023", -# "Dec 2023", -# ], -# }, -# { -# "mapping": "individual/extra_properties/mobility", -# "title": "Functional status", -# "description": "Mobility", -# "datatype": "string", -# "config": { -# "enum": [ -# "I have no problems in walking about", -# "I have slight problems in walking about", -# "I have moderate problems in walking about", -# "I have severe problems in walking about", -# "I am unable to walk about", -# ] -# }, -# "id": "mobility", -# "options": [ -# "I have no problems in walking about", -# "I have slight problems in walking about", -# "I have moderate problems in walking about", -# "I have severe problems in walking about", -# "I am unable to walk about", -# ], -# }, -# { -# "mapping": "individual/extra_properties/covid_severity", -# "title": "Covid severity", -# "description": "Covid severity", -# "datatype": "string", -# "config": {"enum": ["Uninfected", "Mild", "Moderate", "Severe", "Dead"]}, -# "id": "covid_severity", -# "options": ["Uninfected", "Mild", "Moderate", "Severe", "Dead"], -# }, -# { -# "mapping": "individual/phenopackets/phenotypic_features/pftype/label", -# "title": "Phenotypic Features", -# "description": "Individual phenotypic features, observed as either present or absent", -# "datatype": "string", -# "config": {"enum": None}, -# "id": "phenotypic_features", -# "options": [ -# "Loss of appetite", -# "Asthma", -# "Abdominal pain", -# "Headache", -# "Cough", -# "Cardiac arrest", -# "Nausea", -# "Pulmonary hypertension", -# "Viral pneumonia/pneumonitis", -# "HIV Infection", -# "Hyperglycemia", -# "Patient immunosuppressed", -# "Fever", -# "Chest pain", -# "Pneumothorax", -# "Hypertension", -# "Dementia", -# "Loss of sense of smell", -# ], -# }, -# { -# "mapping": "individual/phenopackets/diseases/term/label", -# "title": "Diseases", -# "description": "Diseases observed as either present or absent", -# "datatype": "string", -# "config": {"enum": None}, -# "id": "diseases", -# "options": [ -# "viral pneumonia", -# "Glioblastoma", -# "Polycythemia vera", -# "Miller syndrome", -# "Cystic fibrosis", -# "coronary artery disease, autosomal dominant, 1", -# "Hashimoto Thyroiditis", -# "Rheumatologic disease", -# "Takotsubo cardiomyopathy", -# "autoimmune disorder of cardiovascular system", -# "COVID-19", -# "Marfan syndrome", -# "Non-Hodgkin Lymphoma", -# "diabetes mellitus", -# ], -# }, -# { -# "mapping": "biosample/sampled_tissue/label", -# "mapping_for_search_filter": "individual/biosamples/sampled_tissue/label", -# "title": "Sampled Tissues", -# "description": "Tissue from which the biosample was extracted", -# "datatype": "string", -# "config": {"enum": None}, -# "id": "tissues", -# "options": ["Plasma", "blood", "Serum"], -# }, -# ], -# }, -# { -# "section_title": "Measurements", -# "fields": [ -# { -# "mapping": "individual/extra_properties/lab_test_result_value", -# "title": "Lab Test Result", -# "description": "Numeric measures from a laboratory test", -# "datatype": "number", -# "config": {"bins": [200, 300, 500, 1000, 1500, 2000], "minimum": 0, "units": "mg/L"}, -# "id": "lab_test_result_value", -# "options": [ -# "< 200", -# "[200, 300)", -# "[300, 500)", -# "[500, 1000)", -# "[1000, 1500)", -# "[1500, 2000)", -# "≥ 2000", -# ], -# }, -# { -# "mapping": "individual/phenopackets/measurements", -# "group_by": "assay/id", -# "group_by_value": "NCIT:C16358", -# "value_mapping": "value/quantity/value", -# "title": "BMI", -# "description": "Body Mass Index", -# "datatype": "number", -# "config": {"bins": [18.5, 30], "minimum": 0, "units": "kg/m^2"}, -# "id": "bmi", -# "options": ["< 18", "[18, 30)", "≥ 30"], -# }, -# ], -# }, -# { -# "section_title": "Medical Actions", -# "fields": [ -# { -# "mapping": "individual/phenopackets/medical_actions", -# "group_by": "procedure/code/label", -# "title": "Medical Procedures", -# "description": "A clinical procedure performed on a subject", -# "datatype": "string", -# "config": {"enum": None}, -# "id": "medical_procedures", -# "options": [ -# "Liver Biopsy", -# "Magnetic Resonance Imaging", -# "Positron Emission Tomography", -# "Punch Biopsy", -# "X-Ray Imaging", -# ], -# }, -# { -# "mapping": "individual/phenopackets/medical_actions", -# "group_by": "treatment/agent/label", -# "title": "Medical Treatments", -# "description": "Treatment with an agent such as a drug", -# "datatype": "string", -# "config": {"enum": None}, -# "id": "medical_treatments", -# "options": ["Acetaminophen", "Ibuprofen", "Ondansetron"], -# }, -# ], -# }, -# { -# "section_title": "Interpretations", -# "fields": [ -# { -# "mapping": "individual/phenopackets/interpretations/diagnosis/genomic_interpretations/interpretation_status", -# "title": "Genomic Interpretations", -# "description": "Interpretation for an individual variant or gene (CANDIDATE, CONTRIBUTORY, etc)", -# "datatype": "string", -# "config": {"enum": None}, -# "id": "interpretation_status", -# "options": ["CONTRIBUTORY"], -# }, -# { -# "mapping": "individual/phenopackets/interpretations/diagnosis/genomic_interpretations/variant_interpretation/acmg_pathogenicity_classification", -# "title": "Variant Pathogenicity", -# "description": "ACMG Pathogenicity category for a particular variant (BENIGN, PATHOGENIC, etc)", -# "datatype": "string", -# "config": {"enum": None}, -# "id": "acmg_pathogenicity_classification", -# "options": ["PATHOGENIC"], -# }, -# ], -# }, -# { -# "section_title": "Experiments", -# "fields": [ -# { -# "mapping": "experiment/experiment_type", -# "mapping_for_search_filter": "individual/biosamples/experiment/experiment_type", -# "title": "Experiment Types", -# "description": "Types of experiments performed on a sample", -# "datatype": "string", -# "config": {"enum": None}, -# "id": "experiment_type", -# "options": [ -# "Other", -# "Neutralizing antibody titers", -# "Metabolite profiling", -# "RNA-Seq", -# "WGS", -# "Proteomic profiling", -# ], -# }, -# { -# "mapping": "experiment/study_type", -# "mapping_for_search_filter": "individual/biosamples/experiment/study_type", -# "title": "Study Types", -# "description": "Study type of the experiment (e.g. Genomics, Transcriptomics, etc.)", -# "datatype": "string", -# "config": {"enum": None}, -# "id": "experiment_study_type", -# "options": ["Serology", "Genomics", "Other", "Proteomics", "Transcriptomics", "Metabolomics"], -# }, -# { -# "mapping": "experiment/experiment_results/file_format", -# "mapping_for_search_filter": "individual/biosamples/experiment/experiment_results/file_format", -# "title": "Results File Types", -# "description": "File type of experiment results files", -# "datatype": "string", -# "config": {"enum": None}, -# "id": "experiment_results_file_type", -# "options": ["XLSX", "CRAM", "VCF", "JPEG", "MP4", "DOCX", "CSV", "MARKDOWN"], -# }, -# ], -# }, -# ] - -# KATSU_CONFIG_INTERSECTION = [ -# { -# "section_title": "General", -# "fields": [ -# { -# "mapping": "individual/age_numeric", -# "title": "Age", -# "description": "Age at arrival", -# "datatype": "number", -# "config": { -# "bin_size": 10, -# "taper_left": 10, -# "taper_right": 100, -# "units": "years", -# "minimum": 0, -# "maximum": 100, -# }, -# "id": "age", -# "options": [ -# "< 10", -# "[10, 20)", -# "[20, 30)", -# "[30, 40)", -# "[40, 50)", -# "[50, 60)", -# "[60, 70)", -# "[70, 80)", -# "[80, 90)", -# "[90, 100)", -# ], -# }, -# { -# "mapping": "individual/sex", -# "title": "Sex", -# "description": "Sex at birth", -# "datatype": "string", -# "config": {"enum": None}, -# "id": "sex", -# "options": ["MALE", "FEMALE"], -# }, -# ], -# }, -# ] From 2d33fdeac2f3316531b2a65bedebbaf02677e1b8 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Thu, 15 Aug 2024 15:19:46 +0000 Subject: [PATCH 19/41] network updates --- bento_beacon/network/network.py | 83 ++++++++++----------------------- 1 file changed, 25 insertions(+), 58 deletions(-) diff --git a/bento_beacon/network/network.py b/bento_beacon/network/network.py index 3c400fb5..a58ca0d7 100644 --- a/bento_beacon/network/network.py +++ b/bento_beacon/network/network.py @@ -1,12 +1,8 @@ -from flask import current_app, g, request, Blueprint -from json import JSONDecodeError -import requests -from urllib.parse import urlsplit, urlunsplit -from ..utils.exceptions import APIException, NotFoundException -from .utils import beacon_network_response, network_beacon_get, network_beacon_post -from .network_config import VALID_ENDPOINTS, KATSU_CONFIG_UNION, KATSU_CONFIG_INTERSECTION +from flask import current_app, request, Blueprint +from ..utils.exceptions import NotFoundException +from .utils import network_beacon_get, network_beacon_post, host_beacon_response, filters_intersection, filters_union -network = Blueprint("network", __name__) +network = Blueprint("network", __name__, url_prefix="/network") # TODOs: @@ -17,69 +13,40 @@ # standard beacon info endpoints at the network level: /map, /configuration, etc # handle GET args +# timeout param +# clean up bento config file -@network.route("/network") -@network.route("/network/beacons") -def network_beacons(): - beacons = current_app.config["NETWORK_BEACONS"] - # temp, fake - filters_union = KATSU_CONFIG_UNION - filters_intersection = KATSU_CONFIG_INTERSECTION +@network.route("") +@network.route("/beacons") +def network_beacons(): + beacons_dict = current_app.config["NETWORK_BEACONS"] + # filters handling still experimental return { - "filtersUnion": filters_union, - "filtersIntersection": filters_intersection, - "beacons": list(beacons.values()), + "filtersUnion": current_app.config["ALL_NETWORK_FILTERS"], + "filtersIntersection": current_app.config["COMMON_NETWORK_FILTERS"], + "beacons": list(beacons_dict.values()), } -@network.route("/network/query/", methods=["POST"]) -def dumb_network_query(endpoint): - """ - Beacon network query in a single request and single response. - Returns an aggregate response as well as an array of individual beacon responses. - As slow as the slowest beacon. - """ - beacons = current_app.config["NETWORK_BEACONS"] - if not beacons: - raise APIException(message="beacon network error, network is empty") - - responses = {} - for b in beacons: - url = beacons[b].get("apiUrl") - try: - r = network_beacon_post( - url, - request.json, - endpoint, - ) - except APIException as e: - r["error"] = {"errorMessage": e.message} - continue - - # discard beacon "meta" response field - # it's the same for all requests, including this network request - r.pop("meta", None) - - responses[b] = r - - if not responses: - raise APIException(message="No response from beacon network") - - return beacon_network_response(responses) - - -@network.route("/network/beacons/") -@network.route("/network/beacons//", methods=["GET", "POST"]) -def query(beacon_id, endpoint="overview"): +# returns 404 if endpoint missing +@network.route("/beacons//", methods=["GET", "POST"]) +def query(beacon_id, endpoint): beacon = current_app.config["NETWORK_BEACONS"].get(beacon_id) + if not beacon: raise NotFoundException(message=f"no beacon found with id {beacon_id}") - if endpoint not in VALID_ENDPOINTS: + if endpoint not in current_app.config["NETWORK_VALID_QUERY_ENDPOINTS"]: raise NotFoundException() + # special handling for host beacon, avoid circular http calls + host_id = current_app.config["BEACON_ID"] + if beacon_id == host_id: + return host_beacon_response(endpoint) + + # all other beacons api_url = beacon.get("apiUrl") if request.method == "POST": From e5579f961f28129b16f478e69e806325747ecfb7 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Thu, 15 Aug 2024 15:20:09 +0000 Subject: [PATCH 20/41] network utilts --- bento_beacon/network/utils.py | 151 ++++++++++++++++++---------------- 1 file changed, 82 insertions(+), 69 deletions(-) diff --git a/bento_beacon/network/utils.py b/bento_beacon/network/utils.py index b0128ff1..5284f4ad 100644 --- a/bento_beacon/network/utils.py +++ b/bento_beacon/network/utils.py @@ -1,10 +1,16 @@ -from flask import current_app, g import requests +from flask import current_app from urllib.parse import urlsplit, urlunsplit from json import JSONDecodeError -from .network_config import BEACONS, NETWORK_INIT_TIMEOUT -from ..utils.exceptions import APIException, InvalidQuery -from ..utils.beacon_response import beacon_count_response +from ..utils.exceptions import APIException +from ..utils.katsu_utils import overview_statistics, get_katsu_config_search_fields +from ..endpoints.info import build_service_details, overview +from ..endpoints.biosamples import get_biosamples +from ..endpoints.cohorts import get_cohorts +from ..endpoints.datasets import get_datasets +from ..endpoints.individuals import get_individuals +from ..endpoints.variants import get_variants +from .bento_public_query import fields_intersection, fields_union PUBLIC_SEARCH_FIELDS_PATH = "/api/metadata/api/public_search_fields" DEFAULT_ENDPOINT = "individuals" @@ -13,14 +19,57 @@ "query": {"requestParameters": {}, "filters": [], "includeResultsetResponses": "ALL"}, "bento": {"showSummaryStatistics": True}, } +HOST_VIEWS_BY_ENDPOINT = { + "biosamples": get_biosamples, + "cohorts": get_cohorts, + "datasets": get_datasets, + "individuals": get_individuals, + "variants": get_variants, +} + + +# get network node info for this beacon, which is also hosting the network +# call methods directly instead of circular http calls +def info_for_host_beacon(): + service_details = build_service_details() + + # TODO: fix ugly overlapping overview functions + # requires rolling out changes to all beacons first + bento_overview = overview() + biosample_and_experiment_stats = overview_statistics() + api_url = current_app.config["BEACON_BASE_URL"] + + return { + **service_details, + "apiUrl": api_url, + "b_id": current_app.config["BEACON_ID"], + "overview": { + "individuals": {"count": bento_overview.get("counts", {}).get("individuals")}, + "variants": bento_overview.get("variants", {}), + **biosample_and_experiment_stats, + }, + "querySections": get_katsu_config_search_fields().get("sections", []), + } + +def host_beacon_response(endpoint): + # endpoint already known to be valid + return HOST_VIEWS_BY_ENDPOINT[endpoint]() + +# TODO: timeout param def network_beacon_call(method, url, payload=None): + timeout = current_app.config["NETWORK_INIT_TIMEOUT_SECONDS"] + + current_app.logger.info(f"Calling network url: {url}") + + # remove "verify" below, useful only for dev ############# + try: if method == "GET": - r = requests.get(url, timeout=NETWORK_INIT_TIMEOUT) + r = requests.get(url, timeout=timeout) ############## else: - r = requests.post(url, json=payload, timeout=NETWORK_INIT_TIMEOUT) + r = requests.post(url, json=payload, timeout=timeout) beacon_response = r.json() except (requests.exceptions.RequestException, JSONDecodeError) as e: @@ -41,11 +90,29 @@ def network_beacon_post(root_url, payload={}, endpoint=None): return network_beacon_call("POST", url, payload) +def make_network_filtering_terms(beacons): + all_query_sections = [b["querySections"] for b in beacons.values()] + current_app.config["ALL_NETWORK_FILTERS"] = filters_union(all_query_sections) + current_app.config["COMMON_NETWORK_FILTERS"] = filters_intersection(all_query_sections) + pass + + def init_network_service_registry(): current_app.logger.info("registering beacons") + urls = current_app.config["NETWORK_URLS"] network_beacons = {} failed_beacons = [] - for url in BEACONS: + host_beacon_url = current_app.config["BEACON_BASE_URL"] + current_app.logger.debug(f"host url: {host_beacon_url}") + for url in urls: + + # special handling for calling the beacon this network is hosted on + if url == host_beacon_url: + host_id = current_app.config["BEACON_ID"] + network_beacons[host_id] = info_for_host_beacon() + continue + + # all other beacons try: b = network_beacon_get(url, endpoint="overview") beacon_info = b.get("response") @@ -81,7 +148,7 @@ def init_network_service_registry(): network_beacons[b_id] = beacon_info network_beacons[b_id]["overview"] = overview - # TODO, katsu calls are inconsistent here (v15 katsu does not respond) + # Note: v15 katsu does not respond here # TODO (longer): serve beacon spec filtering terms instead of bento public querySections network_beacons[b_id]["querySections"] = get_public_search_fields(url).get("sections", []) # temp @@ -95,71 +162,17 @@ def init_network_service_registry(): f"{len(failed_beacons)} network beacon{'' if len(failed_beacons) == 1 else 's'} failed to respond: {', '.join(failed_beacons)}" ) + make_network_filtering_terms(network_beacons) current_app.config["NETWORK_BEACONS"] = network_beacons -def beacon_network_response(beacon_responses): - num_total_results, bento_stats = sum_network_responses(beacon_responses) - g.response_info["bento"] = bento_stats - g.response_info["network"] = beacon_responses - - # beacon network hardcoded to counts - # to handle all possible granularities, call build_query_response() instead - return beacon_count_response(num_total_results) - - -def sum_network_responses(beacon_responses): - num_total_results = 0 - # could parameterize response, currently hardcoded in bento_public - experiments_count = 0 - biosamples_count = 0 - sampled_tissue_chart = [] - experiment_type_chart = [] - - for response in beacon_responses.values(): - num_total_results += response.get("responseSummary", {}).get("numTotalResults", 0) - stats = response.get("info", {}).get("bento", {}) - experiments_count += stats.get("experiments", {}).get("count") - biosamples_count += stats.get("biosamples", {}).get("count") - sampled_tissue_chart = merge_charts(sampled_tissue_chart, stats.get("biosamples", {}).get("sampled_tissue", [])) - experiment_type_chart = merge_charts( - experiment_type_chart, stats.get("experiments", {}).get("experiment_type", []) - ) - - bento_stats = { - "biosamples": {"count": biosamples_count, "sampled_tissue": sampled_tissue_chart}, - "experiments": {"count": experiments_count, "experiment_type": experiment_type_chart}, - } - - return num_total_results, bento_stats - - -def chart_to_dict(chart): - return {item["label"]: item["value"] for item in chart} - - -def dict_to_chart(d): - return [{"label": label, "value": value} for label, value in d.items()] - - -def merge_charts(c1, c2): - """ - combine data from two categorical charts - any categories with identical names are merged into a single field with the sum of their values - """ - merged = chart_to_dict(c1) - for cat in c2: - label = cat["label"] - value = cat["value"] - merged[label] = merged.get(label, 0) + value - - return dict_to_chart(merged) +########################################## +# Temp utils for bento public search terms def get_public_search_fields(beacon_url): fields_url = public_search_fields_url(beacon_url) current_app.logger.info(f"trying public fields url {fields_url}") - fields = network_beacon_get(fields_url) return fields @@ -169,9 +182,9 @@ def public_search_fields_url(beacon_url): return urlunsplit((split_url.scheme, "portal." + split_url.netloc, PUBLIC_SEARCH_FIELDS_PATH, "", "")) -def filtersUnion(): - pass +def filters_union(all_search_fields): + return [{"section_title": "All Filters", "fields": fields_union(all_search_fields)}] -def filtersIntersection(): - pass +def filters_intersection(all_search_fields): + return [{"section_title": "Common Filters", "fields": fields_intersection(all_search_fields)}] From ee184ef1c83daafd1781e85450c4fbce79b3dd3f Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Thu, 15 Aug 2024 14:26:25 -0400 Subject: [PATCH 21/41] add network timeout params --- bento_beacon/config_files/config.py | 7 ++----- bento_beacon/network/network.py | 1 - bento_beacon/network/utils.py | 18 +++++++++++++----- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/bento_beacon/config_files/config.py b/bento_beacon/config_files/config.py index 1950e443..bc0fb782 100644 --- a/bento_beacon/config_files/config.py +++ b/bento_beacon/config_files/config.py @@ -215,11 +215,8 @@ def retrieve_config_json(filename): NETWORK_CONFIG = retrieve_config_json("network_config.json") NETWORK_URLS = NETWORK_CONFIG.get("beacons", []) - NETWORK_INIT_TIMEOUT_SECONDS = NETWORK_CONFIG.get("network_init_timeout_seconds", 30) - NETWORK_QUERY_WITH_VARIANTS_TIMEOUT_SECONDS = NETWORK_CONFIG.get("network_query_with_variants_timeout_seconds", 120) - NETWORK_QUERY_WITHOUT_VARIANTS_TIMEOUT_SECONDS = NETWORK_CONFIG.get( - "network_query_without_variants_timeout_seconds", 30 - ) + NETWORK_DEFAULT_TIMEOUT_SECONDS = NETWORK_CONFIG.get("network_default_timeout_seconds", 30) + NETWORK_VARIANTS_QUERY_TIMEOUT_SECONDS = NETWORK_CONFIG.get("network_variants_query_timeout_seconds", GOHAN_TIMEOUT) NETWORK_VALID_QUERY_ENDPOINTS = [ "analyses", "biosamples", diff --git a/bento_beacon/network/network.py b/bento_beacon/network/network.py index a58ca0d7..1e168e98 100644 --- a/bento_beacon/network/network.py +++ b/bento_beacon/network/network.py @@ -13,7 +13,6 @@ # standard beacon info endpoints at the network level: /map, /configuration, etc # handle GET args -# timeout param # clean up bento config file diff --git a/bento_beacon/network/utils.py b/bento_beacon/network/utils.py index 5284f4ad..464891f0 100644 --- a/bento_beacon/network/utils.py +++ b/bento_beacon/network/utils.py @@ -57,17 +57,25 @@ def host_beacon_response(endpoint): return HOST_VIEWS_BY_ENDPOINT[endpoint]() +def has_variants_query(payload): + if not payload: + return False + query = payload.get("requestParameters", {}).get("g_variant") + return bool(query) + + # TODO: timeout param def network_beacon_call(method, url, payload=None): - timeout = current_app.config["NETWORK_INIT_TIMEOUT_SECONDS"] - current_app.logger.info(f"Calling network url: {url}") - - # remove "verify" below, useful only for dev ############# + timeout = ( + current_app.config["NETWORK_VARIANTS_QUERY_TIMEOUT_SECONDS"] + if has_variants_query(payload) + else current_app.config["NETWORK_DEFAULT_TIMEOUT_SECONDS"] + ) try: if method == "GET": - r = requests.get(url, timeout=timeout) ############## + r = requests.get(url, timeout=timeout) else: r = requests.post(url, json=payload, timeout=timeout) beacon_response = r.json() From 1cb958eefc9a82d25203d1ddc5fc79cb799592c4 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Fri, 23 Aug 2024 17:13:31 +0000 Subject: [PATCH 22/41] completed todo --- bento_beacon/network/network.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bento_beacon/network/network.py b/bento_beacon/network/network.py index 1e168e98..48f2c329 100644 --- a/bento_beacon/network/network.py +++ b/bento_beacon/network/network.py @@ -13,8 +13,6 @@ # standard beacon info endpoints at the network level: /map, /configuration, etc # handle GET args -# clean up bento config file - @network.route("") @network.route("/beacons") From 4f47193bacf1e6a7b1f96f73004a6c3772df2914 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Fri, 23 Aug 2024 17:23:30 +0000 Subject: [PATCH 23/41] completed todo --- bento_beacon/network/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bento_beacon/network/utils.py b/bento_beacon/network/utils.py index 464891f0..ed170470 100644 --- a/bento_beacon/network/utils.py +++ b/bento_beacon/network/utils.py @@ -64,7 +64,6 @@ def has_variants_query(payload): return bool(query) -# TODO: timeout param def network_beacon_call(method, url, payload=None): current_app.logger.info(f"Calling network url: {url}") timeout = ( From 0f4cb38f7c68c0164011402d98e28b4a9935e334 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Wed, 28 Aug 2024 20:10:06 +0000 Subject: [PATCH 24/41] parameterize beacon network urls --- bento_beacon/app.py | 16 ++++++++-------- bento_beacon/config_files/config.py | 2 ++ bento_beacon/network/network.py | 6 ++++-- bento_beacon/network/utils.py | 4 ++++ 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/bento_beacon/app.py b/bento_beacon/app.py index c23c3728..dc971aa6 100644 --- a/bento_beacon/app.py +++ b/bento_beacon/app.py @@ -50,12 +50,7 @@ app.register_blueprint(info) -if USE_BEACON_NETWORK: - app.register_blueprint(network) - with app.app_context(): - init_network_service_registry() - -blueprints = { +endpoint_blueprints = { "biosamples": biosamples, "cohorts": cohorts, "datasets": datasets, @@ -64,12 +59,17 @@ } with app.app_context(): - # load blueprints + # load blueprints for endpoints endpoint_sets = current_app.config["BEACON_CONFIG"].get("endpointSets") for endpoint_set in endpoint_sets: if endpoint_set not in BEACON_MODELS: raise APIException(message="beacon config contains unknown endpoint set") - app.register_blueprint(blueprints[endpoint_set]) + app.register_blueprint(endpoint_blueprints[endpoint_set]) + + # load blueprint for network + if current_app.config["USE_BEACON_NETWORK"]: + app.register_blueprint(network) + init_network_service_registry() # get censorship settings from katsu max_filters = None diff --git a/bento_beacon/config_files/config.py b/bento_beacon/config_files/config.py index bc0fb782..9ad1fbdd 100644 --- a/bento_beacon/config_files/config.py +++ b/bento_beacon/config_files/config.py @@ -212,6 +212,8 @@ def retrieve_config_json(filename): # ------------------- # network + USE_BEACON_NETWORK = os.environ.get("BENTO_BEACON_NETWORK_ENABLED", "false").strip().lower() in ("true", "1", "t") + NETWORK_CONFIG = retrieve_config_json("network_config.json") NETWORK_URLS = NETWORK_CONFIG.get("beacons", []) diff --git a/bento_beacon/network/network.py b/bento_beacon/network/network.py index 48f2c329..5a3e0c3a 100644 --- a/bento_beacon/network/network.py +++ b/bento_beacon/network/network.py @@ -1,5 +1,5 @@ from flask import current_app, request, Blueprint -from ..utils.exceptions import NotFoundException +from ..utils.exceptions import APIException, NotFoundException from .utils import network_beacon_get, network_beacon_post, host_beacon_response, filters_intersection, filters_union network = Blueprint("network", __name__, url_prefix="/network") @@ -17,7 +17,9 @@ @network.route("") @network.route("/beacons") def network_beacons(): - beacons_dict = current_app.config["NETWORK_BEACONS"] + beacons_dict = current_app.config.get("NETWORK_BEACONS") + if not beacons_dict: + raise APIException("no beacons found in network config") # filters handling still experimental return { diff --git a/bento_beacon/network/utils.py b/bento_beacon/network/utils.py index ed170470..dd126228 100644 --- a/bento_beacon/network/utils.py +++ b/bento_beacon/network/utils.py @@ -107,6 +107,10 @@ def make_network_filtering_terms(beacons): def init_network_service_registry(): current_app.logger.info("registering beacons") urls = current_app.config["NETWORK_URLS"] + if not urls: + current_app.logger.error("can't find urls for beacon network, did you forget a config file?") + # this isn't driven by a request, so no point serving API error response here + return network_beacons = {} failed_beacons = [] host_beacon_url = current_app.config["BEACON_BASE_URL"] From 0d01716897bad8d706b541400eff04372b3c5569 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Thu, 29 Aug 2024 17:49:33 +0000 Subject: [PATCH 25/41] don't allow filters with empty options list --- bento_beacon/network/bento_public_query.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bento_beacon/network/bento_public_query.py b/bento_beacon/network/bento_public_query.py index 05a664d3..fa738ebe 100644 --- a/bento_beacon/network/bento_public_query.py +++ b/bento_beacon/network/bento_public_query.py @@ -83,7 +83,9 @@ def fields_intersection(searchFields): for f in intersection_dict.values(): entry = {} entry = copy.deepcopy(f[0]) # arbitrarily get name, description, etc from first entry - entry["options"] = options_intersection([e["options"] for e in f]) - intersection_fields.append(entry) + options = options_intersection([e["options"] for e in f]) + if options: + entry["options"] = options + intersection_fields.append(entry) return intersection_fields From ee67e8f0da3d61fb4a02e2ab9799673163df6de0 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Thu, 29 Aug 2024 14:06:20 -0400 Subject: [PATCH 26/41] lint --- bento_beacon/network/bento_public_query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bento_beacon/network/bento_public_query.py b/bento_beacon/network/bento_public_query.py index fa738ebe..56b44a80 100644 --- a/bento_beacon/network/bento_public_query.py +++ b/bento_beacon/network/bento_public_query.py @@ -84,7 +84,7 @@ def fields_intersection(searchFields): entry = {} entry = copy.deepcopy(f[0]) # arbitrarily get name, description, etc from first entry options = options_intersection([e["options"] for e in f]) - if options: + if options: entry["options"] = options intersection_fields.append(entry) From d4728771635aea503d5051371466b69374671069 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Wed, 4 Sep 2024 10:41:07 -0400 Subject: [PATCH 27/41] unused config --- bento_beacon/config_files/config.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/bento_beacon/config_files/config.py b/bento_beacon/config_files/config.py index 9ad1fbdd..4ab9e589 100644 --- a/bento_beacon/config_files/config.py +++ b/bento_beacon/config_files/config.py @@ -153,11 +153,7 @@ class Config: KATSU_PUBLIC_OVERVIEW = "/api/public_overview" KATSU_PUBLIC_RULES = "/api/public_rules" KATSU_TIMEOUT = int(os.environ.get("BEACON_KATSU_TIMEOUT", 180)) - MAP_EXTRA_PROPERTIES_TO_INFO = os.environ.get("MAP_EXTRA_PROPERTIES_TO_INFO", True) - - PHENOPACKETS_SCHEMA_REFERENCE = {"entityType": "individual", "schema": "phenopackets v2"} - MAX_RETRIES_FOR_CENSORSHIP_PARAMS = 2 # ------------------- # gohan From 51dcd6fac2d4346bd8c9d73ecc17728744a86973 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Wed, 4 Sep 2024 15:44:58 +0000 Subject: [PATCH 28/41] snake case --- bento_beacon/network/bento_public_query.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bento_beacon/network/bento_public_query.py b/bento_beacon/network/bento_public_query.py index 56b44a80..653ee4a2 100644 --- a/bento_beacon/network/bento_public_query.py +++ b/bento_beacon/network/bento_public_query.py @@ -15,13 +15,13 @@ def flatten(nested_list): return [item for nested_items in nested_list for item in nested_items] -def fields_dict(searchFields): +def fields_dict(search_fields): """ Given a list of bento_public search fields, one for each instance, return a dictionary of search fields keyed to phenopackets mapping, with an array of all fields for that mapping """ # create a single array of all search fields for all instances, removing nesting - copy = searchFields[:] + copy = search_fields[:] all_fields = [] for sf in copy: @@ -58,8 +58,8 @@ def options_intersection(options_list): # bins should be joined also, although some ordering may disappear # still unclear if this is an useful feature or not # shortcomings here can be addressed by keeping our configs consistent where possible -def fields_union(searchFields): - fields = fields_dict(searchFields) +def fields_union(search_fields): + fields = fields_dict(search_fields) # create one entry for each mapping union_fields = [] @@ -71,9 +71,9 @@ def fields_union(searchFields): return union_fields -def fields_intersection(searchFields): - num_instances = len(searchFields) - fields = fields_dict(searchFields) +def fields_intersection(search_fields): + num_instances = len(search_fields) + fields = fields_dict(search_fields) # remove any fields not in all entries intersection_dict = {mapping: entries for mapping, entries in fields.items() if len(entries) == num_instances} From a09a1177a02be83bb6d57fba610f4343f23b11ff Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Wed, 4 Sep 2024 16:00:22 +0000 Subject: [PATCH 29/41] better handling for missing request headers --- bento_beacon/authz/headers.py | 3 --- bento_beacon/utils/gohan_utils.py | 8 ++++++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/bento_beacon/authz/headers.py b/bento_beacon/authz/headers.py index 9ec32084..05ba24c3 100644 --- a/bento_beacon/authz/headers.py +++ b/bento_beacon/authz/headers.py @@ -2,9 +2,6 @@ def auth_header_getter(r: Request) -> dict[str, str]: - if not request: - # edge case for beacon network init, calling katsu / gohan with no request context - return None token = r.headers.get("authorization") return {"Authorization": token} if token else {} diff --git a/bento_beacon/utils/gohan_utils.py b/bento_beacon/utils/gohan_utils.py index 26b0c720..a398b6ba 100644 --- a/bento_beacon/utils/gohan_utils.py +++ b/bento_beacon/utils/gohan_utils.py @@ -1,4 +1,4 @@ -from flask import current_app +from flask import current_app, request from .exceptions import APIException, InvalidQuery, NotImplemented from ..authz.headers import auth_header_from_request import requests @@ -177,8 +177,12 @@ def gohan_results(url, gohan_args): def gohan_network_call(url, gohan_args): c = current_app.config + + # beacon network setup calls gohan for overview data outside of request context + headers = auth_header_from_request() if request else None + try: - r = requests.get(url, headers=auth_header_from_request(), timeout=c["GOHAN_TIMEOUT"], params=gohan_args) + r = requests.get(url, headers=headers, timeout=c["GOHAN_TIMEOUT"], params=gohan_args) # handle gohan errors or any bad responses if not r.ok: From 2417c6513d9f879f269600373dafe40a19648f43 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Tue, 10 Sep 2024 15:39:32 +0000 Subject: [PATCH 30/41] merge description changes --- bento_beacon/endpoints/info.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bento_beacon/endpoints/info.py b/bento_beacon/endpoints/info.py index 8a436789..6c608718 100644 --- a/bento_beacon/endpoints/info.py +++ b/bento_beacon/endpoints/info.py @@ -130,7 +130,10 @@ def build_service_details(): k_datasets = katsu_datasets() dats_array = list(map(lambda d: d.get("datsFile", {}), k_datasets)) description = " ".join([d.get("description") for d in dats_array if "description" in d]) - if description and info.get("description") is None: + custom_description = info.get("description") + if custom_description: + s["description"] = custom_description + if description and custom_description is None: s["description"] = description current_app.config["SERVICE_DETAILS"] = s From 20d1005a575dbf1eaecf0b272c29dd88f71cde8c Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Tue, 10 Sep 2024 15:39:51 +0000 Subject: [PATCH 31/41] fix variants count for beacon network host --- bento_beacon/network/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bento_beacon/network/utils.py b/bento_beacon/network/utils.py index dd126228..9a867bd1 100644 --- a/bento_beacon/network/utils.py +++ b/bento_beacon/network/utils.py @@ -45,7 +45,7 @@ def info_for_host_beacon(): "b_id": current_app.config["BEACON_ID"], "overview": { "individuals": {"count": bento_overview.get("counts", {}).get("individuals")}, - "variants": bento_overview.get("variants", {}), + "variants": bento_overview.get("counts", {}).get("variants", {}), **biosample_and_experiment_stats, }, "querySections": get_katsu_config_search_fields().get("sections", []), From 631eae2a312ba535273b4e50cd4bb496553b1412 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Wed, 18 Sep 2024 15:37:48 +0000 Subject: [PATCH 32/41] fix for network query sections auth --- bento_beacon/network/utils.py | 2 +- bento_beacon/utils/katsu_utils.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/bento_beacon/network/utils.py b/bento_beacon/network/utils.py index 9a867bd1..b7a26e83 100644 --- a/bento_beacon/network/utils.py +++ b/bento_beacon/network/utils.py @@ -48,7 +48,7 @@ def info_for_host_beacon(): "variants": bento_overview.get("counts", {}).get("variants", {}), **biosample_and_experiment_stats, }, - "querySections": get_katsu_config_search_fields().get("sections", []), + "querySections": get_katsu_config_search_fields(requires_auth="full").get("sections", []), } diff --git a/bento_beacon/utils/katsu_utils.py b/bento_beacon/utils/katsu_utils.py index 78395532..8dcea919 100644 --- a/bento_beacon/utils/katsu_utils.py +++ b/bento_beacon/utils/katsu_utils.py @@ -131,9 +131,11 @@ def search_from_config(config_filters): return response.get("matches", []) -def get_katsu_config_search_fields(): - # Use forwarded auth for getting available search fields, which may be limited based on access level - fields = katsu_get(current_app.config["KATSU_PUBLIC_CONFIG_ENDPOINT"], requires_auth="forwarded") +def get_katsu_config_search_fields(requires_auth="forwarded"): + # standard forwarded auth for normal beacon requests + # "full" auth for beacon network init, which does not have a request context + # any network-specific search field censorship should be managed at the token level (or here) + fields = katsu_get(current_app.config["KATSU_PUBLIC_CONFIG_ENDPOINT"], requires_auth=requires_auth) current_app.config["KATSU_CONFIG_SEARCH_FIELDS"] = fields return fields From 7ad11343936b127372504a6551750093f72dfdcd Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Thu, 19 Sep 2024 19:45:11 +0000 Subject: [PATCH 33/41] use "none" auth for beacon network --- bento_beacon/network/utils.py | 2 +- bento_beacon/utils/katsu_utils.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/bento_beacon/network/utils.py b/bento_beacon/network/utils.py index b7a26e83..32bcca54 100644 --- a/bento_beacon/network/utils.py +++ b/bento_beacon/network/utils.py @@ -48,7 +48,7 @@ def info_for_host_beacon(): "variants": bento_overview.get("counts", {}).get("variants", {}), **biosample_and_experiment_stats, }, - "querySections": get_katsu_config_search_fields(requires_auth="full").get("sections", []), + "querySections": get_katsu_config_search_fields(requires_auth="none").get("sections", []), } diff --git a/bento_beacon/utils/katsu_utils.py b/bento_beacon/utils/katsu_utils.py index 8dcea919..89dd91f3 100644 --- a/bento_beacon/utils/katsu_utils.py +++ b/bento_beacon/utils/katsu_utils.py @@ -131,10 +131,9 @@ def search_from_config(config_filters): return response.get("matches", []) -def get_katsu_config_search_fields(requires_auth="forwarded"): +def get_katsu_config_search_fields(requires_auth): # standard forwarded auth for normal beacon requests - # "full" auth for beacon network init, which does not have a request context - # any network-specific search field censorship should be managed at the token level (or here) + # "none" auth for beacon network init, which does not have a request context fields = katsu_get(current_app.config["KATSU_PUBLIC_CONFIG_ENDPOINT"], requires_auth=requires_auth) current_app.config["KATSU_CONFIG_SEARCH_FIELDS"] = fields return fields @@ -222,7 +221,7 @@ def katsu_resources_to_beacon_resource(r): def katsu_config_filtering_terms(): filtering_terms = [] - sections = get_katsu_config_search_fields().get("sections", []) + sections = get_katsu_config_search_fields(required_auth="forwarded").get("sections", []) for section in sections: for field in section["fields"]: filtering_term = { From c473f2bc86819dba8f28cf51e89a22e59dd1225d Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Wed, 2 Oct 2024 19:10:13 +0000 Subject: [PATCH 34/41] beacon network init fixes --- bento_beacon/app.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bento_beacon/app.py b/bento_beacon/app.py index 37523df4..43376423 100644 --- a/bento_beacon/app.py +++ b/bento_beacon/app.py @@ -24,10 +24,6 @@ REQUEST_SPEC_RELATIVE_PATH = "beacon-v2/framework/json/requests/" BEACON_MODELS = ["analyses", "biosamples", "cohorts", "datasets", "individuals", "runs", "variants"] -# temp -# add to config -USE_BEACON_NETWORK = True - app = Flask(__name__) # find path for beacon-v2 spec @@ -69,7 +65,11 @@ # load blueprint for network if current_app.config["USE_BEACON_NETWORK"]: app.register_blueprint(network) - init_network_service_registry() + try: + init_network_service_registry() + except APIException: + # trouble setting up network, swallow for now + current_app.logger.error("API Error when initializing beacon network") # get censorship settings from katsu max_filters = None From cc3b528131bac624a61cefa3c6d044b03000ec19 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Thu, 3 Oct 2024 14:13:57 +0000 Subject: [PATCH 35/41] comments --- bento_beacon/network/utils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bento_beacon/network/utils.py b/bento_beacon/network/utils.py index 32bcca54..41a1f3f3 100644 --- a/bento_beacon/network/utils.py +++ b/bento_beacon/network/utils.py @@ -12,7 +12,11 @@ from ..endpoints.variants import get_variants from .bento_public_query import fields_intersection, fields_union +# future versions will pull metadata query info directly from network beacons instead of network katsus +# to deprecate in Bento 18 PUBLIC_SEARCH_FIELDS_PATH = "/api/metadata/api/public_search_fields" + + DEFAULT_ENDPOINT = "individuals" OVERVIEW_STATS_QUERY = { "meta": {"apiVersion": "2.0.0"}, @@ -181,6 +185,7 @@ def init_network_service_registry(): # Temp utils for bento public search terms +# deprecate in Bento 18 def get_public_search_fields(beacon_url): fields_url = public_search_fields_url(beacon_url) current_app.logger.info(f"trying public fields url {fields_url}") @@ -188,6 +193,7 @@ def get_public_search_fields(beacon_url): return fields +# deprecate in Bento 18 def public_search_fields_url(beacon_url): split_url = urlsplit(beacon_url) return urlunsplit((split_url.scheme, "portal." + split_url.netloc, PUBLIC_SEARCH_FIELDS_PATH, "", "")) From 0ec58bec82c45a555ea51b09f49a2e1f2af3f9c4 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Thu, 3 Oct 2024 14:15:10 +0000 Subject: [PATCH 36/41] lint --- bento_beacon/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bento_beacon/app.py b/bento_beacon/app.py index 43376423..7766e163 100644 --- a/bento_beacon/app.py +++ b/bento_beacon/app.py @@ -68,7 +68,7 @@ try: init_network_service_registry() except APIException: - # trouble setting up network, swallow for now + # trouble setting up network, swallow for now current_app.logger.error("API Error when initializing beacon network") # get censorship settings from katsu From 6b5d5539ea4965808d422a185e3cf7a556ad947f Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Thu, 3 Oct 2024 15:34:44 +0000 Subject: [PATCH 37/41] fixes for katsu private overview changes --- bento_beacon/network/utils.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/bento_beacon/network/utils.py b/bento_beacon/network/utils.py index 41a1f3f3..63f93e4a 100644 --- a/bento_beacon/network/utils.py +++ b/bento_beacon/network/utils.py @@ -40,7 +40,15 @@ def info_for_host_beacon(): # TODO: fix ugly overlapping overview functions # requires rolling out changes to all beacons first bento_overview = overview() - biosample_and_experiment_stats = overview_statistics() + bento_private_overview = overview_statistics() + experiment_stats = {"count": bento_private_overview.get("count", 0)} + biosample_stats = { + "count": bento_private_overview.get("phenopacket", {}) + .get("data_type_specific", {}) + .get("biosamples", {}) + .get("count", 0) + } + api_url = current_app.config["BEACON_BASE_URL"] return { @@ -50,7 +58,8 @@ def info_for_host_beacon(): "overview": { "individuals": {"count": bento_overview.get("counts", {}).get("individuals")}, "variants": bento_overview.get("counts", {}).get("variants", {}), - **biosample_and_experiment_stats, + "biosamples": biosample_stats, + "experiments": experiment_stats, }, "querySections": get_katsu_config_search_fields(requires_auth="none").get("sections", []), } From cd5833b72fbd88361d534e65019083531e795a51 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Thu, 3 Oct 2024 18:50:05 +0000 Subject: [PATCH 38/41] reorder imports --- bento_beacon/config_files/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bento_beacon/config_files/config.py b/bento_beacon/config_files/config.py index 4d66dc75..0b0fd127 100644 --- a/bento_beacon/config_files/config.py +++ b/bento_beacon/config_files/config.py @@ -1,7 +1,7 @@ import json import os -from ..constants import GRANULARITY_COUNT, GRANULARITY_RECORD import urllib3 +from ..constants import GRANULARITY_COUNT, GRANULARITY_RECORD GA4GH_BEACON_REPO_URL = "https://raw.githubusercontent.com/ga4gh-beacon/beacon-v2" From 7f2c7f6233d01956563680e07be6cc35a81b9e3a Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Thu, 3 Oct 2024 19:27:06 +0000 Subject: [PATCH 39/41] auth type hint --- bento_beacon/utils/katsu_utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bento_beacon/utils/katsu_utils.py b/bento_beacon/utils/katsu_utils.py index 89dd91f3..710f1f42 100644 --- a/bento_beacon/utils/katsu_utils.py +++ b/bento_beacon/utils/katsu_utils.py @@ -8,6 +8,7 @@ from ..authz.access import create_access_header_or_fall_back from ..authz.headers import auth_header_from_request +RequiresAuthOptions = Literal["none", "forwarded", "full"] def katsu_filters_query(beacon_filters, datatype, get_biosample_ids=False): payload = katsu_json_payload(beacon_filters, datatype, get_biosample_ids) @@ -81,7 +82,7 @@ def katsu_network_call(payload, endpoint=None): # used for GET calls at particular katsu endpoints, eg /biosamples -def katsu_get(endpoint, id=None, query="", requires_auth: Literal["none", "forwarded", "full"] = "none"): +def katsu_get(endpoint, id=None, query="", requires_auth: RequiresAuthOptions = "none"): c = current_app.config katsu_base_url = c["KATSU_BASE_URL"] timeout = c["KATSU_TIMEOUT"] @@ -131,7 +132,7 @@ def search_from_config(config_filters): return response.get("matches", []) -def get_katsu_config_search_fields(requires_auth): +def get_katsu_config_search_fields(requires_auth: RequiresAuthOptions): # standard forwarded auth for normal beacon requests # "none" auth for beacon network init, which does not have a request context fields = katsu_get(current_app.config["KATSU_PUBLIC_CONFIG_ENDPOINT"], requires_auth=requires_auth) From 0346e923f4e4e63e7b8f9af7c1883d5c6d64c016 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Thu, 3 Oct 2024 19:35:41 +0000 Subject: [PATCH 40/41] lint --- bento_beacon/utils/katsu_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bento_beacon/utils/katsu_utils.py b/bento_beacon/utils/katsu_utils.py index 710f1f42..41b6c328 100644 --- a/bento_beacon/utils/katsu_utils.py +++ b/bento_beacon/utils/katsu_utils.py @@ -10,6 +10,7 @@ RequiresAuthOptions = Literal["none", "forwarded", "full"] + def katsu_filters_query(beacon_filters, datatype, get_biosample_ids=False): payload = katsu_json_payload(beacon_filters, datatype, get_biosample_ids) response = katsu_network_call(payload) From 2478ba1c4ef65276e0d0c66baa0a4cf049012285 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Thu, 3 Oct 2024 20:46:21 +0000 Subject: [PATCH 41/41] correct network config file name --- bento_beacon/config_files/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bento_beacon/config_files/config.py b/bento_beacon/config_files/config.py index 0b0fd127..022dceea 100644 --- a/bento_beacon/config_files/config.py +++ b/bento_beacon/config_files/config.py @@ -235,7 +235,7 @@ def retrieve_config_json(filename): USE_BEACON_NETWORK = os.environ.get("BENTO_BEACON_NETWORK_ENABLED", "false").strip().lower() in ("true", "1", "t") - NETWORK_CONFIG = retrieve_config_json("network_config.json") + NETWORK_CONFIG = retrieve_config_json("beacon_network_config.json") NETWORK_URLS = NETWORK_CONFIG.get("beacons", []) NETWORK_DEFAULT_TIMEOUT_SECONDS = NETWORK_CONFIG.get("network_default_timeout_seconds", 30)