From f727e1e1b4955d28edc4a0ab3030ca411aef0582 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Thu, 5 Oct 2023 14:58:54 +0000 Subject: [PATCH 01/17] add authz debug mode --- bento_beacon/authz.py | 1 + bento_beacon/config_files/config.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/bento_beacon/authz.py b/bento_beacon/authz.py index 708a467..a229776 100644 --- a/bento_beacon/authz.py +++ b/bento_beacon/authz.py @@ -17,6 +17,7 @@ Config.AUTHZ_URL, enabled=Config.AUTHZ_ENABLED, beacon_meta_callback=build_response_meta, + debug_mode=Config.BENTO_DEBUG ) # for now, these will go unused - Beacon currently does not have a strong concept of Bento projects/datasets diff --git a/bento_beacon/config_files/config.py b/bento_beacon/config_files/config.py index af4217b..48ef0f5 100644 --- a/bento_beacon/config_files/config.py +++ b/bento_beacon/config_files/config.py @@ -20,6 +20,9 @@ class Config: DEFAULT_PAGINATION_PAGE_SIZE = 10 + BENTO_DEBUG = os.environ.get("BENTO_DEBUG", os.environ.get( + "FLASK_DEBUG", "false")).strip().lower() in ('true', '1', 't') + BENTO_DOMAIN = os.environ.get("BENTOV2_DOMAIN") BEACON_BASE_URL = os.environ.get("BEACON_BASE_URL") BENTO_PUBLIC_URL = os.environ.get("BENTOV2_PUBLIC_URL") From dae8ac0c55a1c8efffa94548ffd7b1baf6f9c167 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Thu, 12 Oct 2023 16:21:28 +0000 Subject: [PATCH 02/17] refactor drl url handling --- bento_beacon/utils/handover_utils.py | 75 ++++++++-------------------- 1 file changed, 21 insertions(+), 54 deletions(-) diff --git a/bento_beacon/utils/handover_utils.py b/bento_beacon/utils/handover_utils.py index 0ba5d05..b578bbb 100644 --- a/bento_beacon/utils/handover_utils.py +++ b/bento_beacon/utils/handover_utils.py @@ -1,68 +1,36 @@ -from flask import current_app, request, url_for +from flask import current_app import requests from urllib.parse import urlsplit, urlunsplit from .katsu_utils import katsu_network_call from .exceptions import APIException from .nested_query_utils import auth_header_from_request -# path elements removed by bento gateway -# BEACON_PATH_FRAGMENT = "api/beacon" DRS_TIMEOUT_SECONDS = 10 -# may or may not be needed -# def get_handover_url(): -# base_url_components = urlsplit(request.url) -# handover_scheme = "https" -# handover_path = BEACON_PATH_FRAGMENT + url_for("handover.get_handover") -# handover_base_url = urlunsplit(( -# handover_scheme, -# base_url_components.netloc, -# handover_path, -# base_url_components.query, -# base_url_components.fragment -# )) -# return handover_base_url - - -def drs_internal_url_components(): - return urlsplit(current_app.config["DRS_INTERNAL_URL"]) - - -def drs_external_url_components(): - return urlsplit(current_app.config["DRS_EXTERNAL_URL"]) - -# TODO: either remove or deduplicate with below -# def drs_internal_file_link_for_id(id): -# internal_url_components = drs_internal_url_components() -# path = internal_url_components.path + "/objects/" + id + "/download" -# return urlunsplit(( -# internal_url_components.scheme, -# internal_url_components.netloc, -# path, -# internal_url_components.query, -# internal_url_components.fragment -# )) - - -def drs_external_file_link_for_id(id): - external_url_components = drs_external_url_components() - path = external_url_components.path + "/objects/" + id + "/download" + +def drs_url_components(): + return urlsplit(current_app.config["DRS_URL"]) + + +def drs_file_link_for_id(id): + url_components = drs_url_components() + path = url_components.path + "/objects/" + id + "/download" return urlunsplit(( "https", - external_url_components.netloc, + url_components.netloc, path, - external_url_components.query, - external_url_components.fragment + url_components.query, + url_components.fragment )) def drs_network_call(path, query): - base_url_components = drs_internal_url_components() + base_url_components = drs_url_components() url = urlunsplit(( base_url_components.scheme, base_url_components.netloc, - path, + base_url_components.path + path, query, base_url_components.fragment )) @@ -72,15 +40,16 @@ def drs_network_call(path, query): url, headers=auth_header_from_request(), timeout=DRS_TIMEOUT_SECONDS, + verify=not current_app.config.get("BENTO_DEBUG") ) drs_response = r.json() - # TODO, ideally after auth merge: + # TODO # on handover errors, keep returning rest of results instead of throwing api exception # add optional note in g and add to beacon response # return {} except requests.exceptions.RequestException as e: - current_app.logger.debug(f"drs error: {e.msg}") + current_app.logger.error(f"drs error: {e}") raise APIException(message="error generating handover links") return drs_response @@ -113,7 +82,7 @@ def filenames_by_results_set(ids): unique_filenames = list(set(filenames)) files_by_results_set[r] = unique_filenames - + return files_by_results_set @@ -122,15 +91,13 @@ def drs_link_from_vcf_filename(filename): if not obj: return None - # even with checksum de-duplication, there may be multiple files with the same filename - # (... perhaps you fixed the sample id in the vcf... ) + # there may be multiple files with the same filename # for now, just return the most recent ordered_by_most_recent = sorted( obj, key=lambda entry: entry['created_time'], reverse=True) most_recent_id = ordered_by_most_recent[0].get("id") - # internal_url = drs_internal_file_link_for_id(most_recent_id) - external_url = drs_external_file_link_for_id(most_recent_id) - return external_url + drs_url_for_file = drs_file_link_for_id(most_recent_id) + return drs_url_for_file def vcf_handover_entry(url, note=None): From 5a73ecfea5b32df0566d171c07ca70e3b5bbaec5 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Fri, 13 Oct 2023 20:40:14 +0000 Subject: [PATCH 03/17] just one drs url --- bento_beacon/config_files/config.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bento_beacon/config_files/config.py b/bento_beacon/config_files/config.py index 48ef0f5..a40d9df 100644 --- a/bento_beacon/config_files/config.py +++ b/bento_beacon/config_files/config.py @@ -143,8 +143,7 @@ class Config: # ------------------- # drs - DRS_INTERNAL_URL = os.environ.get("DRS_INTERNAL_URL") - DRS_EXTERNAL_URL = os.environ.get("DRS_EXTERNAL_URL") + DRS_URL = os.environ.get("DRS_URL") # ------------------- # authorization From 221d55ff020a92e4821d544813e97978a4b82150 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Fri, 13 Oct 2023 20:45:41 +0000 Subject: [PATCH 04/17] centralize auth code --- bento_beacon/app.py | 2 +- bento_beacon/authz/__init__.py | 0 bento_beacon/authz/headers.py | 10 +++++++ .../{authz.py => authz/middleware.py} | 28 ++++++++++--------- bento_beacon/endpoints/cohorts.py | 2 +- bento_beacon/endpoints/datasets.py | 2 +- bento_beacon/endpoints/info.py | 2 +- bento_beacon/endpoints/variants.py | 2 +- bento_beacon/utils/gohan_utils.py | 2 +- bento_beacon/utils/handover_utils.py | 2 +- bento_beacon/utils/katsu_utils.py | 2 +- bento_beacon/utils/nested_query_utils.py | 13 --------- 12 files changed, 33 insertions(+), 34 deletions(-) create mode 100644 bento_beacon/authz/__init__.py create mode 100644 bento_beacon/authz/headers.py rename bento_beacon/{authz.py => authz/middleware.py} (57%) delete mode 100644 bento_beacon/utils/nested_query_utils.py diff --git a/bento_beacon/app.py b/bento_beacon/app.py index 4672930..23469e2 100644 --- a/bento_beacon/app.py +++ b/bento_beacon/app.py @@ -11,7 +11,7 @@ from .endpoints.datasets import datasets from .utils.exceptions import APIException from werkzeug.exceptions import HTTPException -from .authz import authz_middleware +from .authz.middleware import authz_middleware from .config_files.config import Config from .utils.beacon_response import beacon_error_response from .utils.beacon_request import save_request_data, validate_request diff --git a/bento_beacon/authz/__init__.py b/bento_beacon/authz/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bento_beacon/authz/headers.py b/bento_beacon/authz/headers.py new file mode 100644 index 0000000..05ba24c --- /dev/null +++ b/bento_beacon/authz/headers.py @@ -0,0 +1,10 @@ +from flask import request, Request + + +def auth_header_getter(r: Request) -> dict[str, str]: + token = r.headers.get("authorization") + return {"Authorization": token} if token else {} + + +def auth_header_from_request() -> dict[str, str]: + return auth_header_getter(request) diff --git a/bento_beacon/authz.py b/bento_beacon/authz/middleware.py similarity index 57% rename from bento_beacon/authz.py rename to bento_beacon/authz/middleware.py index a229776..2dd6408 100644 --- a/bento_beacon/authz.py +++ b/bento_beacon/authz/middleware.py @@ -1,17 +1,8 @@ +from flask import request from bento_lib.auth.middleware.flask import FlaskAuthMiddleware -from .config_files.config import Config -from .utils.beacon_response import build_response_meta - -__all__ = [ - "authz_middleware", - "PERMISSION_QUERY_PROJECT_LEVEL_BOOLEAN", - "PERMISSION_QUERY_DATASET_LEVEL_BOOLEAN", - "PERMISSION_QUERY_PROJECT_LEVEL_COUNTS", - "PERMISSION_QUERY_DATASET_LEVEL_COUNTS", - "PERMISSION_QUERY_DATA", - "PERMISSION_DOWNLOAD_DATA", -] - +from ..config_files.config import Config +from ..utils.beacon_response import build_response_meta +from .headers import auth_header_getter authz_middleware = FlaskAuthMiddleware( Config.AUTHZ_URL, @@ -28,3 +19,14 @@ # these permissions can open up various aspects of handoff / full-search PERMISSION_QUERY_DATA = "query:data" PERMISSION_DOWNLOAD_DATA = "download:data" + + +def check_permissions(permissions: list[str]) -> bool: + auth_res = authz_middleware.authz_post(request, "/policy/evaluate", body={ + "requested_resource": {"everything": True}, + "required_permissions": permissions, + }, + headers_getter=auth_header_getter, # currently same as default, so can be removed + )["result"] + authz_middleware.mark_authz_done(request) + return auth_res diff --git a/bento_beacon/endpoints/cohorts.py b/bento_beacon/endpoints/cohorts.py index 36d44ea..a9acb8c 100644 --- a/bento_beacon/endpoints/cohorts.py +++ b/bento_beacon/endpoints/cohorts.py @@ -1,5 +1,5 @@ from flask import Blueprint, current_app -from ..authz import authz_middleware +from ..authz.middleware import authz_middleware from ..utils.exceptions import NotImplemented from ..utils.beacon_response import beacon_response diff --git a/bento_beacon/endpoints/datasets.py b/bento_beacon/endpoints/datasets.py index 9b66aa6..0e5448e 100644 --- a/bento_beacon/endpoints/datasets.py +++ b/bento_beacon/endpoints/datasets.py @@ -1,5 +1,5 @@ from flask import Blueprint, current_app -from ..authz import authz_middleware +from ..authz.middleware import authz_middleware from ..utils.exceptions import NotImplemented from ..utils.beacon_response import beacon_response from ..utils.katsu_utils import katsu_datasets diff --git a/bento_beacon/endpoints/info.py b/bento_beacon/endpoints/info.py index 42cd68d..afc22e1 100644 --- a/bento_beacon/endpoints/info.py +++ b/bento_beacon/endpoints/info.py @@ -1,7 +1,7 @@ import json from copy import deepcopy from flask import Blueprint, current_app -from ..authz import authz_middleware +from ..authz.middleware import authz_middleware from ..utils.beacon_response import beacon_info_response from ..utils.katsu_utils import ( get_filtering_terms, diff --git a/bento_beacon/endpoints/variants.py b/bento_beacon/endpoints/variants.py index 3044315..c8cee5c 100644 --- a/bento_beacon/endpoints/variants.py +++ b/bento_beacon/endpoints/variants.py @@ -1,5 +1,5 @@ from flask import Blueprint -from ..authz import authz_middleware +from ..authz.middleware import authz_middleware from ..utils.beacon_request import query_parameters_from_request from ..utils.beacon_response import beacon_response, add_info_to_response from ..utils.gohan_utils import query_gohan, gohan_total_variants_count, gohan_totals_by_sample_id diff --git a/bento_beacon/utils/gohan_utils.py b/bento_beacon/utils/gohan_utils.py index 9a18f5c..14a6770 100644 --- a/bento_beacon/utils/gohan_utils.py +++ b/bento_beacon/utils/gohan_utils.py @@ -1,6 +1,6 @@ from flask import current_app from .exceptions import APIException, InvalidQuery, NotImplemented -from .nested_query_utils import auth_header_from_request +from ..authz.headers import auth_header_from_request import requests # ------------------------------------------------------- diff --git a/bento_beacon/utils/handover_utils.py b/bento_beacon/utils/handover_utils.py index b578bbb..5f330f1 100644 --- a/bento_beacon/utils/handover_utils.py +++ b/bento_beacon/utils/handover_utils.py @@ -3,7 +3,7 @@ from urllib.parse import urlsplit, urlunsplit from .katsu_utils import katsu_network_call from .exceptions import APIException -from .nested_query_utils import auth_header_from_request +from ..authz.headers import auth_header_from_request DRS_TIMEOUT_SECONDS = 10 diff --git a/bento_beacon/utils/katsu_utils.py b/bento_beacon/utils/katsu_utils.py index f02e5bf..9be6e3c 100644 --- a/bento_beacon/utils/katsu_utils.py +++ b/bento_beacon/utils/katsu_utils.py @@ -4,7 +4,7 @@ from urllib.parse import urlsplit, urlunsplit from .exceptions import APIException, InvalidQuery from functools import reduce -from .nested_query_utils import auth_header_from_request +from ..authz.headers import auth_header_from_request def katsu_filters_query(beacon_filters, datatype, get_biosample_ids=False): diff --git a/bento_beacon/utils/nested_query_utils.py b/bento_beacon/utils/nested_query_utils.py deleted file mode 100644 index 9b293f8..0000000 --- a/bento_beacon/utils/nested_query_utils.py +++ /dev/null @@ -1,13 +0,0 @@ -from flask import request - -__all__ = [ - "HTTP_AUTHZ_HEADER", - "auth_header_from_request", -] - -HTTP_AUTHZ_HEADER = "Authorization" - - -def auth_header_from_request() -> dict[str, str]: - auth_header = request.headers.get(HTTP_AUTHZ_HEADER) - return {HTTP_AUTHZ_HEADER: auth_header} From 3894d81c3049e5ae3351a6903c60142c328b5f45 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Fri, 13 Oct 2023 20:52:17 +0000 Subject: [PATCH 05/17] whitespace --- bento_beacon/utils/beacon_response.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bento_beacon/utils/beacon_response.py b/bento_beacon/utils/beacon_response.py index 938533c..28ab0b2 100644 --- a/bento_beacon/utils/beacon_response.py +++ b/bento_beacon/utils/beacon_response.py @@ -121,7 +121,7 @@ def build_response_meta(): received_request_summary = received_request() return { "beaconId": current_app.config["BEACON_ID"], - "apiVersion": current_app.config["BEACON_SPEC_VERSION"], + "apiVersion": current_app.config["BEACON_SPEC_VERSION"], "returnedSchemas": returned_schemas, "returnedGranularity": returned_granularity, "receivedRequestSummary": received_request_summary From 413478a347fd95e0384349892094d9fb4c39fdc3 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Mon, 16 Oct 2023 14:25:11 +0000 Subject: [PATCH 06/17] add auth to individuals --- bento_beacon/endpoints/individuals.py | 49 ++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/bento_beacon/endpoints/individuals.py b/bento_beacon/endpoints/individuals.py index 4c6641e..7372bb3 100644 --- a/bento_beacon/endpoints/individuals.py +++ b/bento_beacon/endpoints/individuals.py @@ -1,6 +1,6 @@ from flask import Blueprint from functools import reduce -from ..authz import authz_middleware +from ..authz.middleware import PERMISSION_DOWNLOAD_DATA, PERMISSION_QUERY_DATA, check_permissions from ..utils.beacon_request import ( query_parameters_from_request, summary_stats_requested, @@ -10,21 +10,26 @@ add_info_to_response, add_stats_to_response, add_overview_stats_to_response, - zero_count_response + zero_count_response, + beacon_full_response ) from ..utils.katsu_utils import ( katsu_filters_and_sample_ids_query, katsu_total_individuals_count, - search_from_config + search_from_config, + phenopackets_for_ids ) from ..utils.search import biosample_id_search +from ..utils.handover_utils import handover_for_ids individuals = Blueprint("individuals", __name__) @individuals.route("/individuals", methods=['GET', 'POST']) -@authz_middleware.deco_public_endpoint # TODO: authz - for now. eventually, return more depending on permissions def get_individuals(): + + full_response = check_permissions([PERMISSION_QUERY_DATA]) + variants_query, phenopacket_filters, experiment_filters, config_filters = query_parameters_from_request() no_query = not (variants_query or phenopacket_filters or experiment_filters or config_filters) @@ -32,6 +37,7 @@ def get_individuals(): config_search_only = config_filters and not (variants_query or phenopacket_filters or experiment_filters) # return total count of individuals if no query + # TODO: return default granularity rather than count (default could be bool rather than count) if no_query: add_info_to_response("no query found, returning total count") total_count = katsu_total_individuals_count() @@ -77,9 +83,44 @@ def get_individuals(): if summary_stats_requested(): add_stats_to_response(individual_ids) + if full_response: + return individuals_full_response(individual_ids) + + # TODO: configurable default response rather than hardcoded counts return beacon_response({"count": len(individual_ids), "results": individual_ids}) +# TODO: pagination (ideally after katsu search gets paginated) +def individuals_full_response(ids): + + # temp + # if len(ids) > 100: + # return {"message": "too many ids for full response"} + + handover_permission = check_permissions([PERMISSION_DOWNLOAD_DATA]) + handover = handover_for_ids(ids) if handover_permission else {} + phenopackets_by_result_set = phenopackets_for_ids(ids).get("results", {}) + result_ids = list(phenopackets_by_result_set.keys()) + result_sets = {} + + for r_id in result_ids: + results_this_id = phenopackets_by_result_set.get(r_id, {}).get("matches", []) + results_count = len(results_this_id) + result = { + "id": r_id, + "setType": "individual", + "exists": results_count > 0, + "resultsCount": results_count, + "results": results_this_id, + } + handover_this_id = handover.get(r_id, []) + if handover_this_id: + result["resultsHandovers"] = handover_this_id + result_sets[r_id] = result + + return beacon_full_response(result_sets) + + # ------------------------------------------------------- # unimplemented endpoints # ------------------------------------------------------- From a9db6b18badc2d46cd3f6d45ae91589e78bb9c6c Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Mon, 16 Oct 2023 14:26:29 +0000 Subject: [PATCH 07/17] function naming, handover now conditional --- bento_beacon/utils/beacon_response.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bento_beacon/utils/beacon_response.py b/bento_beacon/utils/beacon_response.py index 28ab0b2..61edf68 100644 --- a/bento_beacon/utils/beacon_response.py +++ b/bento_beacon/utils/beacon_response.py @@ -83,7 +83,7 @@ def beacon_response(results, collection_response=False): return r -def beacon_response_with_handover(result_sets): +def beacon_full_response(result_sets): g.request_data["requestedGranularity"] = "record" r = { "meta": build_response_meta(), From 37cb06e813d0d9a949f662a6492927a2c7757477 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Mon, 16 Oct 2023 14:26:56 +0000 Subject: [PATCH 08/17] clearify comment --- bento_beacon/authz/middleware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bento_beacon/authz/middleware.py b/bento_beacon/authz/middleware.py index 2dd6408..c3fd579 100644 --- a/bento_beacon/authz/middleware.py +++ b/bento_beacon/authz/middleware.py @@ -26,7 +26,7 @@ def check_permissions(permissions: list[str]) -> bool: "requested_resource": {"everything": True}, "required_permissions": permissions, }, - headers_getter=auth_header_getter, # currently same as default, so can be removed + headers_getter=auth_header_getter, # currently same as middleware default, so can be removed )["result"] authz_middleware.mark_authz_done(request) return auth_res From b92a998a3a99c56a2ddbf5763c6c885a039d1cfe Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Mon, 16 Oct 2023 17:12:44 +0000 Subject: [PATCH 09/17] single persmission check without array wrapper --- bento_beacon/authz/middleware.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bento_beacon/authz/middleware.py b/bento_beacon/authz/middleware.py index c3fd579..8a30283 100644 --- a/bento_beacon/authz/middleware.py +++ b/bento_beacon/authz/middleware.py @@ -30,3 +30,7 @@ def check_permissions(permissions: list[str]) -> bool: )["result"] authz_middleware.mark_authz_done(request) return auth_res + + +def check_permission(permission: str) -> bool: + return check_permissions([permission]) From 21d36cb040bc66132148af92c9d6f90d77929deb Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Mon, 16 Oct 2023 17:14:07 +0000 Subject: [PATCH 10/17] simpler permission check --- bento_beacon/endpoints/individuals.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bento_beacon/endpoints/individuals.py b/bento_beacon/endpoints/individuals.py index 7372bb3..8b524c5 100644 --- a/bento_beacon/endpoints/individuals.py +++ b/bento_beacon/endpoints/individuals.py @@ -1,6 +1,6 @@ from flask import Blueprint from functools import reduce -from ..authz.middleware import PERMISSION_DOWNLOAD_DATA, PERMISSION_QUERY_DATA, check_permissions +from ..authz.middleware import PERMISSION_DOWNLOAD_DATA, PERMISSION_QUERY_DATA, check_permission from ..utils.beacon_request import ( query_parameters_from_request, summary_stats_requested, @@ -28,7 +28,7 @@ @individuals.route("/individuals", methods=['GET', 'POST']) def get_individuals(): - full_response = check_permissions([PERMISSION_QUERY_DATA]) + full_response = check_permission(PERMISSION_QUERY_DATA) variants_query, phenopacket_filters, experiment_filters, config_filters = query_parameters_from_request() @@ -97,7 +97,7 @@ def individuals_full_response(ids): # if len(ids) > 100: # return {"message": "too many ids for full response"} - handover_permission = check_permissions([PERMISSION_DOWNLOAD_DATA]) + handover_permission = check_permission(PERMISSION_DOWNLOAD_DATA) handover = handover_for_ids(ids) if handover_permission else {} phenopackets_by_result_set = phenopackets_for_ids(ids).get("results", {}) result_ids = list(phenopackets_by_result_set.keys()) From c16f4ace3d79afcf32e60da72c2f74a009def79a Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Mon, 16 Oct 2023 19:06:00 +0000 Subject: [PATCH 11/17] improve persmissions handling in /individuals --- bento_beacon/endpoints/individuals.py | 61 +++++++++++---------------- 1 file changed, 24 insertions(+), 37 deletions(-) diff --git a/bento_beacon/endpoints/individuals.py b/bento_beacon/endpoints/individuals.py index 8b524c5..cd51b0d 100644 --- a/bento_beacon/endpoints/individuals.py +++ b/bento_beacon/endpoints/individuals.py @@ -1,6 +1,6 @@ from flask import Blueprint from functools import reduce -from ..authz.middleware import PERMISSION_DOWNLOAD_DATA, PERMISSION_QUERY_DATA, check_permission +from ..authz.middleware import authz_middleware, PERMISSION_DOWNLOAD_DATA, PERMISSION_QUERY_DATA, check_permission from ..utils.beacon_request import ( query_parameters_from_request, summary_stats_requested, @@ -27,9 +27,6 @@ @individuals.route("/individuals", methods=['GET', 'POST']) def get_individuals(): - - full_response = check_permission(PERMISSION_QUERY_DATA) - variants_query, phenopacket_filters, experiment_filters, config_filters = query_parameters_from_request() no_query = not (variants_query or phenopacket_filters or experiment_filters or config_filters) @@ -83,11 +80,14 @@ def get_individuals(): if summary_stats_requested(): add_stats_to_response(individual_ids) - if full_response: - return individuals_full_response(individual_ids) - + return individuals_response(individual_ids) + + +def individuals_response(ids): + if check_permission(PERMISSION_QUERY_DATA): + return individuals_full_response(ids) # TODO: configurable default response rather than hardcoded counts - return beacon_response({"count": len(individual_ids), "results": individual_ids}) + return beacon_response({"count": len(ids), "results": ids}) # TODO: pagination (ideally after katsu search gets paginated) @@ -121,34 +121,21 @@ def individuals_full_response(ids): return beacon_full_response(result_sets) -# ------------------------------------------------------- -# unimplemented endpoints -# ------------------------------------------------------- -# these would be appropriate for full-access beacons only - -# replace this one once auth is in place -# @individuals.route("/individuals/", methods=['GET', 'POST']) -# @authn_token_required_flask_wrapper -# def individual_by_id(id): -# # get one individual by id, with handover if available -# return individuals_full_response([id]) +@individuals.route("/individuals/", methods=['GET', 'POST']) +@authz_middleware.deco_require_permissions_on_resource({PERMISSION_QUERY_DATA}) +def individual_by_id(id): + # forbidden / unauthorized if no permissions + return individuals_full_response([id]) -# @individuals.route("/individuals//g_variants", methods=['GET', 'POST']) -# def individual_variants(id): -# # all variants for a particular individual -# # may never be implemented, bad match for our use case (download vcf instead) -# raise NotImplemented() - - -# @individuals.route("/individuals//biosamples", methods=['GET', 'POST']) -# def individual_biosamples(id): -# # all biosamples for a particular individual -# pass - - -# @individuals.route("/individuals//filtering_terms", methods=['GET', 'POST']) -# def individual_filtering_terms(id): -# # filtering terms for a particular individual -# # note that this involves private data -# pass +# ------------------------------------------------------- +# endpoints in beacon model not yet implemented: +# +# /individuals//g_variants +# - may be simpler to download vcf instead +# /individuals//biosamples +# - requires full response code for biosamples (could just serve phenopackets again here) +# /individuals//filtering_terms +# - unclear use case, isn't reading the full response better? +# - requires better ontology support / better filtering terms implementation +# ------------------------------------------------------- From 56fab93dce3b4e36c6533bbfcddc233f6a809c85 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Mon, 16 Oct 2023 19:18:53 +0000 Subject: [PATCH 12/17] check permissions sooner, behaves badly otherwise --- bento_beacon/endpoints/individuals.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/bento_beacon/endpoints/individuals.py b/bento_beacon/endpoints/individuals.py index cd51b0d..187cb3e 100644 --- a/bento_beacon/endpoints/individuals.py +++ b/bento_beacon/endpoints/individuals.py @@ -27,6 +27,9 @@ @individuals.route("/individuals", methods=['GET', 'POST']) def get_individuals(): + + full_response = check_permission(PERMISSION_QUERY_DATA) + variants_query, phenopacket_filters, experiment_filters, config_filters = query_parameters_from_request() no_query = not (variants_query or phenopacket_filters or experiment_filters or config_filters) @@ -76,18 +79,13 @@ def get_individuals(): # baroque syntax but covers all cases individual_ids = list(reduce(set.intersection, (set(ids) for ids in individual_results.values()))) - # conditionally add summary statistics to response if summary_stats_requested(): add_stats_to_response(individual_ids) - return individuals_response(individual_ids) - + if full_response: + return individuals_full_response(individual_ids) -def individuals_response(ids): - if check_permission(PERMISSION_QUERY_DATA): - return individuals_full_response(ids) - # TODO: configurable default response rather than hardcoded counts - return beacon_response({"count": len(ids), "results": ids}) + return beacon_response({"count": len(individual_ids), "results": individual_ids}) # TODO: pagination (ideally after katsu search gets paginated) From 2aa5062168348a7d07e39340a83925f851f9ec56 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Mon, 16 Oct 2023 20:01:47 +0000 Subject: [PATCH 13/17] whitespace, comments --- bento_beacon/endpoints/individuals.py | 4 ++-- bento_beacon/utils/beacon_response.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bento_beacon/endpoints/individuals.py b/bento_beacon/endpoints/individuals.py index 187cb3e..4e1b9a7 100644 --- a/bento_beacon/endpoints/individuals.py +++ b/bento_beacon/endpoints/individuals.py @@ -4,7 +4,7 @@ from ..utils.beacon_request import ( query_parameters_from_request, summary_stats_requested, - ) +) from ..utils.beacon_response import ( beacon_response, add_info_to_response, @@ -128,7 +128,7 @@ def individual_by_id(id): # ------------------------------------------------------- # endpoints in beacon model not yet implemented: -# +# # /individuals//g_variants # - may be simpler to download vcf instead # /individuals//biosamples diff --git a/bento_beacon/utils/beacon_response.py b/bento_beacon/utils/beacon_response.py index 61edf68..d88cf4e 100644 --- a/bento_beacon/utils/beacon_response.py +++ b/bento_beacon/utils/beacon_response.py @@ -20,6 +20,7 @@ def init_response_data(): g.response_info = {} +# TODO: handle multiple messages def add_info_to_response(info): g.response_info["message"] = info From 5b1588a42acf1fdda48e64f894e2b40fe363ec0e Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Tue, 17 Oct 2023 13:36:26 +0000 Subject: [PATCH 14/17] replace __all__ --- bento_beacon/authz/middleware.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/bento_beacon/authz/middleware.py b/bento_beacon/authz/middleware.py index 8a30283..60aa2e4 100644 --- a/bento_beacon/authz/middleware.py +++ b/bento_beacon/authz/middleware.py @@ -4,6 +4,19 @@ from ..utils.beacon_response import build_response_meta from .headers import auth_header_getter +__all__ = [ + "authz_middleware", + "PERMISSION_QUERY_PROJECT_LEVEL_BOOLEAN", + "PERMISSION_QUERY_DATASET_LEVEL_BOOLEAN", + "PERMISSION_QUERY_PROJECT_LEVEL_COUNTS", + "PERMISSION_QUERY_DATASET_LEVEL_COUNTS", + "PERMISSION_QUERY_DATA", + "PERMISSION_DOWNLOAD_DATA", + "check_permissions", + "check_permission" +] + + authz_middleware = FlaskAuthMiddleware( Config.AUTHZ_URL, enabled=Config.AUTHZ_ENABLED, From d699f333af022a6fb8a4c011ac7198c73adbb4e2 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Tue, 17 Oct 2023 16:28:48 +0000 Subject: [PATCH 15/17] use drs access_methods for file url --- bento_beacon/utils/handover_utils.py | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/bento_beacon/utils/handover_utils.py b/bento_beacon/utils/handover_utils.py index 5f330f1..ba1bf9d 100644 --- a/bento_beacon/utils/handover_utils.py +++ b/bento_beacon/utils/handover_utils.py @@ -13,18 +13,6 @@ def drs_url_components(): return urlsplit(current_app.config["DRS_URL"]) -def drs_file_link_for_id(id): - url_components = drs_url_components() - path = url_components.path + "/objects/" + id + "/download" - return urlunsplit(( - "https", - url_components.netloc, - path, - url_components.query, - url_components.fragment - )) - - def drs_network_call(path, query): base_url_components = drs_url_components() url = urlunsplit(( @@ -93,11 +81,12 @@ def drs_link_from_vcf_filename(filename): # there may be multiple files with the same filename # for now, just return the most recent - ordered_by_most_recent = sorted( - obj, key=lambda entry: entry['created_time'], reverse=True) - most_recent_id = ordered_by_most_recent[0].get("id") - drs_url_for_file = drs_file_link_for_id(most_recent_id) - return drs_url_for_file + most_recent = sorted(obj, key=lambda entry: entry['created_time'], reverse=True)[0] + + # return any http access, in the future we may want to return other stuff (Globus, htsget, etc) + access_methods = most_recent.get("access_methods", []) + http_access = next((a for a in access_methods if a.get("type") in ("http", "https")), None) + return http_access.get("access_url", {}).get("url") if http_access else None def vcf_handover_entry(url, note=None): From a7f363134132f59dbf6dac737111b98655542bda Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Tue, 17 Oct 2023 16:43:29 +0000 Subject: [PATCH 16/17] rm redundant --- bento_beacon/authz/middleware.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bento_beacon/authz/middleware.py b/bento_beacon/authz/middleware.py index 60aa2e4..f95aef5 100644 --- a/bento_beacon/authz/middleware.py +++ b/bento_beacon/authz/middleware.py @@ -39,7 +39,6 @@ def check_permissions(permissions: list[str]) -> bool: "requested_resource": {"everything": True}, "required_permissions": permissions, }, - headers_getter=auth_header_getter, # currently same as middleware default, so can be removed )["result"] authz_middleware.mark_authz_done(request) return auth_res From 455ad3a789b2e6e111a323d2d5efdec6643cf367 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Tue, 17 Oct 2023 20:02:01 +0000 Subject: [PATCH 17/17] unused import --- bento_beacon/authz/middleware.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bento_beacon/authz/middleware.py b/bento_beacon/authz/middleware.py index f95aef5..7c47ef8 100644 --- a/bento_beacon/authz/middleware.py +++ b/bento_beacon/authz/middleware.py @@ -2,7 +2,6 @@ from bento_lib.auth.middleware.flask import FlaskAuthMiddleware from ..config_files.config import Config from ..utils.beacon_response import build_response_meta -from .headers import auth_header_getter __all__ = [ "authz_middleware",