diff --git a/src/modules/site-v2/base/utils/auth.py b/src/modules/site-v2/base/utils/auth.py index 20a487a5a..2fbff9adf 100644 --- a/src/modules/site-v2/base/utils/auth.py +++ b/src/modules/site-v2/base/utils/auth.py @@ -172,3 +172,24 @@ def expired_token_callback(_jwt_header, jwt_data): resp = make_response(redirect(url_for('auth.refresh'))) unset_access_cookies(resp) return resp, 302 + + + +def access_token_required(token): + ''' + Require a "Bearer" access token in the request. + ''' + def wrapper(fn): + @wraps(fn) + def decorator(*args, **kwargs): + + # Check for access token in request + access_token = request.headers.get('Authorization') + if access_token != 'Bearer {}'.format(token): + abort(403) + + # Forward to wrapped function + return fn(*args, **kwargs) + + return decorator + return wrapper diff --git a/src/modules/site-v2/base/utils/view_decorators.py b/src/modules/site-v2/base/utils/view_decorators.py new file mode 100644 index 000000000..5174d26da --- /dev/null +++ b/src/modules/site-v2/base/utils/view_decorators.py @@ -0,0 +1,224 @@ +import bleach +from functools import wraps +from typing import Type + +from flask import abort, redirect, request, url_for, flash, jsonify +from flask_wtf import FlaskForm + +from base.utils.auth import user_is_admin +from base.utils.tools import lookup_report, get_upload_err_msg +from constants import TOOL_INPUT_DATA_VALID_FILE_EXTENSIONS + +from caendr.models.datastore import DatasetRelease, Species +from caendr.models.error import NotFoundError, ReportLookupError, EmptyReportDataError, EmptyReportResultsError, FileUploadError, DataValidationError +from caendr.models.job_pipeline import JobPipeline +from caendr.services.logger import logger +from caendr.utils.local_files import LocalUploadFile + + + +def parse_species(f): + ''' + Parse `species_name` string argument from the URL into a `Species` object. + + Aborts with `404` if species name was not valid. + ''' + + @wraps(f) + def decorator(*args, species_name, **kwargs): + + # Parse the species & release from the URL + try: + species = Species.from_name(species_name, from_url=True) + + # The `from_name` method always raises a NotFoundError if the name is not valid + except NotFoundError: + return abort(404) + + # If species name provided with underscore instead of dash, redirect to dashed version of URL + if species.get_slug() != species_name: + return redirect( url_for(request.endpoint, *args, species_name=species.get_slug(), **kwargs) ) + + # Pass the objects to the function + return f(*args, species=species, **kwargs) + + return decorator + + + +def parse_species_and_release(f): + ''' + Parse `species_name` and `release_version` string arguments from the URL + into a `Species` object and `DatasetRelease` object, respectively. + + If `release_version` is omitted, defaults to latest release for the given species. + + Aborts with `404` if either was not valid. + ''' + + @wraps(f) + def decorator(*args, species_name, release_version=None, **kwargs): + + # Parse the species & release from the URL + try: + species = Species.from_name(species_name, from_url=True) + release = DatasetRelease.from_name(release_version, species_name=species.name) + + # The `from_name` method always raises a NotFoundError if the name is not valid + except NotFoundError: + return abort(404) + + # If species name provided with underscore instead of dash, redirect to dashed version of URL + if species.get_slug() != species_name: + return redirect( url_for(request.endpoint, *args, species_name=species.get_slug(), release_version=release_version, **kwargs) ) + + # Pass the objects to the function + return f(*args, species=species, release=release, **kwargs) + + return decorator + + + +def parse_job_id(pipeline_class: Type[JobPipeline], fetch=True, check_data_exists=True): + ''' + Parse `report_id` string argument from the URL into a `JobPipeline` subclass object, + and pre-fetches the data and results if desired. + + Aborts with `404` if report ID was invalid, or optionally if no input data could be fetched. + + Arguments: + - `pipeline_class`: The `JobPipeline` subclass to use to lookup the report. + - `fetch`: If `True`, pre-fetch the input `data` and output `result`, and pass them to the wrapped function as keyword arguments. + - `check_data_exists`: If `True`, abort with `404` if the fetched input data is `None`. + ''' + + def wrapper(f): + + @wraps(f) + def decorator(*args, report_id, **kwargs): + + # Fetch requested phenotype report + # Ensures the report exists and the current user has permission to view it + try: + job = lookup_report(pipeline_class.get_kind(), report_id) + + # If the report lookup request is invalid, show an error message + except ReportLookupError as ex: + flash(ex.msg, 'danger') + abort(ex.code) + + # Optionally bail out here -- don't bother fetching data/results + if not fetch: + return f(*args, job=job, **kwargs) + + # Try getting & parsing the report data file and results + # If result is None, job hasn't finished computing yet + try: + data, result = job.fetch() + + # Error reading one of the report files + except (EmptyReportDataError, EmptyReportResultsError) as ex: + logger.error(f'Error fetching {pipeline_class.get_kind()} report {ex.id}: {ex.description}') + return abort(404, description = ex.description) + + # Error with the submission data + # This should only be possible if a report was somehow created with invalid data, + # e.g. not enough traits in a Phenotype Analysis report + except DataValidationError as ex: + logger.error(f'Error fetching {pipeline_class.get_kind()} report {id}: {ex}') + flash(ex.msg, 'error') + return abort(400, description = ex.msg) + + # General error + except Exception as ex: + logger.error(f'Error fetching {pipeline_class.get_kind()} report {id}: {ex}') + return abort(400, description = 'Something went wrong') + + # Check that data file exists, if desired + if check_data_exists and data is None: + logger.error(f'Error fetching {pipeline_class.get_kind()} report {id}: Input data file does not exist') + return abort(404) + + # Pass the objects to the function + return f(*args, job=job, data=data, result=result, **kwargs) + + return decorator + return wrapper + + + +def validate_form(form_class: Type[FlaskForm], from_json: bool = False, err_msg: str = None, flash_err_msg: bool = True): + ''' + Parse the request form into the given form type, validate the fields, and inject the data as a dict. + + Aborts with `400` if form validation fails. + + TODO: What happens with non-None `form_class` and `from_json = True`? Can FlaskForm initialize that way? + + Passes the following args to the wrapped function: + - `form_data`: A dict of cleaned / validated fields from the form. + - `no_cache`: Whether the user wants to skip caching the form results. Can only be set if user is admin. + + Arguments: + - `form_class`: The `FlaskForm` subclass to use for parsing/validation. If `None`, cleans the individual fields but performs no form validation. + - `from_json`: If `True`, use the request `.get_json()` as the fields instead. + - `err_msg`: An error message to add to the response if validation fails. + - `flash_err_msg`: If `True`, flashes the `err_msg` in addition to returning it. + ''' + + def wrapper(f): + + def _clean_field(value): + ''' Helper function: apply bleach.clean to value, if applicable ''' + try: + return bleach.clean(value) + except TypeError: + return value + + @wraps(f) + def decorator(*args, **kwargs): + + # If user is admin, allow them to bypass cache with URL variable + no_cache = bool(user_is_admin() and request.args.get("nocache", False)) + + # Pull the raw data from either the form or the JSON body + raw_data = request.get_json() if from_json else request.form + + # If no form class provided + if form_class is None: + return f(*args, form_data={ k: _clean_field(v) for k, v in raw_data.items() }, no_cache=no_cache, **kwargs) + + # Construct the Flask form object + form = form_class(request.form) + + # Validate form fields + if not form.validate_on_submit(): + if err_msg and flash_err_msg: + flash(err_msg, 'danger') + return jsonify({ 'message': err_msg, 'errors': form.errors }), 400 + + # Read & clean fields from form, excluding CSRF token & file upload(s) + form_data = { + field.name: _clean_field(field.data) for field in form if field.name in request.form and field.id != 'csrf_token' + } + + # If no file(s) uploaded, evaluate here + if not len(request.files): + return f(*args, form_data=form_data, no_cache=no_cache, **kwargs) + + # Upload input file to server temporarily and add to the list of form fields + # TODO: This hardcodes the field name 'file' for a *single* file upload -- generalize whatever file field(s) are present + try: + with LocalUploadFile(request.files['file'], valid_file_extensions=TOOL_INPUT_DATA_VALID_FILE_EXTENSIONS) as local_file: + + # Pass the objects to the function + return f(*args, form_data={**form_data, 'file': local_file}, no_cache=no_cache, **kwargs) + + # If the file upload failed, display an error message + except FileUploadError as ex: + message = get_upload_err_msg(ex.code) + flash(message, 'danger') + return jsonify({ 'message': message }), ex.code + + return decorator + return wrapper diff --git a/src/modules/site-v2/base/views/api/notifications.py b/src/modules/site-v2/base/views/api/notifications.py index 43eb6c539..f997818ad 100644 --- a/src/modules/site-v2/base/views/api/notifications.py +++ b/src/modules/site-v2/base/views/api/notifications.py @@ -1,5 +1,6 @@ from flask import jsonify, Blueprint, url_for, abort, request +from base.utils.auth import access_token_required from base.utils.tools import lookup_report from base.views.tools import pairwise_indel_finder_bp, genetic_mapping_bp, heritability_calculator_bp @@ -28,13 +29,9 @@ def notifications(): @api_notifications_bp.route('/job-finish///', methods=['GET']) +@access_token_required(API_SITE_ACCESS_TOKEN) def job_finish(kind, id, status): - # Validate that this request came from the pipeline API - access_token = request.headers.get('Authorization') - if access_token != 'Bearer {}'.format(API_SITE_ACCESS_TOKEN): - abort(403) - # Fetch requested report, aborting if kind is invalid or report cannot be found try: job = lookup_report(kind, id, validate_user=False) @@ -51,7 +48,7 @@ def job_finish(kind, id, status): # Complete message if status == JobStatus.COMPLETE: template = REPORT_SUCCESS_EMAIL_TEMPLATE.strip('\n') - link = url_for(bp + '.report', id=report.id, _external=True) + link = url_for(bp + '.report', report_id=report.id, _external=True) # Error message elif status == JobStatus.ERROR: diff --git a/src/modules/site-v2/base/views/data/downloads.py b/src/modules/site-v2/base/views/data/downloads.py index f81c21999..6b7543adc 100644 --- a/src/modules/site-v2/base/views/data/downloads.py +++ b/src/modules/site-v2/base/views/data/downloads.py @@ -3,6 +3,8 @@ from base.utils.auth import jwt_required from extensions import cache +from base.utils.view_decorators import parse_species, parse_species_and_release + from caendr.api.strain import get_bam_bai_download_link, fetch_bam_bai_download_script, generate_bam_bai_download_script from caendr.models.datastore import DatasetRelease, Species from caendr.models.error import NotFoundError @@ -49,13 +51,8 @@ def download_script(species_name, release_version): @data_downloads_bp.route('/download///') @cache.memoize(60*60) @jwt_required() -def download_bam_bai_file(species_name='', strain_name='', ext=''): - - # Parse the species & release from the URL - try: - species = Species.from_name(species_name, from_url=True) - except NotFoundError: - return abort(404) +@parse_species +def download_bam_bai_file(species: Species, strain_name='', ext=''): # Get the download link for this strain signed_download_url = get_bam_bai_download_link(species, strain_name, ext) or '' @@ -71,14 +68,8 @@ def download_bam_bai_file(species_name='', strain_name='', ext=''): @data_downloads_bp.route('/download//latest/bam-bai-download-script', methods=['GET']) @data_downloads_bp.route('/download///bam-bai-download-script', methods=['GET']) -def download_bam_bai_script(species_name, release_version=None): - - # Parse the species & release from the URL - try: - species = Species.from_name(species_name, from_url=True) - release = DatasetRelease.from_name(release_version, species_name=species.name) - except NotFoundError: - return abort(404) +@parse_species_and_release +def download_bam_bai_script(species: Species, release: DatasetRelease): # Compute the desired filename from the species & release filename = BAM_BAI_DOWNLOAD_SCRIPT_NAME.get_string(**{ diff --git a/src/modules/site-v2/base/views/data/releases.py b/src/modules/site-v2/base/views/data/releases.py index 7004c3ad1..c66e0326d 100644 --- a/src/modules/site-v2/base/views/data/releases.py +++ b/src/modules/site-v2/base/views/data/releases.py @@ -18,6 +18,7 @@ from extensions import cache from base.forms import VBrowserForm from base.utils.auth import jwt_required +from base.utils.view_decorators import parse_species_and_release from caendr.api.strain import query_strains from caendr.api.isotype import get_isotypes @@ -25,7 +26,6 @@ from caendr.models.sql import Strain, StrainAnnotatedVariant from caendr.services.cloud.storage import BlobURISchema, generate_blob_uri from caendr.services.dataset_release import get_all_dataset_releases, get_browser_tracks_path, get_release_bucket, find_dataset_release -from caendr.models.error import NotFoundError, SpeciesUrlNameError from caendr.utils.env import get_env_var @@ -37,17 +37,6 @@ ) -def interpret_url_vars(species_name, release_version): - species = Species.from_name(species_name, from_url=True) - - if species.get_slug() != species_name: - raise SpeciesUrlNameError(species.get_slug()) - - releases = get_all_dataset_releases(order='-version', species=species.name) - release = find_dataset_release(releases, release_version) - - return species, releases, release - # ============= # # Data Page # @@ -67,31 +56,20 @@ def data_releases(): }) -@releases_bp.route('//latest') -@releases_bp.route('//') +@releases_bp.route('//latest') +@releases_bp.route('//') @cache.memoize(60*60) -def data_release_list(species, release_version=None): +@parse_species_and_release +def data_release_list(species: Species, release: DatasetRelease): """ Default data page - lists available releases. """ - # Look up the species and release version - try: - species, releases, release = interpret_url_vars(species, release_version) - - # If species name provided with underscore instead of dash, redirect to dashed version of URL - except SpeciesUrlNameError as ex: - return redirect(url_for('data_releases.data_release_list', species=ex.species_name, release_version=release_version)) - - # If either could not be found, return an error page - except NotFoundError: - return abort(404) - # Package common params into an object params = { 'species': species, 'RELEASE': release, - 'RELEASES': releases, + 'RELEASES': get_all_dataset_releases(order='-version', species=species.name), 'release_bucket': get_release_bucket(), 'release_path': release.get_versioned_path_template().get_string(SPECIES = species.name), 'fasta_path': release.get_fasta_filepath(schema=BlobURISchema.HTTPS) if release.check_fasta_file_exists() else None, @@ -170,22 +148,11 @@ def data_v01(params, files): # ======================= # # Alignment Data Page # # ======================= # -@releases_bp.route('//latest/alignment') -@releases_bp.route('///alignment') +@releases_bp.route('//latest/alignment') +@releases_bp.route('///alignment') @cache.memoize(60*60) -def alignment_data(species, release_version=None): - - # Look up the species and release version - try: - species, releases, release = interpret_url_vars(species, release_version) - - # If species name provided with underscore instead of dash, redirect to dashed version of URL - except SpeciesUrlNameError as ex: - return redirect(url_for('data_releases.alignment_data', species=ex.species_name, release_version=release_version)) - - # If either could not be found, return an error page - except NotFoundError: - return abort(404) +@parse_species_and_release +def alignment_data(species: Species, release: DatasetRelease): # Pre-2020 releases don't have data organized the same way # TODO: Error page? Redirect to main release page? @@ -200,9 +167,9 @@ def alignment_data(species, release_version=None): 'species': species, 'RELEASE': release, - 'RELEASES': releases, + 'RELEASES': get_all_dataset_releases(order='-version', species=species.name), - 'strain_listing': query_strains(release_version=release_version, species=species.name), + 'strain_listing': query_strains(release_version=release['version'], species=species.name), }) # DATASET_RELEASE, WORMBASE_VERSION = list(filter(lambda x: x[0] == release_version, RELEASES))[0] # REPORTS = ["alignment"] @@ -212,27 +179,16 @@ def alignment_data(species, release_version=None): # =========================== # # Strain Issues Data Page # # =========================== # -@releases_bp.route('//latest/strain_issues') -@releases_bp.route('///strain_issues') +@releases_bp.route('//latest/strain_issues') +@releases_bp.route('///strain_issues') @cache.memoize(60*60) -def strain_issues(species, release_version=None): +@parse_species_and_release +def strain_issues(species: Species, release: DatasetRelease): """ Strain Issues page Lists all strains with known issues for a given species & release. """ - # Look up the species and release version - try: - species, releases, release = interpret_url_vars(species, release_version) - - # If species name provided with underscore instead of dash, redirect to dashed version of URL - except SpeciesUrlNameError as ex: - return redirect(url_for('data_releases.strain_issues', species=ex.species_name, release_version=release_version)) - - # If either could not be found, return an error page - except NotFoundError: - return abort(404) - # Pre-2020 releases don't have data organized the same way # TODO: Error page? Redirect to main release page? if release.report_type == DatasetRelease.V1: @@ -246,7 +202,7 @@ def strain_issues(species, release_version=None): 'species': species, 'RELEASE': release, - 'RELEASES': releases, + 'RELEASES': get_all_dataset_releases(order='-version', species=species.name), - 'strain_listing_issues': query_strains(release_version=release_version, species=species.name, issues=True), + 'strain_listing_issues': query_strains(release_version=release['version'], species=species.name, issues=True), }) diff --git a/src/modules/site-v2/base/views/tools/genetic_mapping.py b/src/modules/site-v2/base/views/tools/genetic_mapping.py index 5c8e3fc76..65bb78351 100755 --- a/src/modules/site-v2/base/views/tools/genetic_mapping.py +++ b/src/modules/site-v2/base/views/tools/genetic_mapping.py @@ -2,24 +2,18 @@ from caendr.services.logger import logger from flask import Blueprint, render_template, request, redirect, url_for, flash, abort -import bleach from flask import jsonify from base.forms import MappingForm from base.utils.auth import get_jwt, jwt_required, admin_required, get_current_user, user_is_admin -from base.utils.tools import get_upload_err_msg, lookup_report, list_reports, try_submit -from constants import TOOL_INPUT_DATA_VALID_FILE_EXTENSIONS +from base.utils.tools import list_reports, try_submit +from base.utils.view_decorators import parse_job_id, validate_form -from caendr.services.cloud.storage import BlobURISchema, generate_blob_uri, get_blob, get_blob_list, check_blob_exists +from caendr.services.cloud.storage import BlobURISchema, generate_blob_uri from caendr.models.datastore import Species, NemascanReport -from caendr.models.error import ( - FileUploadError, - ReportLookupError, -) from caendr.models.job_pipeline import NemascanPipeline from caendr.models.status import JobStatus from caendr.utils.env import get_env_var -from caendr.utils.local_files import LocalUploadFile MODULE_SITE_BUCKET_ASSETS_NAME = get_env_var('MODULE_SITE_BUCKET_ASSETS_NAME') @@ -69,57 +63,29 @@ def genetic_mapping(): @genetic_mapping_bp.route('/submit', methods=['POST']) @jwt_required() -def submit(): - form = MappingForm(request.form) - user = get_current_user() - - # If user is admin, allow them to bypass cache with URL variable - no_cache = bool(user_is_admin() and request.args.get("nocache", False)) - - # Validate form fields - # Checks that species is in species list & label is not empty - if not form.validate_on_submit(): - msg = "You must include a description of your data and a CSV file to upload." - flash(msg, "danger") - return jsonify({ 'message': msg }), 400 - - # Read fields from form - label = bleach.clean(request.form.get('label')) - species = bleach.clean(request.form.get('species')) - - # Upload input file to server temporarily, and start the job - try: - with LocalUploadFile(request.files.get('file'), valid_file_extensions=TOOL_INPUT_DATA_VALID_FILE_EXTENSIONS) as file: - - # Package submission data together into dict - data = { 'label': label, 'species': species, 'file': file } - - # Try submitting the job & returning a JSON status message - response, code = try_submit(NemascanReport.kind, user, data, no_cache) +@validate_form(MappingForm, err_msg='You must include a description of your data and a CSV file to upload.') +def submit(form_data, no_cache=False): - # If there was an error, flash it - if code != 200 and int(request.args.get('reloadonerror', 1)): - flash(response['message'], 'danger') + # Try submitting the job & returning a JSON status message + response, code = try_submit(NemascanReport.kind, get_current_user(), form_data, no_cache) - # If the response contains a caching message, flash it - elif response.get('message') and response.get('ready', False): - flash(response.get('message'), 'success') + # If there was an error, flash it + if code != 200 and int(request.args.get('reloadonerror', 1)): + flash(response['message'], 'danger') - # Return the response - return jsonify( response ), code + # If the response contains a caching message, flash it + elif response.get('message') and response.get('ready', False): + flash(response.get('message'), 'success') - # If the file upload failed, display an error message - except FileUploadError as ex: - message = get_upload_err_msg(ex.code) - flash(message, 'danger') - return jsonify({ 'message': message }), ex.code + # Return the response + return jsonify( response ), code @genetic_mapping_bp.route('/all-results', methods=['GET'], endpoint='all_results') @genetic_mapping_bp.route('/my-results', methods=['GET'], endpoint='my_results') @jwt_required() def list_results(): - show_all = request.path.endswith('all-results') + show_all = request.endpoint.endswith('all_results') user = get_current_user() # Only show malformed Entities to admin users @@ -149,19 +115,10 @@ def list_results(): }) -@genetic_mapping_bp.route('/report/', methods=['GET']) +@genetic_mapping_bp.route('/report/', methods=['GET']) @jwt_required() -def report(id): - - # Fetch requested mapping report - # Ensures the report exists and the user has permission to view it - try: - job: NemascanPipeline = lookup_report(NemascanReport.kind, id) - - # If the report lookup request is invalid, show an error message - except ReportLookupError as ex: - flash(ex.msg, 'danger') - abort(ex.code) +@parse_job_id(NemascanPipeline, fetch=False) +def report(job: NemascanPipeline): # Get the trait name, if it exists trait = job.report['trait'] @@ -176,7 +133,7 @@ def report(id): # Job status 'mapping_status': job.report['status'], - 'id': id, + 'report_id': job.report.id, # Links to the input data file and report output files, if they exist 'data_download_url': job.report.input_filepath( schema = BlobURISchema.HTTPS, check_if_exists = True ), @@ -186,19 +143,10 @@ def report(id): }) -@genetic_mapping_bp.route('/report//fullscreen', methods=['GET']) +@genetic_mapping_bp.route('/report//fullscreen', methods=['GET']) @jwt_required() -def report_fullscreen(id): - - # Fetch requested mapping report - # Ensures the report exists and the user has permission to view it - try: - job: NemascanPipeline = lookup_report(NemascanReport.kind, id) - - # If the report lookup request is invalid, show an error message - except ReportLookupError as ex: - flash(ex.msg, 'danger') - abort(ex.code) +@parse_job_id(NemascanPipeline, fetch=False) +def report_fullscreen(job: NemascanPipeline): # Download the report files, if they exist report_contents = job.fetch_output() @@ -232,35 +180,21 @@ def report_status(id): return jsonify(payload) -@genetic_mapping_bp.route('/report//results', methods=['GET']) +@genetic_mapping_bp.route('/report//results', methods=['GET']) @jwt_required() -def results(id): - - # Fetch requested mapping report - # Ensures the report exists and the user has permission to view it - try: - job: NemascanPipeline = lookup_report(NemascanReport.kind, id) - - # If the report lookup request is invalid, show an error message - except ReportLookupError as ex: - flash(ex.msg, 'danger') - abort(ex.code) +@parse_job_id(NemascanPipeline, fetch=False) +def results(job: NemascanPipeline): # Get the trait, if it exists trait = job.report['trait'] - # # Old way to compute list of blobs, that was hidden beneath 'return' - # # Can this be deleted? - # data_blob = RESULT_BLOB_PATH.format(data_hash=ns.data_hash) - # blobs = list_files(data_blob) - # Get the list of files in this report, truncating all names to everything after second-to-last '/' file_list = [ { "name": '/'.join( blob.name.rsplit('/', 2)[1:] ), "url": blob.public_url, } - for blob in job.report.list_output_blobs() + for blob in job.report.list_output_directory() ] return render_template('tools/genetic_mapping/result_files.html', **{ diff --git a/src/modules/site-v2/base/views/tools/heritability_calculator.py b/src/modules/site-v2/base/views/tools/heritability_calculator.py index 85ea559c6..d59df482d 100644 --- a/src/modules/site-v2/base/views/tools/heritability_calculator.py +++ b/src/modules/site-v2/base/views/tools/heritability_calculator.py @@ -12,27 +12,20 @@ abort) from caendr.services.logger import logger from datetime import datetime -import bleach from base.forms import HeritabilityForm from base.utils.auth import jwt_required, admin_required, get_jwt, get_current_user, user_is_admin -from base.utils.tools import get_upload_err_msg, lookup_report, list_reports, try_submit -from constants import TOOL_INPUT_DATA_VALID_FILE_EXTENSIONS - -from caendr.models.error import ( - EmptyReportDataError, - EmptyReportResultsError, - FileUploadError, - ReportLookupError, -) +from base.utils.tools import list_reports, try_submit +from base.utils.view_decorators import parse_job_id, validate_form + from caendr.models.datastore import Species, HeritabilityReport from caendr.models.job_pipeline import HeritabilityPipeline from caendr.models.status import JobStatus +from caendr.models.job_pipeline import HeritabilityPipeline from caendr.api.strain import get_strains from caendr.utils.data import unique_id, get_object_hash from caendr.utils.env import get_env_var -from caendr.utils.local_files import LocalUploadFile -from caendr.services.cloud.storage import get_blob, generate_blob_uri, BlobURISchema +from caendr.services.cloud.storage import generate_blob_uri, BlobURISchema from caendr.services.persistent_logger import PersistentLogger @@ -93,7 +86,7 @@ def create(): @heritability_calculator_bp.route('/my-results', methods=['GET'], endpoint='my_results') @jwt_required() def list_results(): - show_all = request.path.endswith('all-results') + show_all = request.endpoint.endswith('all_results') user = get_current_user() # Only show malformed Entities to admin users @@ -126,128 +119,56 @@ def list_results(): @heritability_calculator_bp.route('/submit', methods=["POST"]) @jwt_required() -def submit(): - form = HeritabilityForm(request.form) - user = get_current_user() - - # Validate form fields - # Checks that species is in species list & label is not empty - if not form.validate_on_submit(): - msg = "You must include a description of your data and a CSV file to upload." - flash(msg, "danger") - return jsonify({ 'message': msg }), 400 +@validate_form(HeritabilityForm, err_msg='You must include a description of your data and a CSV file to upload.') +def submit(form_data, no_cache=False): - # If user is admin, allow them to bypass cache with URL variable - no_cache = bool(user_is_admin() and request.args.get("nocache", False)) + # Try submitting the job & returning a JSON status message + response, code = try_submit( HeritabilityReport.kind, get_current_user(), form_data, no_cache=no_cache ) - # Read fields from form - label = bleach.clean(request.form.get('label')) - species = bleach.clean(request.form.get('species')) + # If there was an error, flash it + if code != 200 and int(request.args.get('reloadonerror', 1)): + flash(response['message'], 'danger') - # Upload input file to server temporarily, and start the job - try: - with LocalUploadFile(request.files.get('file'), valid_file_extensions=TOOL_INPUT_DATA_VALID_FILE_EXTENSIONS) as local_file: + # If the response contains a caching message, flash it + elif response.get('message') and response.get('ready', False): + flash(response.get('message'), 'success') - # Package submission data together into dict - data = { 'label': label, 'species': species, 'file': local_file } + # Return the response + return jsonify( response ), code - # Try submitting the job & returning a JSON status message - response, code = try_submit(HeritabilityReport.kind, user, data, no_cache) - # If there was an error, flash it - if code != 200 and int(request.args.get('reloadonerror', 1)): - flash(response['message'], 'danger') - - # If the response contains a caching message, flash it - elif response.get('message') and response.get('ready', False): - flash(response.get('message'), 'success') - - # Return the response - return jsonify( response ), code - - # If the file upload failed, display an error message - except FileUploadError as ex: - message = get_upload_err_msg(ex.code) - flash(message, 'danger') - return jsonify({ 'message': message }), ex.code +@heritability_calculator_bp.route("/report//logs") +@jwt_required() +@parse_job_id(HeritabilityPipeline, fetch=False) +def view_logs(job: HeritabilityPipeline): + # Collect all the log files in this report's work folder + blobs = job.report.list_work_directory( + filter=lambda blob: 'google/logs/action' in blob.name or '.command' in blob.name + ) -@heritability_calculator_bp.route("/report//logs") -@jwt_required() -def view_logs(id): - hr = HeritabilityReport.get_ds(id) - # get workflow bucket - from google.cloud import storage - storage_client = storage.Client() - bucket_name = os.getenv('MODULE_API_PIPELINE_TASK_WORK_BUCKET_NAME', None) - - if bucket_name is None: - return None - prefix = f"{hr.data_hash}" - # caendr-nextflow-work-bucket/938f561278fbdd4a546155f37cdaf47f/d4/ed062b62843eb156a22d303e0ce84b/google/logs - - blobs = storage_client.list_blobs(bucket_name, prefix=prefix, delimiter=None) - filepaths = [ blob.name for blob in blobs ] - log_filepaths = [ filepath for filepath in filepaths if "google/logs/action" in filepath or ".command" in filepath ] - + # Filter out all empty log files logs = [] - for log_filepath in log_filepaths: - data = get_blob(bucket_name, log_filepath).download_as_string().decode('utf-8').strip() - if data == "": - continue - log = { - 'blob_name': log_filepath, - 'data': data - } - logs.append(log) + for blob in blobs: + data = blob.download_as_string().decode('utf-8').strip() + if data != '': + logs.append({ 'blob_name': blob.name, 'data': data }) - return render_template("tools/heritability_calculator/logs.html", **locals()) + return render_template("tools/heritability_calculator/logs.html", logs=logs) -@heritability_calculator_bp.route("/report/", methods=['GET']) +@heritability_calculator_bp.route("/report/", methods=['GET']) @jwt_required() -def report(id): - - user = get_current_user() - - # Fetch requested heritability report - # Ensures the report exists and the user has permission to view it - try: - job: HeritabilityPipeline = lookup_report(HeritabilityReport.kind, id, user=user) - - # If the report lookup request is invalid, show an error message - except ReportLookupError as ex: - flash(ex.msg, 'danger') - abort(ex.code) - - # Try getting & parsing the report data file and results - # If result is None, job hasn't finished computing yet - try: - data, result = job.fetch() - ready = result is not None - - # Error reading one of the report files - except (EmptyReportDataError, EmptyReportResultsError) as ex: - logger.error(f'Error fetching Heritability report {ex.id}: {ex.description}') - return abort(404, description = ex.description) - - # General error - except Exception as ex: - logger.error(f'Error fetching Heritability report {id}: {ex}') - return abort(400, description = 'Something went wrong') - - # No data file found - if data is None: - logger.error(f'Error fetching Heritability report {id}: Input data file does not exist') - return abort(404) - +@parse_job_id(HeritabilityPipeline) +def report(job: HeritabilityPipeline, data, result): + ready = result is not None trait = data[0]['TraitName'] # # TODO: Is this used? It looks like the error message(s) come from the entity's PipelineOperation # service_name = os.getenv('HERITABILITY_CONTAINER_NAME') # persistent_logger = PersistentLogger(service_name) - # error = persistent_logger.get(id) + # error = persistent_logger.get(job.report.id) return render_template("tools/heritability_calculator/result.html", **{ 'title': "Heritability Results", @@ -264,7 +185,7 @@ def report(id): 'error': job.get_error(), 'data_url': job.report.input_filepath(schema=BlobURISchema.HTTPS), - 'logs_url': url_for('heritability_calculator.view_logs', id = id), + 'logs_url': url_for('heritability_calculator.view_logs', report_id = job.report.id), 'JobStatus': JobStatus, }) diff --git a/src/modules/site-v2/base/views/tools/pairwise_indel_finder.py b/src/modules/site-v2/base/views/tools/pairwise_indel_finder.py index 5c7b09c48..2acde147c 100644 --- a/src/modules/site-v2/base/views/tools/pairwise_indel_finder.py +++ b/src/modules/site-v2/base/views/tools/pairwise_indel_finder.py @@ -5,11 +5,12 @@ from base.forms import PairwiseIndelForm from base.utils.auth import jwt_required, admin_required, get_current_user, user_is_admin -from base.utils.tools import lookup_report, list_reports, try_submit +from base.utils.tools import list_reports, try_submit +from base.utils.view_decorators import parse_job_id, validate_form from caendr.models.datastore.browser_track import BrowserTrackDefault from caendr.models.datastore import Species, IndelPrimerReport, DatasetRelease -from caendr.models.error import NotFoundError, NonUniqueEntity, ReportLookupError, EmptyReportDataError, EmptyReportResultsError +from caendr.models.error import NotFoundError, NonUniqueEntity from caendr.models.job_pipeline import IndelFinderPipeline from caendr.models.status import JobStatus from caendr.services.dataset_release import get_dataset_release @@ -133,7 +134,7 @@ def pairwise_indel_finder(): @pairwise_indel_finder_bp.route('/my-results', methods=['GET'], endpoint='my_results') @jwt_required() def list_results(): - show_all = request.path.endswith('all-results') + show_all = request.endpoint.endswith('all_results') user = get_current_user() # Only show malformed Entities to admin users @@ -169,44 +170,25 @@ def list_results(): @pairwise_indel_finder_bp.route("/query-indels", methods=["POST"]) @jwt_required() -def query(): +@validate_form(PairwiseIndelForm) +def query(form_data, no_cache=False): - # Validate query form - form = PairwiseIndelForm() - if form.validate_on_submit(): + # If either of the strains is missing, raise a 422 Unprocessable Entity error + if form_data.get('strain_1') is None or form_data.get('strain_2') is None: + return {}, 422 - # Extract fields - species = form.data['species'] - strain_1 = form.data['strain_1'] - strain_2 = form.data['strain_2'] - chrom = form.data['chromosome'] - start = form.data['start'] - stop = form.data['stop'] - - # Run query and return results - results = query_indels_and_mark_overlaps(species, strain_1, strain_2, chrom, start, stop) - return jsonify({ 'results': results }) - - # If form not valid, return errors - return jsonify({ 'errors': form.errors }) + # Pass the form fields to the query function & return the result + return jsonify({ 'results': query_indels_and_mark_overlaps(**form_data) }) @pairwise_indel_finder_bp.route('/submit', methods=["POST"]) @jwt_required() -def submit(): - - # Get current user - user = get_current_user() - - # Get info about data - data = request.get_json() - - # If user is admin, allow them to bypass cache with URL variable - no_cache = bool(user_is_admin() and request.args.get("nocache", False)) +@validate_form(None, from_json=True) +def submit(form_data, no_cache=False): # Try submitting the job & getting a JSON status message - response, code = try_submit(IndelPrimerReport.kind, user, data, no_cache) + response, code = try_submit(IndelPrimerReport.kind, get_current_user(), form_data, no_cache) # If there was an error, flash it if code != 200 and int(request.args.get('reloadonerror', 1)): @@ -217,10 +199,11 @@ def submit(): -@pairwise_indel_finder_bp.route("/report/") -@pairwise_indel_finder_bp.route("/report//download/") +@pairwise_indel_finder_bp.route("/report/", methods=['GET']) +@pairwise_indel_finder_bp.route("/report//download/", methods=['GET']) @jwt_required() -def report(id, file_ext=None): +@parse_job_id(IndelFinderPipeline) +def report(job: IndelFinderPipeline, data, result, file_ext=None): # Validate file extension, if provided if file_ext: @@ -230,37 +213,8 @@ def report(id, file_ext=None): else: file_format = None - # Fetch requested primer report - # Ensures the report exists and the user has permission to view it - try: - job: IndelFinderPipeline = lookup_report(IndelPrimerReport.kind, id) - - # If the report lookup request is invalid, show an error message - except ReportLookupError as ex: - flash(ex.msg, 'danger') - abort(ex.code) - - - # Try getting the report data file and results - # If result is None, job hasn't finished computing yet - try: - data, result = job.fetch() - ready = result is not None - - # Error reading one of the report files - except (EmptyReportDataError, EmptyReportResultsError) as ex: - logger.error(f'Error fetching Indel Finder report {ex.id}: {ex.description}') - return abort(404, description = ex.description) - - # General error - except Exception as ex: - logger.error(f'Error fetching Indel Finder report {id}: {ex}') - return abort(400, description = 'Something went wrong') - - # No data file found - if data is None: - logger.error(f'Error fetching Indel Finder report {id}: Input data file does not exist') - return abort(404) + # Report is ready if result exists + ready = result is not None # If the result is empty, make it an empty dict, for more straightforward field access in the rest of the function if not ready: @@ -306,7 +260,7 @@ def report(id, file_ext=None): # GCP data info 'data_hash': job.report.data_hash, - 'id': id, + 'report_id': job.report.id, # Job status 'empty': result.get('empty'), diff --git a/src/modules/site-v2/base/views/tools/phenotype_database.py b/src/modules/site-v2/base/views/tools/phenotype_database.py index 86d2717ff..e7b523d1f 100644 --- a/src/modules/site-v2/base/views/tools/phenotype_database.py +++ b/src/modules/site-v2/base/views/tools/phenotype_database.py @@ -20,10 +20,11 @@ from base.forms import EmptyForm from base.utils.auth import jwt_required, get_current_user, user_is_admin -from base.utils.tools import lookup_report, list_reports, try_submit +from base.utils.tools import list_reports, try_submit +from base.utils.view_decorators import parse_job_id, validate_form from caendr.models.datastore import PhenotypeReport, Species -from caendr.models.error import ReportLookupError, EmptyReportDataError, EmptyReportResultsError, NotFoundError, DataValidationError +from caendr.models.error import NotFoundError from caendr.models.job_pipeline import PhenotypePipeline from caendr.models.status import JobStatus from caendr.models.sql import PhenotypeMetadata @@ -223,25 +224,15 @@ def submit_traits(): @phenotype_database_bp.route('/submit', methods=["POST"]) @jwt_required() -def submit(): +@validate_form(None, from_json=True) +def submit(form_data, no_cache=False): - # Read & clean fields from JSON data - data = { - field: bleach.clean(request.json.get(field)) - for field in {'species', 'trait_1', 'trait_1_dataset'} - } - - # Read & clean values for trait 2, if given - trait_2 = request.json.get('trait_2') - trait_2_dataset = request.json.get('trait_2_dataset') - data['trait_2'] = bleach.clean(trait_2) if trait_2 is not None else None - data['trait_2_dataset'] = bleach.clean(trait_2_dataset) if trait_2_dataset is not None else None - - # If user is admin, allow them to bypass cache with URL variable - no_cache = bool(user_is_admin() and request.args.get("nocache", False)) + # Make sure these keys exist in the form data, even if they weren't provided in the submission + form_data['trait_2'] = form_data.get('trait_2', None) + form_data['trait_2_dataset'] = form_data.get('trait_2_dataset', None) # Try submitting the job & getting a JSON status message - response, code = try_submit(PhenotypeReport.kind, get_current_user(), data, no_cache) + response, code = try_submit(PhenotypeReport.kind, get_current_user(), form_data, no_cache) # If there was an error, flash it if code != 200 and int(request.args.get('reloadonerror', 1)): @@ -259,7 +250,7 @@ def submit(): @phenotype_database_bp.route('/my-results', methods=['GET'], endpoint='my_results') @jwt_required() def list_results(): - show_all = request.path.endswith('all-results') + show_all = request.endpoint.endswith('all_results') user = get_current_user() # Only show malformed Entities to admin users @@ -292,10 +283,11 @@ def list_results(): }) -@phenotype_database_bp.route("/report/", methods=['GET']) -@phenotype_database_bp.route("/report//download/", methods=['GET']) +@phenotype_database_bp.route("/report/", methods=['GET']) +@phenotype_database_bp.route("/report//download/", methods=['GET']) @jwt_required() -def report(id, file_ext=None): +@parse_job_id(PhenotypePipeline) +def report(job: PhenotypePipeline, data, result, file_ext=None): # Validate file extension, if provided if file_ext: @@ -305,42 +297,6 @@ def report(id, file_ext=None): else: file_format = None - # Fetch requested phenotype report - # Ensures the report exists and the user has permission to view it - try: - job: PhenotypePipeline = lookup_report(PhenotypeReport.kind, id) - - # If the report lookup request is invalid, show an error message - except ReportLookupError as ex: - flash(ex.msg, 'danger') - abort(ex.code) - - # Try getting & parsing the report data file and results - # If result is None, job hasn't finished computing yet - try: - data, result = job.fetch() - - # Error reading one of the report files - except (EmptyReportDataError, EmptyReportResultsError) as ex: - logger.error(f'Error fetching Phenotype report {ex.id}: {ex.description}') - return abort(404, description = ex.description) - - # Error with the submission data - # This should only be possible if a report was somehow created with invalid data, e.g. not enough traits - except DataValidationError as ex: - flash(ex.msg, 'error') - return abort(400, description = ex.msg) - - # General error - except Exception as ex: - logger.error(f'Error fetching Phenotype report {id}: {ex}') - return abort(400, description = 'Something went wrong') - - # No data file found - if data is None: - logger.error(f'Error fetching Phenotype report {id}: Input data does not exist') - return abort(404) - # If a file format was specified, return a downloadable file with the results if file_format is not None: columns = ['strain', *data['trait_names']] diff --git a/src/modules/site-v2/templates/_includes/report_list_macros.j2 b/src/modules/site-v2/templates/_includes/report_list_macros.j2 index 784b2f086..3d974b328 100644 --- a/src/modules/site-v2/templates/_includes/report_list_macros.j2 +++ b/src/modules/site-v2/templates/_includes/report_list_macros.j2 @@ -6,7 +6,7 @@ {%- set render_value = item[column.field] if (column.field and item[column.field]) else column.value %} {%- if create_link %} - + {%- endif %} {%- if column.render %} {{ column.render(render_value) }} {% else %} {{ render_value }} {% endif %} {%- if create_link %} diff --git a/src/modules/site-v2/templates/_scripts/submit-job.js b/src/modules/site-v2/templates/_scripts/submit-job.js index 78c11fabb..d280dc0c4 100644 --- a/src/modules/site-v2/templates/_scripts/submit-job.js +++ b/src/modules/site-v2/templates/_scripts/submit-job.js @@ -57,9 +57,9 @@ function {{func_name}}(data, modal_id, config={}) { // TODO: Redirect modal if (result.ready && auto_redirect) { if (new_tab) { - window.open(`{{ url_for(tool_name + '.report', id='') }}${ result.id }`, '_blank'); + window.open(`{{ url_for(tool_name + '.report', report_id='') }}${ result.id }`, '_blank'); } else { - window.location.href = `{{ url_for(tool_name + '.report', id='') }}${ result.id }`; + window.location.href = `{{ url_for(tool_name + '.report', report_id='') }}${ result.id }`; } } diff --git a/src/modules/site-v2/templates/data/alignment.html b/src/modules/site-v2/templates/data/alignment.html index 5f0fa4787..cdb0d2a7b 100755 --- a/src/modules/site-v2/templates/data/alignment.html +++ b/src/modules/site-v2/templates/data/alignment.html @@ -3,7 +3,7 @@ {% block content %}

Bolded strains are isotype reference strains, and tabbed non-bold face strains are strains within an isotype group but not the isotype reference strain.

-

Only strains with whole-genome sequencing data have BAM files for download. If you do not see a strain, check the Strain Issues page. +

Only strains with whole-genome sequencing data have BAM files for download. If you do not see a strain, check the Strain Issues page. Some strains have been flagged and removed from distribution and analysis pipelines for a variety of reasons.

{% include('releases/download_tab_strain_v2.html') %} diff --git a/src/modules/site-v2/templates/data/landing.html b/src/modules/site-v2/templates/data/landing.html index fed2aae9e..7197eadb2 100755 --- a/src/modules/site-v2/templates/data/landing.html +++ b/src/modules/site-v2/templates/data/landing.html @@ -9,7 +9,7 @@
diff --git a/src/modules/site-v2/templates/data/releases.html b/src/modules/site-v2/templates/data/releases.html index 723f9dd97..d84cd9191 100644 --- a/src/modules/site-v2/templates/data/releases.html +++ b/src/modules/site-v2/templates/data/releases.html @@ -9,7 +9,7 @@

Releases

{% for RELEASE in RELEASES %} {% endfor %} diff --git a/src/modules/site-v2/templates/data/v2.html b/src/modules/site-v2/templates/data/v2.html index bafe66521..b30bc5c88 100644 --- a/src/modules/site-v2/templates/data/v2.html +++ b/src/modules/site-v2/templates/data/v2.html @@ -91,14 +91,14 @@

Datasets

Strain Issues This link + href="{{ url_for('data_releases.strain_issues', species_name = species.get_slug(), release_version = release_version) }}">link contains all strain issues for this release Alignment Data This link + href="{{ url_for('data_releases.alignment_data', species_name = species.get_slug(), release_version = release_version) }}">link contains all alignment data as BAM or BAI files. diff --git a/src/modules/site-v2/templates/tools/genetic_mapping/mapping.html b/src/modules/site-v2/templates/tools/genetic_mapping/mapping.html index ac912e333..d37f9dfd0 100644 --- a/src/modules/site-v2/templates/tools/genetic_mapping/mapping.html +++ b/src/modules/site-v2/templates/tools/genetic_mapping/mapping.html @@ -191,7 +191,7 @@

Example Data

// Setting up Release Notes link $('#nav-linkTwo-tab').click(function() { const species = $('#speciesSelect').val() - let url = "{{ url_for('data_releases.data_release_list', species='SPECIES') }}"; + let url = "{{ url_for('data_releases.data_release_list', species_name='SPECIES') }}"; if (species) { url = url.replace("SPECIES", species); } else { diff --git a/src/modules/site-v2/templates/tools/genetic_mapping/report.html b/src/modules/site-v2/templates/tools/genetic_mapping/report.html index ac0bad4d6..1916de181 100644 --- a/src/modules/site-v2/templates/tools/genetic_mapping/report.html +++ b/src/modules/site-v2/templates/tools/genetic_mapping/report.html @@ -17,7 +17,7 @@

Your Results

Download Input Data - + View Result Files @@ -30,7 +30,7 @@

Your Results

Download Report {% if report_url %} - + {% else %} {% endif %} @@ -50,7 +50,7 @@

Your Results

{% if mapping_status == 'COMPLETE' %}
{% if report_url %} - + {% else %}
diff --git a/src/modules/site-v2/templates/tools/genome_browser/gbrowser.html b/src/modules/site-v2/templates/tools/genome_browser/gbrowser.html index 2f23565be..2774431c0 100755 --- a/src/modules/site-v2/templates/tools/genome_browser/gbrowser.html +++ b/src/modules/site-v2/templates/tools/genome_browser/gbrowser.html @@ -485,7 +485,7 @@

Track Information

// Setting up Release Notes link $('#nav-linkTwo-tab').click(function() { const species = $('#speciesSelect').val() - let url = "{{ url_for('data_releases.data_release_list', species='SPECIES') }}"; + let url = "{{ url_for('data_releases.data_release_list', species_name='SPECIES') }}"; if (species) { url = url.replace("SPECIES", species); } else { diff --git a/src/modules/site-v2/templates/tools/heritability_calculator/heritability-calculator.html b/src/modules/site-v2/templates/tools/heritability_calculator/heritability-calculator.html index 35f5047cd..a46a96576 100644 --- a/src/modules/site-v2/templates/tools/heritability_calculator/heritability-calculator.html +++ b/src/modules/site-v2/templates/tools/heritability_calculator/heritability-calculator.html @@ -189,7 +189,7 @@

Example Data

// Setting up Release Notes link $('#nav-linkTwo-tab').click(function() { const species = $('#speciesSelect').val() - let url = "{{ url_for('data_releases.data_release_list', species='SPECIES') }}"; + let url = "{{ url_for('data_releases.data_release_list', species_name='SPECIES') }}"; if (species) { url = url.replace("SPECIES", species); } else { diff --git a/src/modules/site-v2/templates/tools/heritability_calculator/result.html b/src/modules/site-v2/templates/tools/heritability_calculator/result.html index 95c5d877c..2f81b4421 100644 --- a/src/modules/site-v2/templates/tools/heritability_calculator/result.html +++ b/src/modules/site-v2/templates/tools/heritability_calculator/result.html @@ -72,7 +72,7 @@
Download PDF Download TSV {%- if session["is_admin"] %} - View Logs + View Logs {%- endif %}
diff --git a/src/modules/site-v2/templates/tools/heritability_calculator/submit_OLD.html b/src/modules/site-v2/templates/tools/heritability_calculator/submit_OLD.html index 1f47d948d..0acaa5914 100644 --- a/src/modules/site-v2/templates/tools/heritability_calculator/submit_OLD.html +++ b/src/modules/site-v2/templates/tools/heritability_calculator/submit_OLD.html @@ -347,7 +347,7 @@ window.location = "{{ url_for('heritability_calculator.user_results') }}"; return; } else { - url = "{{ url_for('heritability_calculator.report', id='') }}" + result.id; + url = "{{ url_for('heritability_calculator.report', report_id='') }}" + result.id; console.log(url); window.location = url; return; diff --git a/src/modules/site-v2/templates/tools/pairwise_indel_finder/submit.html b/src/modules/site-v2/templates/tools/pairwise_indel_finder/submit.html index 665673445..8c60237d2 100644 --- a/src/modules/site-v2/templates/tools/pairwise_indel_finder/submit.html +++ b/src/modules/site-v2/templates/tools/pairwise_indel_finder/submit.html @@ -656,7 +656,7 @@

Thank You

// Setting up Release Notes link $('#nav-linkTwo-tab').click(function() { const species = $('#speciesSelect').val() - let url = "{{ url_for('data_releases.data_release_list', species='SPECIES') }}"; + let url = "{{ url_for('data_releases.data_release_list', species_name='SPECIES') }}"; if (species) { url = url.replace("SPECIES", species); } else { diff --git a/src/modules/site-v2/templates/tools/pairwise_indel_finder/view.html b/src/modules/site-v2/templates/tools/pairwise_indel_finder/view.html index 56b3e4eba..ffdd0f51a 100644 --- a/src/modules/site-v2/templates/tools/pairwise_indel_finder/view.html +++ b/src/modules/site-v2/templates/tools/pairwise_indel_finder/view.html @@ -73,7 +73,7 @@

No Results

@@ -85,7 +85,7 @@

Download Results

Print to PDF CSV
diff --git a/src/modules/site-v2/templates/tools/phenotype_database/report.html b/src/modules/site-v2/templates/tools/phenotype_database/report.html index 2bca7bdc3..c594cf1f9 100644 --- a/src/modules/site-v2/templates/tools/phenotype_database/report.html +++ b/src/modules/site-v2/templates/tools/phenotype_database/report.html @@ -124,7 +124,7 @@

Description:

- Download TSV Print Report diff --git a/src/modules/site-v2/templates/tools/phenotype_database/submit-traits.html b/src/modules/site-v2/templates/tools/phenotype_database/submit-traits.html index 242050494..12ed7eb6c 100644 --- a/src/modules/site-v2/templates/tools/phenotype_database/submit-traits.html +++ b/src/modules/site-v2/templates/tools/phenotype_database/submit-traits.html @@ -244,7 +244,7 @@ submit_job(data, '#confirmationModal', {'auto_redirect': false}) .done((result) => { document.getElementById('goToReportButton').addEventListener('click', () => { - window.location.href = `{{ url_for('phenotype_database.report', id='') }}${ result.id }`; + window.location.href = `{{ url_for('phenotype_database.report', report_id='') }}${ result.id }`; }) }) // If validation fails, just flash the error diff --git a/src/modules/site-v2/templates/tools/variant_annotation/vbrowser.html b/src/modules/site-v2/templates/tools/variant_annotation/vbrowser.html index 71d36831c..7f81ed81d 100755 --- a/src/modules/site-v2/templates/tools/variant_annotation/vbrowser.html +++ b/src/modules/site-v2/templates/tools/variant_annotation/vbrowser.html @@ -869,7 +869,7 @@ // Setting up Release Notes link $('#nav-linkTwo-tab').click(function() { const species = $('#speciesSelect').val() - let url = "{{ url_for('data_releases.data_release_list', species='SPECIES') }}"; + let url = "{{ url_for('data_releases.data_release_list', species_name='SPECIES') }}"; if (species) { url = url.replace("SPECIES", species); } else { diff --git a/src/pkg/caendr/caendr/models/error.py b/src/pkg/caendr/caendr/models/error.py index b6d65ffea..36375c21a 100755 --- a/src/pkg/caendr/caendr/models/error.py +++ b/src/pkg/caendr/caendr/models/error.py @@ -258,10 +258,6 @@ def __init__(self, description=None, code=500): super().__init__() -class SpeciesUrlNameError(InternalError): - def __init__(self, species_name): - self.species_name = species_name - class InvalidTokenError(InternalError): def __init__(self, token, source): self.token = token diff --git a/src/pkg/caendr/caendr/models/report/bucketed_report.py b/src/pkg/caendr/caendr/models/report/bucketed_report.py index 72c1a88dd..6a20a70be 100644 --- a/src/pkg/caendr/caendr/models/report/bucketed_report.py +++ b/src/pkg/caendr/caendr/models/report/bucketed_report.py @@ -36,6 +36,11 @@ class BucketedReport(Report): def _generate_uri(cls, bucket: str, *path: str, schema: BlobURISchema=None): pass + @classmethod + @abstractmethod + def _list_files(cls, bucket: str, *prefix: str, filter=None): + pass + # @@ -119,7 +124,8 @@ def _output_prefix(self): # # Directory functions # These probably should not be overwritten, unless you're sure you know what you're doing. - # Instead, look into overwriting the bucket and prefix functions to customize directory lookups. + # Instead, look into overwriting the bucket functions, prefix functions, and _generate_uri + # to customize directory lookups. # def report_directory(self, *path, schema: BlobURISchema = None): @@ -136,3 +142,32 @@ def input_directory(self, *path, schema: BlobURISchema = None): def output_directory(self, *path, schema: BlobURISchema = None): return self.report_directory( self._output_prefix, *path, schema=schema ) + + + + # + # Directory listing functions + # Each returns the list of blobs in the given directory, with an optional extra path & filter. + # + # These probably should not be overwritten, unless you're sure you know what you're doing. + # Instead, look into overwriting the bucket functions, prefix functions, and _list_files + # to customize directory lookups. + # + + def _list_directory(self, f_dir, *path, filter=None): + return self._list_files( *f_dir(*path, schema=BlobURISchema.PATH), filter=filter ) + + def list_report_directory(self, *path, filter=None): + return self._list_directory( self.report_directory, *path, filter=filter ) + + def list_data_directory(self, *path, filter=None): + return self._list_directory( self.data_directory, *path, filter=filter ) + + def list_work_directory(self, *path, filter=None): + return self._list_directory( self.work_directory, *path, filter=filter ) + + def list_input_directory(self, *path, filter=None): + return self._list_directory( self.input_directory, *path, filter=filter ) + + def list_output_directory(self, *path, filter=None): + return self._list_directory( self.output_directory, *path, filter=filter ) diff --git a/src/pkg/caendr/caendr/models/report/gcp_report.py b/src/pkg/caendr/caendr/models/report/gcp_report.py index 5e0797f1a..cf66cea92 100644 --- a/src/pkg/caendr/caendr/models/report/gcp_report.py +++ b/src/pkg/caendr/caendr/models/report/gcp_report.py @@ -45,6 +45,10 @@ class GCPReport(BucketedReport): def _generate_uri(cls, bucket: str, *path: str, schema: BlobURISchema=None): return generate_blob_uri(bucket, *path, schema=schema) + @classmethod + def _list_files(cls, bucket: str, *prefix: str, filter=None): + return get_blob_list(bucket, *prefix, filter=filter) + # # Bucket names @@ -134,9 +138,6 @@ def fetch_input(self): def fetch_output(self): return get_blob_if_exists( *self.output_filepath(schema=BlobURISchema.PATH) ) - def list_output_blobs(self): - return get_blob_list( *self.output_directory(schema=BlobURISchema.PATH) ) - # diff --git a/src/pkg/caendr/caendr/services/cloud/storage.py b/src/pkg/caendr/caendr/services/cloud/storage.py index fecbce95f..3421016ad 100644 --- a/src/pkg/caendr/caendr/services/cloud/storage.py +++ b/src/pkg/caendr/caendr/services/cloud/storage.py @@ -74,13 +74,21 @@ def get_blob_if_exists(bucket_name: str, *path: str, fallback=None) -> Optional[ return fallback -def get_blob_list(bucket_name: str, *prefix: str) -> List[Blob]: +def get_blob_list(bucket_name: str, *prefix: str, filter=None) -> List[Blob]: ''' Returns a list of all blobs with `prefix` (directory) in `bucket_name`. If no `prefix` is provided (or all values are empty), lists all blobs in the bucket. ''' + + # Get all the blobs in the given bucket bucket = storageClient.get_bucket(bucket_name) items = bucket.list_blobs(prefix=join_path(*prefix)) + + # Apply the filter, if one was given + if filter is not None: + items = [ b for b in items if filter(b) ] + + # Return the items as a list return list(items)