diff --git a/src/modules/site-v2/application.py b/src/modules/site-v2/application.py index 32822875a..210700880 100755 --- a/src/modules/site-v2/application.py +++ b/src/modules/site-v2/application.py @@ -4,7 +4,7 @@ from datetime import datetime from caendr.services.logger import logger -from flask import Flask, render_template, request, redirect +from flask import Flask, render_template, request, redirect, flash, g, session from flask_wtf.csrf import CSRFProtect from flask_httpauth import HTTPBasicAuth from werkzeug.security import generate_password_hash, check_password_hash @@ -17,7 +17,7 @@ from caendr.models.error import BasicAuthError from caendr.services.cloud.postgresql import db, health_database_status -from base.utils.markdown import render_markdown, render_ext_markdown +from base.utils.markdown import render_markdown, render_ext_markdown, render_markdown_inline @@ -128,6 +128,7 @@ def create_app(config=config): configure_jinja(app) configure_ssl(app) + register_announcement_handlers(app) password_protect_site(app) # app.teardown_request(close_active_connections) @@ -274,7 +275,8 @@ def inject(): get_env=get_env, basename=os.path.basename, render_markdown=render_markdown, - render_ext_markdown=render_ext_markdown + render_ext_markdown=render_ext_markdown, + render_markdown_inline=render_markdown_inline, ) @app.context_processor @@ -309,6 +311,87 @@ def _jinja2_filter_species_italic(text): def _jinja2_filter_percent(n): return f'{round(n * 100, 2)}%' + @app.template_filter('markdown') + def _jinja2_filter_markdown(text): + return render_markdown_inline(text) + + +def register_announcement_handlers(app): + ''' + Add site-wide announcements to relevant requests. + ''' + from caendr.models.datastore import Announcement + + def can_show_announcements(request, response=None, check_session_flags=False) -> bool: + ''' + Helper function to determine whether a request should flash the active site-wide announcements. + + Announcements only possible for a non-redirecting GET request that returns HTML, excluding static files. + Additionally, specific endpoints can block announcements using the block_announcements decorator + or the block_announcements_from_bp function. + ''' + + # Request conditions: Must be a GET request to a non-static path + conditions = [ + request.endpoint != 'static', + request.method == 'GET', + ] + + # Response conditions: Must return HTML, and must not be redirecting + if response: + conditions += [ + response.mimetype == 'text/html', + not (response.status_code >= 300 and response.status_code < 400), + ] + + # Sessions flag(s) + if check_session_flags: + conditions += [ + not getattr(g, 'block_site_announcements', False), + ] + + # Check all conditions + return all(conditions) + + @app.context_processor + def inject_announcements(): + ''' + Before rendering any Jinja template, queue up and flash all the site-wide announcements + that apply to this URL. + + This has to happen before the template is rendered, otherwise announcements will be queued + for the *next* page load. + + This function is registered as a Jinja template context processor, so it will only be run + before a Jinja template is rendered. This prevents announcements from bleeding over to the + next request if e.g. a redirect or a static HTML file is returned instead. + ''' + + # Make sure this request can show announcements + # If not, we have to return an empty object, since this is a context processor + if not can_show_announcements(request, check_session_flags=True): + return {} + + # Make sure site announcements set exists + # Store as a set to help prevent flashing duplicate announcements + if not hasattr(g, 'site_announcements'): + g.site_announcements = set() + + # Queue up all the active announcements that apply to this path + for a in Announcement.query_ds(filters=[("active", "=", True)], deleted=False): + if a.matches_path(request.path): + g.site_announcements.add(a) + + # Flash all the announcements that apply to this path + # We have to do this before the template is rendered, otherwise they'll be queued for the *next* page + for a in g.site_announcements: + flash(a['content'], a['style'].get_bootstrap_color()) + + # Since this is a context processor, we have to return a dict of variables to add to the + # Jinja template rendering context + # In this case, we don't need to add any + return {} + def register_errorhandlers(app): def render_error(e="generic"): diff --git a/src/modules/site-v2/base/forms/forms.py b/src/modules/site-v2/base/forms/forms.py index c8988c1e7..79b37fd00 100755 --- a/src/modules/site-v2/base/forms/forms.py +++ b/src/modules/site-v2/base/forms/forms.py @@ -27,6 +27,7 @@ Optional, ValidationError) from wtforms.fields.html5 import EmailField +from wtforms.widgets import CheckboxInput from constants import PRICES, SECTOR_OPTIONS, SHIPPING_OPTIONS, PAYMENT_OPTIONS, TOOL_INPUT_DATA_VALID_FILE_EXTENSIONS, TRAIT_CATEGORY_OPTIONS @@ -37,6 +38,7 @@ from caendr.services.indel_primer import get_indel_primer_chrom_choices from caendr.services.markdown import get_content_type_form_options from caendr.models.datastore import User, Species, DatasetRelease, TraitFile +from caendr.models.datastore.announcement import AnnouncementType from caendr.api.strain import query_strains from base.forms.validators import (validate_duplicate_strain, validate_duplicate_isotype, @@ -119,6 +121,14 @@ class RecoverUserForm(FlaskForm): email = EmailField('Email Address', [Required(), Email(), Length(min=6, max=320)]) recaptcha = RecaptchaField() + +class AnnouncementForm(FlaskForm): + """ Edit form for site announcements """ + active = BooleanField('Active') + content = StringField('Content', [Optional()]) + url_list = TextAreaField('URL Patterns', [Optional()]) + style = SelectField('Style', [Optional()], choices=[(x.name, x.value) for x in AnnouncementType]) + class MarkdownForm(FlaskForm): """ markdown editing form """ _CONTENT_TYPES = get_content_type_form_options() diff --git a/src/modules/site-v2/base/utils/announcements.py b/src/modules/site-v2/base/utils/announcements.py new file mode 100644 index 000000000..723a18671 --- /dev/null +++ b/src/modules/site-v2/base/utils/announcements.py @@ -0,0 +1,27 @@ +from functools import wraps + +from flask import g + + + +def block_announcements(f): + ''' + Prevent an endpoint from flashing site-wide announcements. + ''' + @wraps(f) + def decorator(*args, **kwargs): + try: + g.block_site_announcements = True + except: + pass + return f(*args, **kwargs) + return decorator + + +def block_announcements_from_bp(bp): + ''' + Prevent all endpoints in the given blueprint from flashing site-wide announcements. + ''' + @bp.before_request + def _block_announcements(): + g.block_site_announcements = True diff --git a/src/modules/site-v2/base/utils/markdown.py b/src/modules/site-v2/base/utils/markdown.py index ec2ed0f58..b0cd689da 100644 --- a/src/modules/site-v2/base/utils/markdown.py +++ b/src/modules/site-v2/base/utils/markdown.py @@ -1,3 +1,4 @@ +import bleach import os import markdown import requests @@ -10,6 +11,11 @@ MODULE_SITE_BUCKET_PUBLIC_NAME = os.environ.get('MODULE_SITE_BUCKET_PUBLIC_NAME') + +def render_markdown_inline(content): + return Markup(markdown.markdown(bleach.clean(content))) + + def render_markdown(filename, directory="base/static/content/markdown"): path = os.path.join(directory, filename) if not os.path.exists(path): diff --git a/src/modules/site-v2/base/utils/view_decorators.py b/src/modules/site-v2/base/utils/view_decorators.py index 5174d26da..4e6f094ba 100644 --- a/src/modules/site-v2/base/utils/view_decorators.py +++ b/src/modules/site-v2/base/utils/view_decorators.py @@ -9,7 +9,7 @@ from base.utils.tools import lookup_report, get_upload_err_msg from constants import TOOL_INPUT_DATA_VALID_FILE_EXTENSIONS -from caendr.models.datastore import DatasetRelease, Species +from caendr.models.datastore import DatasetRelease, Species, Entity from caendr.models.error import NotFoundError, ReportLookupError, EmptyReportDataError, EmptyReportResultsError, FileUploadError, DataValidationError from caendr.models.job_pipeline import JobPipeline from caendr.services.logger import logger @@ -147,7 +147,54 @@ def decorator(*args, report_id, **kwargs): -def validate_form(form_class: Type[FlaskForm], from_json: bool = False, err_msg: str = None, flash_err_msg: bool = True): +def parse_entity_id(entity_class: Type[Entity], required: bool = True, kw_name_id: str = 'entity_id', kw_name_entity: str = 'entity'): + ''' + Given an entity ID as a keyword argument, lookup and inject the entity with that ID. + ''' + def wrapper(f): + @wraps(f) + def decorator(*args, **kwargs): + + # Extract the entity ID from the keywords using the given name + entity_id = kwargs.get(kw_name_id) + kwargs = { key: val for key, val in kwargs.items() if key != kw_name_id } + + # If no ID given, optionally raise error + if entity_id is None: + if required: + abort(404) + else: + e = None + + # If ID given, try retrieving entity from datastore + else: + try: + e = entity_class.get_ds(entity_id) + + # If not found, abort with 404 + except NotFoundError as ex: + logger.error(f'Could not find {entity_class.kind} with ID {entity_id}: {ex}') + abort(404, description = f'Could not find an {entity_class.kind} object with the given ID.') + + # General error: include default message + except Exception as ex: + logger.error(f'Error retrieving {entity_class.kind} with ID {entity_id}: {ex}') + abort(500, description = 'Something went wrong') + + # If entity does not exist, abort with 404 + if e is None: + logger.error(f'Could not find {entity_class.kind} with ID {entity_id}') + abort(404, description = f'Could not find an {entity_class.kind} object with the given ID.') + + # Inject retrieved entity into function call + return f(*args, **{kw_name_entity: e}, **kwargs) + + return decorator + return wrapper + + + +def validate_form(form_class: Type[FlaskForm], from_json: bool = False, err_msg: str = None, flash_err_msg: bool = True, methods = None): ''' Parse the request form into the given form type, validate the fields, and inject the data as a dict. @@ -164,6 +211,8 @@ def validate_form(form_class: Type[FlaskForm], from_json: bool = False, err_msg: - `from_json`: If `True`, use the request `.get_json()` as the fields instead. - `err_msg`: An error message to add to the response if validation fails. - `flash_err_msg`: If `True`, flashes the `err_msg` in addition to returning it. + - `methods`: A list of request methods to expect a form from. If `None`, checks for a form in all requests; + otherwise, does not try to extract a form from any request method not in the list. ''' def wrapper(f): @@ -181,6 +230,11 @@ def decorator(*args, **kwargs): # If user is admin, allow them to bypass cache with URL variable no_cache = bool(user_is_admin() and request.args.get("nocache", False)) + # If a list of methods is provided, make sure the current request method is in it + # If it's not, then there should not be any form data for this request + if methods is not None and request.method not in methods: + return f(*args, form_data=None, no_cache=no_cache, **kwargs) + # Pull the raw data from either the form or the JSON body raw_data = request.get_json() if from_json else request.form diff --git a/src/modules/site-v2/base/views/admin/admin.py b/src/modules/site-v2/base/views/admin/admin.py index d96dd04a7..03b123595 100755 --- a/src/modules/site-v2/base/views/admin/admin.py +++ b/src/modules/site-v2/base/views/admin/admin.py @@ -1,12 +1,17 @@ -from flask import (render_template, - Blueprint) +from flask import render_template, Blueprint, abort from config import config +from base.forms import AnnouncementForm from base.utils.auth import admin_required +from base.utils.view_decorators import parse_entity_id +from caendr.models.datastore import Announcement from caendr.services.cloud.secret import get_secret from caendr.services.cloud.sheets import GOOGLE_SHEET_PREFIX +from caendr.models.datastore.announcement import AnnouncementType + + ANDERSEN_LAB_STRAIN_SHEET = get_secret('ANDERSEN_LAB_STRAIN_SHEET') CENDR_PUBLICATIONS_SHEET = get_secret('CENDR_PUBLICATIONS_SHEET') @@ -36,3 +41,40 @@ def admin_publications_sheet(): title = "CaeNDR Publications Sheet" sheet_url = f"{GOOGLE_SHEET_PREFIX}/{CENDR_PUBLICATIONS_SHEET}" return render_template('admin/google_sheet.html', **locals()) + + +@admin_bp.route('/announcements', methods=['GET']) +@admin_required() +def announcements(): + ''' + Manage the site announcements. + ''' + return render_template('admin/announcements/list.html', **{ + 'title': 'Site Announcements', + 'form': AnnouncementForm(), + + 'AnnouncementType': AnnouncementType, + }) + + +@admin_bp.route('/announcements/create', methods=['GET']) +@admin_bp.route('/announcements/edit/', methods=['GET']) +@admin_required() +@parse_entity_id(Announcement, required=False, kw_name_id='entity_id', kw_name_entity='announcement') +def announcements_edit(announcement: Announcement = None): + ''' + Manage the site announcements. + ''' + + # Initialize the form with the existing object (or None if creating new) + form = AnnouncementForm(obj=announcement) + if announcement: + form.style.data = announcement['style'].name + + return render_template('admin/announcements/edit.html', **{ + 'title': ('Edit' if announcement else 'Create') + ' Announcement', + 'form': form, + + 'announcement': announcement, + 'AnnouncementType': AnnouncementType, + }) diff --git a/src/modules/site-v2/base/views/api/notifications.py b/src/modules/site-v2/base/views/api/notifications.py index f997818ad..50455ae2e 100644 --- a/src/modules/site-v2/base/views/api/notifications.py +++ b/src/modules/site-v2/base/views/api/notifications.py @@ -1,14 +1,20 @@ from flask import jsonify, Blueprint, url_for, abort, request -from base.utils.auth import access_token_required +from caendr.services.logger import logger + +from base.forms import AnnouncementForm +from base.utils.auth import access_token_required, admin_required from base.utils.tools import lookup_report from base.views.tools import pairwise_indel_finder_bp, genetic_mapping_bp, heritability_calculator_bp -from caendr.models.datastore import NemascanReport, HeritabilityReport, IndelPrimerReport -from caendr.models.error import ReportLookupError +from base.utils.view_decorators import parse_entity_id, validate_form + +from caendr.models.datastore import NemascanReport, HeritabilityReport, IndelPrimerReport, Announcement +from caendr.models.error import ReportLookupError, NotFoundError from caendr.models.status import JobStatus from caendr.services.email import REPORT_SUCCESS_EMAIL_TEMPLATE, REPORT_ERROR_EMAIL_TEMPLATE from caendr.services.cloud.secret import get_secret +from caendr.utils.json import jsonify_request API_SITE_ACCESS_TOKEN = get_secret('CAENDR_API_SITE_ACCESS_TOKEN') @@ -28,6 +34,12 @@ def notifications(): abort(404) + +# +# Job Status Notifications +# + + @api_notifications_bp.route('/job-finish///', methods=['GET']) @access_token_required(API_SITE_ACCESS_TOKEN) def job_finish(kind, id, status): @@ -70,3 +82,82 @@ def job_finish(kind, id, status): report_link = f'{link}', ), }) + + + +# +# Site Announcements +# + + +@api_notifications_bp.route('/announcements', methods=['GET']) +@admin_required() +@jsonify_request +def announcement_list(): + ''' + Get the list of all site announcements. + ''' + return { + 'data': [ e.serialize(include_name=True) for e in Announcement.query_ds(deleted=False) ] + } + + +@api_notifications_bp.route('/announcement', methods=['POST']) +@api_notifications_bp.route('/announcement/', methods=['GET', 'PATCH', 'DELETE']) +@admin_required() +@parse_entity_id(Announcement, required=False, kw_name_id='entity_id', kw_name_entity='announcement') +@validate_form(AnnouncementForm, methods=['POST', 'PATCH']) +@jsonify_request +def announcement(announcement: Announcement = None, form_data = None, no_cache: bool = False): + ''' + Manage one of the site announcements. + + Methods: + GET: Get the data for the given announcement ID. + POST: Create a new announcement. + PATCH: Update an existing announcement. + DELETE: Delete an existing announcement. + ''' + + # GET Request + # Return the list of announcement entities + if request.method == 'GET': + return announcement.serialize() + + # POST Request + # Create a new announcement, and return its unique ID + if request.method == 'POST': + new_announcement = Announcement(**{ + prop: form_data.get(prop) for prop in Announcement.get_props_set() + }) + new_announcement.save() + return { 'id': new_announcement.name } + + # PATCH Request + # Lookup the desired announcement and update its properties + if request.method == 'PATCH': + + # Extract the new property values from the form, casting "active" to a bool + new_values = { + prop: form_data.get(prop) + for prop in Announcement.get_props_set() + if form_data.get(prop) is not None + } + if form_data.get('active') is not None: + new_values['active'] = form_data.get('active') == 'true' + + # Update the announcement object + announcement.set_properties(**new_values) + announcement.save() + return { 'id': announcement.name } + + # DELETE Request + # Lookup the desired announcement and soft delete it + if request.method == 'DELETE': + announcement.soft_delete() + announcement.save() + return {}, 200 + + # If somehow the method didn't match any of the above, + # return a Method Not Allowed error + abort(405) diff --git a/src/modules/site-v2/base/views/api/trait.py b/src/modules/site-v2/base/views/api/trait.py index ccc3080eb..55a789b2c 100644 --- a/src/modules/site-v2/base/views/api/trait.py +++ b/src/modules/site-v2/base/views/api/trait.py @@ -1,9 +1,20 @@ -from flask import request, Blueprint, abort +import bleach +from enum import Enum +from functools import wraps +import json + +from flask import request, Blueprint, abort, jsonify from caendr.services.logger import logger -from extensions import cache +from extensions import cache, compress + +from base.utils.auth import jwt_required, get_current_user, user_is_admin -from caendr.models.datastore import TraitFile, Species +from caendr.api.phenotype import query_phenotype_metadata, get_trait, filter_trait_query, get_trait_categories +from caendr.services.cloud.postgresql import rollback_on_error_handler + +from caendr.models.datastore import Entity, TraitFile, Species, User from caendr.models.error import NotFoundError +from caendr.models.sql import PhenotypeMetadata from caendr.utils.json import jsonify_request @@ -13,27 +24,340 @@ +# +# Helper Classes +# + + +class EndpointType(Enum): + ''' + Enum class for trait API endpoint types. + Requires endpoints to have the format "{ prefix }_{ value }", where `value` is one of the enum values. + ''' + + PUBLIC = 'public' + PRIVATE = 'private' + ALL = 'all' + + @classmethod + def full(cls, endpoint_prefix, endpoint_type): + return f'{ api_trait_bp.name }.{ endpoint_prefix }_{ endpoint_type.value }' + + @classmethod + def matches(cls, endpoint, endpoint_type): + return endpoint.split('_')[-1] == endpoint_type.value + + @classmethod + def matches_any(cls, endpoint, endpoint_type_set): + return any( cls.matches(endpoint, endpoint_type) for endpoint_type in endpoint_type_set ) + + + +# +# Helper Functions +# + + def filter_trait_files(tf): return tf.is_public and not tf.is_bulk_file -@api_trait_bp.route('/all', methods=['GET']) +def get_clean(source, key, value=None, _type=None): + v = source.get(key, value) + + # Clean value + if isinstance(v, str): v = bleach.clean(v) + elif isinstance(v, list): v = [ bleach.clean(x) for x in v ] + + # Optional typecasting + if _type and v is not None: + if issubclass(_type, Entity): + v_entity = _type.get_ds(v) + if v_entity is None: + raise NotFoundError(_type, {'name': v}) + v = v_entity + else: + v = _type(v) + + return v + + +def query_traits_error_handler(err_msg): + ''' + Wrapper for trait query endpoints. + + If the wrapped function aborts with an error code, that code will be used + by the response. Otherwise, based on the error type, either a `404` or a + `500` will be returned. + ''' + + def decorator(f): + @wraps(f) + def inner(*args, **kwargs): + try: + return f(*args, **kwargs) + + # Error handling + except Exception as ex: + + # Choose error code based on error type + if hasattr(ex, 'code'): + err_code = ex.code + elif isinstance(ex, NotFoundError): + err_code = 404 + else: + err_code = 500 + + # Log the full error, and return the response with an abridged message + logger.error(f'{err_msg}: {ex}') + return {'message': f'{err_msg}'}, err_code + + return inner + return decorator + + +def validate_user(): + ''' + Validate that the requesting user can access the current endpoint. + Requires endpoint to be formatted according to `EndpointType` Enum. + ''' + + # On the "public" endpoint, no user validation required + if EndpointType.matches( request.endpoint, EndpointType.PUBLIC ): + return True + + # On the "private" endpoint, user must be logged in + elif EndpointType.matches( request.endpoint, EndpointType.PRIVATE ): + return get_current_user() is not None + + # On the "all" endpoint, user must be an admin + elif EndpointType.matches( request.endpoint, EndpointType.ALL ): + return user_is_admin() + + # If some other endpoint is being requested here somehow, abort + abort(404) + + +def validate_endpoint_type(endpoint_prefix): + ''' + Validate that the user has permission to access this endpoint, based on the `EndpointType` schema. + + See `validate_user` for details. + ''' + + def decorator(f): + @wraps(f) + def inner(*args, **kwargs): + + # Validate that the current user has access to the specific endpoint they're requesting + if not validate_user(): + abort(403) + + return f(*args, **kwargs) + + return inner + return decorator + + + +# +# Query Endpoints: List Traits +# + + +@api_trait_bp.route('/list/sql/public', endpoint='query_list_sql_public', methods=['POST']) +@api_trait_bp.route('/list/sql/private', endpoint='query_list_sql_private', methods=['POST']) +@api_trait_bp.route('/list/sql/all', endpoint='query_list_sql_all', methods=['POST']) @cache.memoize(60*60) +@jwt_required(optional=True) +@validate_endpoint_type('query_list_sql') +@query_traits_error_handler('Failed to retrieve the list of traits') @jsonify_request -def query_all(): +def query_list_sql(): ''' - Query all trait files, optionally split into different lists based on species. + Query the trait database using SQL-style pagination. + + Defines three separate endpoints: + - `public`: Queries traits that have been submitted to the public phenotype database. + - `private`: Queries all traits that belong to the requesting user. Log-in required. + - `all`: Queries all traits that have been uploaded. Admin users only. + + Accepts the following URL variables to filter the request: + - `dataset`: The phenotype dataset. + - `species`: The species for the trait. + - `user`: The user that submitted the trait. + + Note that on the `private` endpoint, filtering the `user` parameter by anything other than + the current user will return no results, since the two user filters are exclusive. + This is still a syntactically valid request, but it is semantically invalid. + ''' + + # On the "private" endpoint, only consider traits belonging to the current user + if EndpointType.matches( request.endpoint, EndpointType.PRIVATE ): + current_user_filter = get_current_user() + else: + current_user_filter = None + + # On the "private" and "all" endpoints, include private (unpublished) traits in the query + include_private_traits = EndpointType.matches_any( request.endpoint, { EndpointType.PRIVATE, EndpointType.ALL } ) + + # Get search parameters + selected_tags = get_clean(request.json, 'selected_tags', []) + search_val = get_clean(request.json, 'search_val', '').lower() + + # Get other query filters + try: + filter_dataset = get_clean(request.args, 'dataset') + filter_species = get_clean(request.args, 'species', _type=Species) + filter_user = get_clean(request.args, 'user', _type=User) + except NotFoundError as ex: + abort(422, description=ex.description) + + # Get query pagination values + page = get_clean(request.json, 'page', 1, _type=int) + current_page = get_clean(request.json, 'current_page', 1, _type=int) + per_page = 10 + + # Create the initial query + # The filters here determine which traits are part of the "full" query, + # based on the request endpoint + query = query_phenotype_metadata( + include_private=include_private_traits, + dataset=filter_dataset, user=current_user_filter, + ) + + # Filter by search values, if provided + query = filter_trait_query( + query, search_val=search_val, tags=selected_tags, species=filter_species, user=filter_user, + ) + + # Paginate the query, rolling back on error + with rollback_on_error_handler(): + pagination = query.paginate(page=page, per_page=per_page) + + # Format return data + return { + 'data': [ + tr.to_json() for tr in pagination.items + ], + 'pagination': { + 'has_next': pagination.has_next, + 'has_prev': pagination.has_prev, + 'prev_num': pagination.prev_num, + 'next_num': pagination.next_num, + 'total_pages': pagination.pages, + 'current_page': current_page + }, + } + + + +@api_trait_bp.route('/list/datatable/public', endpoint='query_list_datatable_public', methods=['GET']) +@api_trait_bp.route('/list/datatable/private', endpoint='query_list_datatable_private', methods=['GET']) +@api_trait_bp.route('/list/datatable/all', endpoint='query_list_datatable_all', methods=['GET']) +@cache.memoize(60*60) +@jwt_required(optional=True) +@validate_endpoint_type('query_list_datatable') +@query_traits_error_handler('Failed to retrieve the list of traits') +@jsonify_request +def query_list_datatable(): ''' + Query the trait database using DataTable-style pagination. + + Defines three separate endpoints: + - `public`: Queries traits that have been submitted to the public phenotype database. + - `private`: Queries all traits that belong to the requesting user. Log-in required. + - `all`: Queries all traits that have been uploaded. Admin users only. + + Accepts the following URL variables to filter the request: + - `dataset`: The phenotype dataset. + - `species`: The species for the trait. + - `user`: The user that submitted the trait. + + Note that on the `private` endpoint, filtering the `user` parameter by anything other than + the current user will return no results, since the two user filters are exclusive. + This is still a syntactically valid request, but it is semantically invalid. + ''' + + # On the private endpoint, only consider traits belonging to the current user + if EndpointType.matches( request.endpoint, EndpointType.PRIVATE ): + current_user_filter = get_current_user() + else: + current_user_filter = None + + # On the "private" and "all" endpoints, include private (unpublished) traits in the query + include_private_traits = EndpointType.matches_any( request.endpoint, { EndpointType.PRIVATE, EndpointType.ALL } ) + + # Load full search object from request + search_raw = get_clean(request.args, 'search[value]', '') + if search_raw: + try: + search_full = json.loads(search_raw) + except: + abort(422, description="Invalid search") + + # Treat non-dict values as search strings + # Use the original raw string here so JSON casting doesn't change the value + # (e.g. JSON "true" becoming Python "True") + if not isinstance(search_full, dict): + search_full = { 'search_val': search_raw } + + else: + search_full = {} - # Optionally split into an object w species names for keys - if request.args.get('split_by_species', False): - return { - species: [ tf.serialize() for tf in tf_list ] - for species, tf_list in TraitFile.query_ds_split_species(filter=filter_trait_files).items() - } + # Get search parameters from search object + search_value = get_clean(search_full, 'search_val', '').lower() + selected_tags = get_clean(search_full, 'selected_tags', []) + + # Get other query filters + try: + filter_dataset = get_clean(request.args, 'dataset') + filter_species = get_clean(request.args, 'species', _type=Species) + filter_user = get_clean(request.args, 'user', _type=User) + except NotFoundError as ex: + abort(422, description=ex.description) + + # Get query pagination values + draw = get_clean(request.args, 'draw', _type=int) + start = get_clean(request.args, 'start', _type=int) + length = get_clean(request.args, 'length', _type=int) + + # Create the initial query + # The filters here determine which traits are part of the "full" query, + # based on the request endpoint + query = query_phenotype_metadata( + include_private=include_private_traits, + dataset=filter_dataset, user=current_user_filter, + ) + + # Count the full size of the query + # Any rows filtered out before this line aren't available in this request at all, + # and any rows filtered out after this line are considered filtered rows in the full request + total_records = query.count() + + # Filter by search values, if provided + query = filter_trait_query( + query, search_val=search_value, tags=selected_tags, species=filter_species, user=filter_user, + ) + + # Query PhenotypeMetadata (include phenotype values for each trait) + with rollback_on_error_handler(): + data = query.offset(start).limit(length).from_self().\ + join(PhenotypeMetadata.phenotype_values).all() + + # Count how many rows matched the filters + filtered_records = query.count() + + # Format return data + return { + 'data': [ + trait.to_json_with_values() for trait in data + ], + 'draw': draw, + 'recordsTotal': total_records, + 'recordsFiltered': filtered_records, + } - # Otherwise, return all trait files in one list - return [ tf.serialize() for tf in TraitFile.query_ds(ignore_errs=True) if filter_trait_files(tf) ] @api_trait_bp.route('/', methods=['GET']) @@ -56,3 +380,51 @@ def query_species(species_name): for tf in TraitFile.query_ds(ignore_errs=True, filters=['species', '=', species.name]) if filter_trait_files(tf) ] + + + +# +# Query Endpoints: Single Trait +# + + +@api_trait_bp.route('/metadata', methods=['POST']) +@cache.memoize(60*60) +@compress.compressed() +@query_traits_error_handler('Failed to retrieve trait metadata') +@jsonify_request +def query_trait_metadata(): + """ + Get traits data for non-bulk files in JSON format (include phenotype values) + """ + + # Get the trait name from the request + trait_id = get_clean(request.json, 'trait_id') + if not trait_id: + abort(400, description='No trait ID provided.') + + # Try getting the trait from the database + trait = get_trait(trait_id) + if trait is None: + abort(404, description=f'Invalid trait ID {trait_id}') + + # Return the full trait metadata + return trait.to_json_with_values() + + + +# +# Query Endpoint: Trait Categories +# + + +@api_trait_bp.route('/categories', methods=['GET']) +@cache.memoize(60*60) +@compress.compressed() +@query_traits_error_handler('Failed to retrieve trait categories') +@jsonify_request +def query_trait_categories(): + """ + Get list of trait categories. + """ + return get_trait_categories() diff --git a/src/modules/site-v2/base/views/auth/auth.py b/src/modules/site-v2/base/views/auth/auth.py index 6b176dbdc..353c28bcc 100644 --- a/src/modules/site-v2/base/views/auth/auth.py +++ b/src/modules/site-v2/base/views/auth/auth.py @@ -23,10 +23,16 @@ from caendr.services.cloud.secret import get_secret from base.views.auth.oauth import transfer_cart +from base.utils.announcements import block_announcements_from_bp + + PASSWORD_PEPPER = get_secret('PASSWORD_PEPPER') auth_bp = Blueprint('auth', __name__, template_folder='templates') +block_announcements_from_bp(auth_bp) + + @auth_bp.route('/') def auth(): return redirect(url_for('auth.choose_login')) diff --git a/src/modules/site-v2/base/views/tools/genetic_mapping.py b/src/modules/site-v2/base/views/tools/genetic_mapping.py index 65bb78351..de9805233 100755 --- a/src/modules/site-v2/base/views/tools/genetic_mapping.py +++ b/src/modules/site-v2/base/views/tools/genetic_mapping.py @@ -5,6 +5,7 @@ from flask import jsonify from base.forms import MappingForm +from base.utils.announcements import block_announcements from base.utils.auth import get_jwt, jwt_required, admin_required, get_current_user, user_is_admin from base.utils.tools import list_reports, try_submit from base.utils.view_decorators import parse_job_id, validate_form @@ -145,6 +146,7 @@ def report(job: NemascanPipeline): @genetic_mapping_bp.route('/report//fullscreen', methods=['GET']) @jwt_required() +@block_announcements @parse_job_id(NemascanPipeline, fetch=False) def report_fullscreen(job: NemascanPipeline): diff --git a/src/modules/site-v2/base/views/tools/phenotype_database.py b/src/modules/site-v2/base/views/tools/phenotype_database.py index e162dd68b..66ac16073 100644 --- a/src/modules/site-v2/base/views/tools/phenotype_database.py +++ b/src/modules/site-v2/base/views/tools/phenotype_database.py @@ -12,7 +12,7 @@ from extensions import cache, compress from sqlalchemy import or_, func -from caendr.api.phenotype import query_phenotype_metadata, get_trait, filter_trait_query_by_text, filter_trait_query_by_tags +from caendr.api.phenotype import query_phenotype_metadata, get_trait_categories from caendr.services.cloud.postgresql import rollback_on_error_handler from caendr.services.logger import logger @@ -56,42 +56,11 @@ def phenotype_database(): Phenotype Database table (non-bulk) """ form = EmptyForm() - - # Get the list of traits for non-bulk files + + # Get the list of unique tags try: - query = query_phenotype_metadata() - - # Get the list of unique tags - tags = [ tr.tags.split(', ') for tr in query if tr.tags ] - tags_list = [tg for tr_tag in tags for tg in tr_tag] - unique_tags = list(set(tags_list)) - - if request.method == 'POST': - selected_tags = request.json.get('selected_tags', []) - search_val = request.json.get('search_val', '') - page = int(request.json.get('page', 1)) - current_page = int(request.json.get('current_page', 1)) - per_page = 10 - - # Filter by search value and tags, if provided - query = filter_trait_query_by_text(query, search_val) - query = filter_trait_query_by_tags(query, selected_tags) - - # Paginate the query, rolling back on error - with rollback_on_error_handler(): - pagination = query.paginate(page=page, per_page=per_page) - - json_data = [ tr.to_json() for tr in pagination.items ] - pagination_data = { - 'has_next': pagination.has_next, - 'has_prev': pagination.has_prev, - 'prev_num': pagination.prev_num, - 'next_num': pagination.next_num, - 'total_pages': pagination.pages, - 'current_page': current_page - } - return jsonify({'data': json_data, 'pagination': pagination_data }) - + categories = get_trait_categories() + except Exception as ex: logger.error(f'Failed to retrieve the list of traits: {ex}') abort(500, description='Failed to retrieve the list of traits') @@ -102,73 +71,11 @@ def phenotype_database(): "tool_alt_parent_breadcrumb": { "title": "Tools", "url": url_for('tools.tools') }, # Data - 'categories': unique_tags, + 'categories': categories, 'form': form }) -@phenotype_database_bp.route('/traits-zhang') -@cache.memoize(60*60) -@compress.compressed() -def get_zhang_traits_json(): - """ - Phenotype Database table (bulk) - Fetch table content by request for each page and render the data for Datatables() - """ - try: - # get parameters for query - draw = request.args.get('draw', type=int) - start = request.args.get('start', type=int) - length = request.args.get('length', type=int) - search_value = bleach.clean(request.args.get('search[value]', '')).lower() - - query = query_phenotype_metadata(is_bulk_file=True) - total_records = query.count() - - # Filter by search value, if provided - query = filter_trait_query_by_text(query, search_value) - - # Query PhenotypeMetadata (include phenotype values for each trait) - with rollback_on_error_handler(): - data = query.offset(start).limit(length).from_self().\ - join(PhenotypeMetadata.phenotype_values).all() - - json_data = [ trait.to_json_with_values() for trait in data ] - - filtered_records = query.count() - - response_data = { - "draw": draw, - "recordsTotal": total_records, - "recordsFiltered": filtered_records, - "data": json_data - } - - except Exception as ex: - logger.error(f'Failed to retrieve the list of traits: {ex}') - response_data = [] - return jsonify(response_data) - -@phenotype_database_bp.route('/traits-list', methods=['POST']) -@cache.memoize(60*60) -@compress.compressed() -def get_traits_json(): - """ - Get traits data for non-bulk files in JSON format (include phenotype values) - """ - trait_id = request.json.get('trait_id') - err_msg = f'Failed to retrieve metadata for trait {trait_id}' - - if trait_id: - try: - trait = get_trait(trait_id).to_json_with_values() - return jsonify(trait) - - except Exception as ex: - logger.error(f'{err_msg}: {ex}') - - return jsonify({ 'message': err_msg }), 404 - # # Submission Flow @@ -191,14 +98,13 @@ def submit_traits(): # Check for URL vars specifying an initial trait # These will be inherited from submit_start - initial_trait_name = request.args.get('trait') - initial_trait_set = request.args.get('dataset') + initial_trait_id = request.args.get('trait') # Try looking up the specified trait - if initial_trait_name: + if initial_trait_id: try: - initial_trait = Trait.from_id(initial_trait_name) - except NotFoundError: + initial_trait = Trait.from_id(initial_trait_id) + except (NotFoundError, ValueError): flash('That trait could not be found.', 'danger') initial_trait = None else: diff --git a/src/modules/site-v2/templates/_includes/alert.html b/src/modules/site-v2/templates/_includes/alert.html index cf5d26a91..c87510fef 100644 --- a/src/modules/site-v2/templates/_includes/alert.html +++ b/src/modules/site-v2/templates/_includes/alert.html @@ -1,4 +1,17 @@ -
-

{{ msg }}

+{% if not dismissable is defined %} + {% set dismissable = true %} +{% endif %} + +
+
+ {%- if msg is defined %} + {{ msg | markdown }} + {%- endif %} +
+ {%- if dismissable %} + {%- endif %}
\ No newline at end of file diff --git a/src/modules/site-v2/templates/_includes/head.html b/src/modules/site-v2/templates/_includes/head.html index df9f30d32..aa040cea5 100755 --- a/src/modules/site-v2/templates/_includes/head.html +++ b/src/modules/site-v2/templates/_includes/head.html @@ -60,6 +60,10 @@ + + + + diff --git a/src/modules/site-v2/templates/_includes/macros.html b/src/modules/site-v2/templates/_includes/macros.html index 3a8e96837..15a3e1f24 100755 --- a/src/modules/site-v2/templates/_includes/macros.html +++ b/src/modules/site-v2/templates/_includes/macros.html @@ -45,6 +45,8 @@ {% if form_prefix %}{{ form_prefix }}{% endif %} {% if field.type == 'RadioField' %} {{ field(**kwargs) }} + {% elif field.type == 'BooleanField' %} + {{ field(**kwargs) }} {% elif field.type == 'FileField' %}
{{ field(class_="form-control", required=kwargs.get('required', field.flags.required), **kwargs) }} diff --git a/src/modules/site-v2/templates/_includes/phenotype-database-table.html b/src/modules/site-v2/templates/_includes/phenotype-database-table.html index 9822e049c..7220be637 100644 --- a/src/modules/site-v2/templates/_includes/phenotype-database-table.html +++ b/src/modules/site-v2/templates/_includes/phenotype-database-table.html @@ -21,6 +21,10 @@ {% endblock %} + +{% set offcanvas_id = 'phenotype-offcanvas' %} + +
+ +
+
{{ render_field(form.active, class_='form-check-input d-block') }}
+
{{ render_field(form.style) }}
+
{{ render_field(form.url_list) }}
+ + +
+
+ + + +{% endblock %} + + +{% block script %} + + + + + + +{% endblock %} + diff --git a/src/modules/site-v2/templates/admin/announcements/list.html b/src/modules/site-v2/templates/admin/announcements/list.html new file mode 100644 index 000000000..313cf30b5 --- /dev/null +++ b/src/modules/site-v2/templates/admin/announcements/list.html @@ -0,0 +1,226 @@ +{% extends "_layouts/default.html" %} + +{% block content %} +
+
+
+ New Announcement +
+
+ +
+
+
+ + + + + + + + + + + + + + +
ActiveTextURLsStyleEditDelete
+
+
+
+ + +{% endblock %} + + + +{% block script %} + +{% endblock %} diff --git a/src/modules/site-v2/templates/tools/genetic_mapping/mapping.html b/src/modules/site-v2/templates/tools/genetic_mapping/mapping.html index d37f9dfd0..faa4ccab8 100644 --- a/src/modules/site-v2/templates/tools/genetic_mapping/mapping.html +++ b/src/modules/site-v2/templates/tools/genetic_mapping/mapping.html @@ -133,7 +133,7 @@

Example Data

{% from "_scripts/submit-job.js" import def_submit_job %} {% from "_includes/macros.html" import update_species_field %} diff --git a/src/pkg/caendr/caendr/api/phenotype.py b/src/pkg/caendr/caendr/api/phenotype.py index ba9d06643..1b4d22538 100644 --- a/src/pkg/caendr/caendr/api/phenotype.py +++ b/src/pkg/caendr/caendr/api/phenotype.py @@ -1,59 +1,124 @@ import bleach import os +from typing import Optional, Union, Iterable from sqlalchemy import or_, func -from caendr.models.datastore import Species -from caendr.models.error import BadRequestError +from caendr.models.datastore import Species, User from caendr.models.sql import PhenotypeMetadata from caendr.services.cloud.postgresql import rollback_on_error -def query_phenotype_metadata(is_bulk_file=False, include_values=False, species: str = None): + +def query_phenotype_metadata( + include_values: bool = False, + include_private: bool = False, + is_bulk_file: Optional[bool] = None, + dataset: Optional[str] = None, + species: Optional[Union[Species, str]] = None, + user: Optional[Union[User, str]] = None, +): """ - Returns the list of traits with the corresponding metadata. + Create a trait metadata SQL query, with some optional initial filters. + Returns a query object that can be further refined. Args: - - is_bulk_file: by default returns data for non-bulk files - if 'is_bulk_file' set to True returns traits metadata for Zhang Expression file - - phenotype_values: if True, include phenotype values for each trait - - species: filters by species + - `include_values`: If `True`, includes the phenotype value measurements for each trait. + - `include_private`: If `True`, includes private (unpublished) traits in the query. + - `is_bulk_file`: Optionally filter by whether the traits belong to a bulk dataset. + If `None`, does not filter by this parameter. + - `dataset`: Optionally filters by dataset. + - `species`: Optionally filters by species. + - `user`: Optionally filters by submitting user. """ + + # Create the initial query query = PhenotypeMetadata.query - # Get traits for bulk file - if is_bulk_file: - query = query.filter_by(is_bulk_file=True) - else: - # Get traits for non-bulk files - query = query.filter_by(is_bulk_file=False) - - # Query by species - if species is not None: - if species in Species.all().keys(): - query = query.filter_by(species_name=species) - else: - raise BadRequestError(f'Unrecognized species ID "{species}".') - + # TODO: Filter by public (published) / private (unpublished) + if not include_private: + pass + + # Optionally query by bulk file + if is_bulk_file is not None: + query = query.filter_by(is_bulk_file=bool(is_bulk_file)) + + # Optionally query by dataset + if dataset is not None: + query = query.filter_by(dataset=dataset) + + # Optionally query by species + # None values handled in function + query = filter_trait_query_by_species(query, species) + query = filter_trait_query_by_user(query, user) + # Include phenotype values for traits if include_values: - query = query.join(PhenotypeMetadata.phenotype_values) + query = query.join(PhenotypeMetadata.phenotype_values) + + # Order alphabetically by trait names + query = order_trait_query_by_name(query) return query def get_all_traits_metadata(): - """ - Returns metadata for all traits - """ - return PhenotypeMetadata.query.all() + """ + Returns metadata for all traits + """ + return PhenotypeMetadata.query.all() def get_trait(trait_id): - return PhenotypeMetadata.query.get(trait_id) + return PhenotypeMetadata.query.get(trait_id) + + +def order_trait_query_by_name(query): + ''' + Sort a Phenotype Database trait query alphabetically by the display name(s). + ''' + return query.order_by( + + # Order by display names, in order + PhenotypeMetadata.trait_name_display_1.asc(), + PhenotypeMetadata.trait_name_display_2.asc(), + PhenotypeMetadata.trait_name_display_3.asc(), + + # Fallback to internal CaeNDR trait name + PhenotypeMetadata.trait_name_caendr.asc(), + ) + + + +# +# Query Filters +# +# Conditionally add common filter types to a query object +# + + +def filter_trait_query( + query, + search_val: Optional[str] = None, + tags: Optional[Iterable[str]] = None, + species: Optional[Union[Species, str]] = None, + user: Optional[Union[User, str]] = None, + ): + ''' + Combined filtering function. + ''' + query = filter_trait_query_by_text(query, search_val) + query = filter_trait_query_by_tags(query, tags) + query = filter_trait_query_by_species(query, species) + query = filter_trait_query_by_user(query, user) + return query -def filter_trait_query_by_text(query, search_val): +def filter_trait_query_by_text(query, search_val: Optional[str]): + ''' + Filter by a text search value on the text fields. + Generic "search" functionality. + ''' print(search_val) if search_val and len(search_val): query = query.filter( @@ -70,9 +135,89 @@ def filter_trait_query_by_text(query, search_val): return query -def filter_trait_query_by_tags(query, tags): - if len(tags): +def filter_trait_query_by_tags(query, tags: Optional[Iterable[str]]): + ''' + Filter by trait tags. + ''' + if tags and len(tags): query = query.filter(or_( PhenotypeMetadata.tags.ilike(f"%{bleach.clean(tag)}%") for tag in tags )) return query + + +def filter_trait_query_by_user(query, user: Optional[Union[User, str]]): + ''' + Filter by submitting user. + + If username is invalid, passes error raised by `User` class. + ''' + if user: + + # Cast string value to User object using unique datastore ID + if isinstance(user, str): + user = User.get_ds(user) + + # Validate user type + if not isinstance(user, User): + raise ValueError(f'Expected user, got {user}') + + # Filter by the username + query = query.filter_by(submitted_by=user.full_name) + + return query + + +def filter_trait_query_by_species(query, species: Optional[Union[Species, str]]): + ''' + Filter by species. + + If species is invalid, passes error raised by `Species` class. + ''' + if species is not None: + + # Cast string value to Species object + if isinstance(species, str): + species = Species.from_name(species) + + # Validate species type + if not isinstance(species, Species): + raise ValueError(f'Expected species identifier, got {species}') + + # Filter by the species name + query = query.filter_by(species_name=species.name) + + # Return the (possibly filtered) query + return query + + + +# +# Trait Categories +# + + +@rollback_on_error +def get_trait_categories(query = None): + ''' + Get the list of trait categories. + + If a query is provided, only returns categories represented in that query. + Otherwise, creates a new query. + + TODO: Currently, uses PhenotypeMetadata table queries to get the full list + of categories, as strings. In the future we may want to explicitly + store this list somewhere and attach metadata. + ''' + # If no query given, default to full-database search + # TODO: We can speed this up a lot by excluding Zhang traits... what is the most data-agnostic way to do that? + if query is None: + query = query_phenotype_metadata() + + # Parse the list of tags from each row + # `filter` with None removes all non-truthy values, i.e. empty tag sets + tags = filter(None, ( tr.get_tags() for tr in query )) + + # Flatten the list of lists into a set, and sort the result + tags_list = { tg for tr_tag in tags for tg in tr_tag } + return sorted(tags_list) diff --git a/src/pkg/caendr/caendr/models/datastore/__init__.py b/src/pkg/caendr/caendr/models/datastore/__init__.py index aebc57a4d..fc15dface 100644 --- a/src/pkg/caendr/caendr/models/datastore/__init__.py +++ b/src/pkg/caendr/caendr/models/datastore/__init__.py @@ -38,6 +38,7 @@ from .phenotype_report import PhenotypeReport # Subclasses ReportEntity, HashableEntity; imports TraitFile # Other +from .announcement import Announcement from .profile import Profile from .markdown import Markdown from .cart import Cart # Subclasses DeletableEntity @@ -69,6 +70,7 @@ def get_class_by_kind(kind): HeritabilityReport.kind: HeritabilityReport, NemascanReport.kind: NemascanReport, + Announcement.kind: Announcement, GeneBrowserTracks.kind: GeneBrowserTracks, Markdown.kind: Markdown, Species.kind: Species, diff --git a/src/pkg/caendr/caendr/models/datastore/announcement.py b/src/pkg/caendr/caendr/models/datastore/announcement.py new file mode 100644 index 000000000..5f8fce6db --- /dev/null +++ b/src/pkg/caendr/caendr/models/datastore/announcement.py @@ -0,0 +1,187 @@ +import bleach +from enum import Enum +import markdown +import re + +from flask import Markup + +from caendr.models.datastore import DeletableEntity +from caendr.utils.data import unique_id + + + +class AnnouncementType(Enum): + GENERAL = 'General' + MAINTENANCE = 'Maintenance' + ERROR = 'Error' + + def get_bootstrap_color(self): + __BOOTSTRAP_COLOR_MAP = { + AnnouncementType.GENERAL: 'success', + AnnouncementType.MAINTENANCE: 'warning', + AnnouncementType.ERROR: 'danger', + } + return __BOOTSTRAP_COLOR_MAP[self] + + + + +class Announcement(DeletableEntity): + kind = 'announcement' + + exclude_from_indexes = ('content', 'url_list') + + + def __init__(self, name_or_obj = None, *args, **kwargs): + + # If nothing passed for name_or_obj, create a new ID to use for this object + if name_or_obj is None: + name_or_obj = unique_id() + self.set_properties_meta(id = name_or_obj) + + # Initialize from superclass + super().__init__(name_or_obj, *args, **kwargs) + + + + # + # Props + # + + @classmethod + def get_props_set(cls): + return { + *super().get_props_set(), + 'active', + 'content', + 'url_list', + 'style', + } + + + @property + def active(self): + return self._get_raw_prop('active', False) + + @active.setter + def active(self, val): + self._set_raw_prop('active', bool(val)) + + + @property + def content(self): + return bleach.clean( self._get_raw_prop('content', '') ) + + @content.setter + def content(self, val): + if not isinstance(val, str): + raise ValueError(f'Prop "content" must be a string, got: {val}') + return self._set_raw_prop( 'content', bleach.clean( val ) ) + + + @property + def style(self): + return self._get_enum_prop(AnnouncementType, 'style', None) + + @style.setter + def style(self, val): + return self._set_enum_prop(AnnouncementType, 'style', val) + + + @property + def url_list(self): + return self._get_raw_prop('url_list', '') + + @url_list.setter + def url_list(self, val): + ''' + Validate the list of URL patterns before saving. + ''' + + # Accept newline-separated string or iterable of patterns + # Convert to a list for validating + if isinstance(val, str): + val = [ pattern.strip() for pattern in val.split('\n') ] + if not isinstance(val, list): + try: + val = [*val] + except: + raise ValueError('URL list must be an iterable or a newline-separated string') + + # Validate each pattern + for idx, path in enumerate(val): + if not self.validate_path(path): + raise ValueError(f'Invalid pattern on line {idx}: "{path}"') + + # If validation succeeded, convert to newline-separated string and save + return self._set_raw_prop('url_list', '\n'.join(val)) + + + + # + # Extra Props + # + + @property + def content_html(self): + ''' + The content of this announcement as HTML. + ''' + return Markup(markdown.markdown( self['content'] )) + + + def serialize(self, **kwargs): + props = super().serialize(**kwargs) + props['content_html'] = self.content_html + return props + + + + # + # URL Paths + # + + @classmethod + def validate_path(cls, path: str) -> bool: + ''' + Each path must meet the following constraints: + - The first character is a forward slash + - If there is an asterisk (wildcard), it must: + - be the final character + - be preceded by a forward slash + + Sample valid paths: + / + /* + /foo/bar/baz/* + + Sample invalid paths: + foo/bar + /foo* + /foo*/bar + ''' + # Validate the above rules using Regex: + # ^\/ First character is a forward slash '/' + # [^\*]* String of any length that doesn't include a '*' character + # ((?<=\/)\*)?$ Optionally, last character may be '*', and if so previous character must be '/' + return bool(re.match('^\/[^\*]*((?<=\/)\*)?$', path)) + + + def matches_path(self, path: str) -> bool: + + # Loop through each line in the URL list + for pattern in self['url_list'].split('\n'): + + # Turn our wildcard into a Regex wildcard, if given + if pattern.endswith('/*'): + pattern = pattern[:-2] + suffix = '(/.*)?' + else: + suffix = '' + + # Check this pattern + if re.match(f'^{ pattern }{ suffix }$', path): + return True + + # If no patterns matched, path does not match + return False diff --git a/src/pkg/caendr/caendr/models/datastore/deletable_entity.py b/src/pkg/caendr/caendr/models/datastore/deletable_entity.py index 5e545b351..eb4562727 100644 --- a/src/pkg/caendr/caendr/models/datastore/deletable_entity.py +++ b/src/pkg/caendr/caendr/models/datastore/deletable_entity.py @@ -30,7 +30,18 @@ def get_props_set(cls): *super().get_props_set(), 'is_deleted' } - + + + + @classmethod + def query_ds(cls, *args, deleted=None, **kwargs): + + if deleted is None: + return super().query_ds(*args, **kwargs) + + return [ + match for match in super().query_ds(*args, **kwargs) if (match['is_deleted'] == deleted) + ] @classmethod diff --git a/src/pkg/caendr/caendr/models/datastore/entity.py b/src/pkg/caendr/caendr/models/datastore/entity.py index 71e1cad48..a3fe0b43a 100644 --- a/src/pkg/caendr/caendr/models/datastore/entity.py +++ b/src/pkg/caendr/caendr/models/datastore/entity.py @@ -235,7 +235,7 @@ def __iter__(self): return ( (k, self[k]) for k in props if self[k] is not None ) - def serialize(self, include_meta=True): + def serialize(self, include_meta=True, include_name=False): ''' Get a `dict` of all props in the entity, mapped to serializable values. @@ -252,6 +252,10 @@ def serialize(self, include_meta=True): prop: getattr(self, prop) for prop in self.get_props_set_meta() }) + # Add unique ID ("name"), if applicable + if include_name: + props['name'] = self.name + # Map non-serializable values to raw strings # TODO: Can we pull this out somewhere? # E.g. assigning "types" to props and handling automatically, and/or using _get/_set_raw_prop? diff --git a/src/pkg/caendr/caendr/models/sql/phenotype_metadata.py b/src/pkg/caendr/caendr/models/sql/phenotype_metadata.py index ead869172..ebf413300 100644 --- a/src/pkg/caendr/caendr/models/sql/phenotype_metadata.py +++ b/src/pkg/caendr/caendr/models/sql/phenotype_metadata.py @@ -40,6 +40,8 @@ class PhenotypeMetadata(DictSerializable, db.Model): __tablename__ = 'phenotype_metadata' + + def to_json_with_values(self): """ Converts PhenotypeMetadata instance to JSON in the joined queries @@ -48,8 +50,17 @@ def to_json_with_values(self): phenotype_values = [ v.to_json() for v in self.phenotype_values ] json_trait['phenotype_values'] = phenotype_values return json_trait - - + + + def get_tags(self): + ''' + Get the category tags as a set of strings. + If no tag set defined, returns None. + ''' + if self.tags: + return { tg.strip() for tg in self.tags.split(',') } + + def add(self, trait_obj): new_trait = PhenotypeMetadata( id = trait_obj.name, @@ -78,4 +89,4 @@ def add(self, trait_obj): def delete(self): db.session.delete(self) - db.session.commit() \ No newline at end of file + db.session.commit() diff --git a/src/pkg/caendr/caendr/models/trait.py b/src/pkg/caendr/caendr/models/trait.py index 17ad9e008..a1ca207e1 100644 --- a/src/pkg/caendr/caendr/models/trait.py +++ b/src/pkg/caendr/caendr/models/trait.py @@ -70,6 +70,24 @@ def __init__(self, dataset = None, trait_name = None, trait_file_id = None, trai # Constructors # + @staticmethod + def from_id(trait_id: str) -> 'Trait': + ''' + Instantiate a `Trait` object from a unique trait ID. + The given ID must exist in the PhenotypeMetadata SQL table, otherwise a `ValueError` will be raised. + ''' + + # Get the SQL row with the given trait ID + sql_row = PhenotypeMetadata.query.get(trait_id) + if sql_row is None: + raise ValueError(f'Invalid trait ID {trait_id}') + + # Construct a Trait object using the data in the SQL row + return Trait( + trait_name = sql_row.trait_name_caendr, + dataset = sql_row.dataset, + ) + @staticmethod def from_dataset(dataset: str, trait_name: Optional[str] = None) -> 'Trait': return Trait( dataset = dataset, trait_name = trait_name )