From 1dc03b4ab05408e55d313568b70f03522a2736ec Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Thu, 4 Apr 2024 13:45:36 -0500 Subject: [PATCH 01/73] Move trait querying to API --- src/modules/site-v2/base/views/api/trait.py | 46 +++++++++++++++---- .../base/views/tools/phenotype_database.py | 30 +----------- .../_includes/phenotype-database-table.html | 2 +- 3 files changed, 38 insertions(+), 40 deletions(-) diff --git a/src/modules/site-v2/base/views/api/trait.py b/src/modules/site-v2/base/views/api/trait.py index ccc3080eb..b043f7510 100644 --- a/src/modules/site-v2/base/views/api/trait.py +++ b/src/modules/site-v2/base/views/api/trait.py @@ -2,6 +2,9 @@ from caendr.services.logger import logger from extensions import cache +from caendr.api.phenotype import query_phenotype_metadata, get_trait, filter_trait_query_by_text, filter_trait_query_by_tags +from caendr.services.cloud.postgresql import rollback_on_error_handler + from caendr.models.datastore import TraitFile, Species from caendr.models.error import NotFoundError from caendr.utils.json import jsonify_request @@ -17,23 +20,46 @@ def filter_trait_files(tf): return tf.is_public and not tf.is_bulk_file -@api_trait_bp.route('/all', methods=['GET']) +@api_trait_bp.route('/query', methods=['POST']) @cache.memoize(60*60) @jsonify_request -def query_all(): +def query(): ''' Query all trait files, optionally split into different lists based on species. ''' - # Optionally split into an object w species names for keys - if request.args.get('split_by_species', False): - return { - species: [ tf.serialize() for tf in tf_list ] - for species, tf_list in TraitFile.query_ds_split_species(filter=filter_trait_files).items() - } + # Get query filters + selected_tags = request.json.get('selected_tags', []) + search_val = request.json.get('search_val', '') + + # Get query pagination values + page = int(request.json.get('page', 1)) + current_page = int(request.json.get('current_page', 1)) + per_page = 10 + + # Filter by search value and tags, if provided + query = query_phenotype_metadata() + query = filter_trait_query_by_text(query, search_val) + query = filter_trait_query_by_tags(query, selected_tags) + + # Paginate the query, rolling back on error + with rollback_on_error_handler(): + pagination = query.paginate(page=page, per_page=per_page) - # Otherwise, return all trait files in one list - return [ tf.serialize() for tf in TraitFile.query_ds(ignore_errs=True) if filter_trait_files(tf) ] + # Format return data + return { + 'data': [ + tr.to_json() for tr in pagination.items + ], + 'pagination': { + 'has_next': pagination.has_next, + 'has_prev': pagination.has_prev, + 'prev_num': pagination.prev_num, + 'next_num': pagination.next_num, + 'total_pages': pagination.pages, + 'current_page': current_page + }, + } @api_trait_bp.route('/', methods=['GET']) diff --git a/src/modules/site-v2/base/views/tools/phenotype_database.py b/src/modules/site-v2/base/views/tools/phenotype_database.py index cead747ad..f47b8a967 100644 --- a/src/modules/site-v2/base/views/tools/phenotype_database.py +++ b/src/modules/site-v2/base/views/tools/phenotype_database.py @@ -79,40 +79,12 @@ def phenotype_database(): """ form = EmptyForm() - # Get the list of traits for non-bulk files + # Get the list of unique tags try: query = query_phenotype_metadata() - - # Get the list of unique tags tags = [ tr.tags.split(', ') for tr in query if tr.tags ] tags_list = [tg for tr_tag in tags for tg in tr_tag] unique_tags = list(set(tags_list)) - - if request.method == 'POST': - selected_tags = request.json.get('selected_tags', []) - search_val = request.json.get('search_val', '') - page = int(request.json.get('page', 1)) - current_page = int(request.json.get('current_page', 1)) - per_page = 10 - - # Filter by search value and tags, if provided - query = filter_trait_query_by_text(query, search_val) - query = filter_trait_query_by_tags(query, selected_tags) - - # Paginate the query, rolling back on error - with rollback_on_error_handler(): - pagination = query.paginate(page=page, per_page=per_page) - - json_data = [ tr.to_json() for tr in pagination.items ] - pagination_data = { - 'has_next': pagination.has_next, - 'has_prev': pagination.has_prev, - 'prev_num': pagination.prev_num, - 'next_num': pagination.next_num, - 'total_pages': pagination.pages, - 'current_page': current_page - } - return jsonify({'data': json_data, 'pagination': pagination_data }) except Exception as ex: logger.error(f'Failed to retrieve the list of traits: {ex}') diff --git a/src/modules/site-v2/templates/_includes/phenotype-database-table.html b/src/modules/site-v2/templates/_includes/phenotype-database-table.html index cefbe1512..60db7023c 100644 --- a/src/modules/site-v2/templates/_includes/phenotype-database-table.html +++ b/src/modules/site-v2/templates/_includes/phenotype-database-table.html @@ -280,7 +280,7 @@

} async function sendPostRequest(data) { - const resp = await fetch("{{ url_for('phenotype_database.phenotype_database') }}", { + const resp = await fetch("{{ url_for('api_trait.query') }}", { method: "POST", headers: { "Content-Type": "application/json", From 29f4c825a029bb9151648e5ff8a94b045fffbad9 Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Thu, 4 Apr 2024 16:12:29 -0500 Subject: [PATCH 02/73] Filter non-Zhang traits by "dataset" rather than bulk_file directly --- src/modules/site-v2/base/views/api/trait.py | 7 +++-- .../_includes/phenotype-database-table.html | 2 +- src/pkg/caendr/caendr/api/phenotype.py | 27 ++++++++++++------- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/modules/site-v2/base/views/api/trait.py b/src/modules/site-v2/base/views/api/trait.py index b043f7510..424369694 100644 --- a/src/modules/site-v2/base/views/api/trait.py +++ b/src/modules/site-v2/base/views/api/trait.py @@ -31,14 +31,17 @@ def query(): # Get query filters selected_tags = request.json.get('selected_tags', []) search_val = request.json.get('search_val', '') + filter_dataset = request.json.get('dataset', None) # Get query pagination values page = int(request.json.get('page', 1)) current_page = int(request.json.get('current_page', 1)) per_page = 10 - # Filter by search value and tags, if provided - query = query_phenotype_metadata() + # Create the initial query + query = query_phenotype_metadata(dataset=filter_dataset) + + # Filter by search values, if provided query = filter_trait_query_by_text(query, search_val) query = filter_trait_query_by_tags(query, selected_tags) diff --git a/src/modules/site-v2/templates/_includes/phenotype-database-table.html b/src/modules/site-v2/templates/_includes/phenotype-database-table.html index 60db7023c..12559812d 100644 --- a/src/modules/site-v2/templates/_includes/phenotype-database-table.html +++ b/src/modules/site-v2/templates/_includes/phenotype-database-table.html @@ -286,7 +286,7 @@

"Content-Type": "application/json", 'X-CSRF-TOKEN': data.csrf_token }, - body: JSON.stringify(data) + body: JSON.stringify({...data, 'dataset': 'caendr'}), }) if (resp.ok) { diff --git a/src/pkg/caendr/caendr/api/phenotype.py b/src/pkg/caendr/caendr/api/phenotype.py index dd766d488..71e2069fa 100644 --- a/src/pkg/caendr/caendr/api/phenotype.py +++ b/src/pkg/caendr/caendr/api/phenotype.py @@ -1,5 +1,6 @@ import bleach import os +from typing import Optional, Union from sqlalchemy import or_, func @@ -9,7 +10,12 @@ from caendr.services.cloud.postgresql import rollback_on_error -def query_phenotype_metadata(is_bulk_file=False, include_values=False, species: str = None): +def query_phenotype_metadata( + include_values = False, + is_bulk_file: Optional[bool] = None, + species: Optional[str] = None, + dataset: Optional[str] = None, +): """ Returns the list of traits with the corresponding metadata. @@ -19,16 +25,19 @@ def query_phenotype_metadata(is_bulk_file=False, include_values=False, species: - phenotype_values: if True, include phenotype values for each trait - species: filters by species """ + + # Create the initial query query = PhenotypeMetadata.query - # Get traits for bulk file - if is_bulk_file: - query = query.filter_by(is_bulk_file=True) - else: - # Get traits for non-bulk files - query = query.filter_by(is_bulk_file=False) + # Optionally query by bulk file + if is_bulk_file is not None: + query = query.filter_by(is_bulk_file=bool(is_bulk_file)) + + # Optionally query by dataset + if dataset is not None: + query = query.filter_by(dataset=dataset) - # Query by species + # Optionally query by species if species is not None: if species in Species.all().keys(): query = query.filter_by(species_name=species) @@ -37,7 +46,7 @@ def query_phenotype_metadata(is_bulk_file=False, include_values=False, species: # Include phenotype values for traits if include_values: - query = query.join(PhenotypeMetadata.phenotype_values) + query = query.join(PhenotypeMetadata.phenotype_values) return query From 33fe4fdc53987fb5072602b25982f3475956294a Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Thu, 4 Apr 2024 16:16:43 -0500 Subject: [PATCH 03/73] Move species filter to helper func w better validation --- src/pkg/caendr/caendr/api/phenotype.py | 33 ++++++++++++++++++++------ 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/src/pkg/caendr/caendr/api/phenotype.py b/src/pkg/caendr/caendr/api/phenotype.py index 71e2069fa..68dba2eaa 100644 --- a/src/pkg/caendr/caendr/api/phenotype.py +++ b/src/pkg/caendr/caendr/api/phenotype.py @@ -5,7 +5,6 @@ from sqlalchemy import or_, func from caendr.models.datastore import Species -from caendr.models.error import BadRequestError from caendr.models.sql import PhenotypeMetadata from caendr.services.cloud.postgresql import rollback_on_error @@ -13,7 +12,7 @@ def query_phenotype_metadata( include_values = False, is_bulk_file: Optional[bool] = None, - species: Optional[str] = None, + species: Optional[Union[Species, str]] = None, dataset: Optional[str] = None, ): """ @@ -38,11 +37,8 @@ def query_phenotype_metadata( query = query.filter_by(dataset=dataset) # Optionally query by species - if species is not None: - if species in Species.all().keys(): - query = query.filter_by(species_name=species) - else: - raise BadRequestError(f'Unrecognized species ID "{species}".') + # None values handled in function + filter_trait_query_by_species(query, species) # Include phenotype values for traits if include_values: @@ -85,3 +81,26 @@ def filter_trait_query_by_tags(query, tags): PhenotypeMetadata.tags.ilike(f"%{bleach.clean(tag)}%") for tag in tags )) return query + + +def filter_trait_query_by_species(query, species: Optional[Union[Species, str]]): + ''' + Filter by species. + + If species is invalid, passes error raised by `Species` class. + ''' + if species is not None: + + # Cast string values to Species object + if isinstance(species, str): + species = Species.from_name(species) + + # Validate species type + if not isinstance(species, Species): + raise ValueError(f'Expected species identifier, got {species}') + + # Filter by the species name + query = query.filter_by(species_name=species.name) + + # Return the (possibly filtered) query + return query From c344052b42c2e6f7b830b6f08169516f1e9e5245 Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Thu, 4 Apr 2024 16:23:11 -0500 Subject: [PATCH 04/73] Add section & docstring comments, typing --- src/modules/site-v2/base/views/api/trait.py | 13 ++++++++++++- src/pkg/caendr/caendr/api/phenotype.py | 21 ++++++++++++++++++--- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/modules/site-v2/base/views/api/trait.py b/src/modules/site-v2/base/views/api/trait.py index 424369694..8d0344bc8 100644 --- a/src/modules/site-v2/base/views/api/trait.py +++ b/src/modules/site-v2/base/views/api/trait.py @@ -16,10 +16,21 @@ +# +# Helper Functions +# + + def filter_trait_files(tf): return tf.is_public and not tf.is_bulk_file + +# +# Query Endpoints +# + + @api_trait_bp.route('/query', methods=['POST']) @cache.memoize(60*60) @jsonify_request @@ -28,7 +39,7 @@ def query(): Query all trait files, optionally split into different lists based on species. ''' - # Get query filters + # Get query filters (search parameters) selected_tags = request.json.get('selected_tags', []) search_val = request.json.get('search_val', '') filter_dataset = request.json.get('dataset', None) diff --git a/src/pkg/caendr/caendr/api/phenotype.py b/src/pkg/caendr/caendr/api/phenotype.py index 68dba2eaa..5f14c1077 100644 --- a/src/pkg/caendr/caendr/api/phenotype.py +++ b/src/pkg/caendr/caendr/api/phenotype.py @@ -1,6 +1,6 @@ import bleach import os -from typing import Optional, Union +from typing import Optional, Union, Iterable from sqlalchemy import or_, func @@ -58,7 +58,19 @@ def get_trait(trait_name): return PhenotypeMetadata.query.get(trait_name) -def filter_trait_query_by_text(query, search_val): + +# +# Query Filters +# +# Conditionally add common filter types to a query object +# + + +def filter_trait_query_by_text(query, search_val: Optional[str]): + ''' + Filter by a text search value on the text fields. + Generic "search" functionality. + ''' print(search_val) if search_val and len(search_val): query = query.filter( @@ -75,7 +87,10 @@ def filter_trait_query_by_text(query, search_val): return query -def filter_trait_query_by_tags(query, tags): +def filter_trait_query_by_tags(query, tags: Optional[Iterable[str]]): + ''' + Filter by trait tags. + ''' if len(tags): query = query.filter(or_( PhenotypeMetadata.tags.ilike(f"%{bleach.clean(tag)}%") for tag in tags From 4a7eab060e7a3b7a49c84f2910031008dec2a228 Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Fri, 5 Apr 2024 13:23:03 -0500 Subject: [PATCH 05/73] Use bleach to clean request params --- src/modules/site-v2/base/views/api/trait.py | 25 ++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/modules/site-v2/base/views/api/trait.py b/src/modules/site-v2/base/views/api/trait.py index 8d0344bc8..03a8165d5 100644 --- a/src/modules/site-v2/base/views/api/trait.py +++ b/src/modules/site-v2/base/views/api/trait.py @@ -1,3 +1,5 @@ +import bleach + from flask import request, Blueprint, abort from caendr.services.logger import logger from extensions import cache @@ -25,6 +27,19 @@ def filter_trait_files(tf): return tf.is_public and not tf.is_bulk_file +def get_clean(source, key, value=None, _type=None): + v = source.get(key, value) + + # Clean value + if isinstance(v, str): v = bleach.clean(v) + elif isinstance(v, list): v = [ bleach.clean(x) for x in v ] + + # Optional typecasting + if _type: v = _type(v) + + return v + + # # Query Endpoints @@ -40,13 +55,13 @@ def query(): ''' # Get query filters (search parameters) - selected_tags = request.json.get('selected_tags', []) - search_val = request.json.get('search_val', '') - filter_dataset = request.json.get('dataset', None) + selected_tags = get_clean(request.json, 'selected_tags', []) + search_val = get_clean(request.json, 'search_val', '').lower() + filter_dataset = get_clean(request.json, 'dataset') # Get query pagination values - page = int(request.json.get('page', 1)) - current_page = int(request.json.get('current_page', 1)) + page = get_clean(request.json, 'page', 1, _type=int) + current_page = get_clean(request.json, 'current_page', 1, _type=int) per_page = 10 # Create the initial query From 18f8acc824ddf50823526edae275e49f2fc150c8 Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Fri, 5 Apr 2024 14:28:39 -0500 Subject: [PATCH 06/73] Minor query filtering bugfixes - Actually add the species filter to the `query` - Make sure `tags` is defined before trying to take length --- src/pkg/caendr/caendr/api/phenotype.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pkg/caendr/caendr/api/phenotype.py b/src/pkg/caendr/caendr/api/phenotype.py index 5f14c1077..83d256d5c 100644 --- a/src/pkg/caendr/caendr/api/phenotype.py +++ b/src/pkg/caendr/caendr/api/phenotype.py @@ -38,7 +38,7 @@ def query_phenotype_metadata( # Optionally query by species # None values handled in function - filter_trait_query_by_species(query, species) + query = filter_trait_query_by_species(query, species) # Include phenotype values for traits if include_values: @@ -91,7 +91,7 @@ def filter_trait_query_by_tags(query, tags: Optional[Iterable[str]]): ''' Filter by trait tags. ''' - if len(tags): + if tags and len(tags): query = query.filter(or_( PhenotypeMetadata.tags.ilike(f"%{bleach.clean(tag)}%") for tag in tags )) From cdc252aca1312b5aad8e149b96c74608802a68cc Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Fri, 5 Apr 2024 14:31:11 -0500 Subject: [PATCH 07/73] Convenience func to apply multiple query filters at once --- src/modules/site-v2/base/views/api/trait.py | 5 ++--- src/pkg/caendr/caendr/api/phenotype.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/modules/site-v2/base/views/api/trait.py b/src/modules/site-v2/base/views/api/trait.py index 03a8165d5..7b3d82ca6 100644 --- a/src/modules/site-v2/base/views/api/trait.py +++ b/src/modules/site-v2/base/views/api/trait.py @@ -4,7 +4,7 @@ from caendr.services.logger import logger from extensions import cache -from caendr.api.phenotype import query_phenotype_metadata, get_trait, filter_trait_query_by_text, filter_trait_query_by_tags +from caendr.api.phenotype import query_phenotype_metadata, get_trait, filter_trait_query from caendr.services.cloud.postgresql import rollback_on_error_handler from caendr.models.datastore import TraitFile, Species @@ -68,8 +68,7 @@ def query(): query = query_phenotype_metadata(dataset=filter_dataset) # Filter by search values, if provided - query = filter_trait_query_by_text(query, search_val) - query = filter_trait_query_by_tags(query, selected_tags) + query = filter_trait_query(query, search_val=search_val, tags=selected_tags) # Paginate the query, rolling back on error with rollback_on_error_handler(): diff --git a/src/pkg/caendr/caendr/api/phenotype.py b/src/pkg/caendr/caendr/api/phenotype.py index 83d256d5c..627e2ab02 100644 --- a/src/pkg/caendr/caendr/api/phenotype.py +++ b/src/pkg/caendr/caendr/api/phenotype.py @@ -66,6 +66,21 @@ def get_trait(trait_name): # +def filter_trait_query( + query, + search_val: Optional[str] = None, + tags: Optional[Iterable[str]] = None, + species: Optional[Union[Species, str]] = None, + ): + ''' + Combined filtering function. + ''' + query = filter_trait_query_by_text(query, search_val) + query = filter_trait_query_by_tags(query, tags) + query = filter_trait_query_by_species(query, species) + return query + + def filter_trait_query_by_text(query, search_val: Optional[str]): ''' Filter by a text search value on the text fields. From 705b19b527ef765494fb206ccef1e3d18dd6527e Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Fri, 5 Apr 2024 14:38:15 -0500 Subject: [PATCH 08/73] Return better errors from API endpoint --- src/modules/site-v2/base/views/api/trait.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/modules/site-v2/base/views/api/trait.py b/src/modules/site-v2/base/views/api/trait.py index 7b3d82ca6..cfbda87e3 100644 --- a/src/modules/site-v2/base/views/api/trait.py +++ b/src/modules/site-v2/base/views/api/trait.py @@ -1,4 +1,5 @@ import bleach +from functools import wraps from flask import request, Blueprint, abort from caendr.services.logger import logger @@ -40,6 +41,21 @@ def get_clean(source, key, value=None, _type=None): return v +def query_traits_error_handler(f): + ''' + Wrapper for trait query endpoints. + If query raises an error, returns an empty response and a `500` error. + ''' + @wraps(f) + def inner(*args, **kwargs): + try: + return f(*args, **kwargs) + except Exception as ex: + logger.error(f'Failed to retrieve the list of traits: {ex}') + return {}, 500 + return inner + + # # Query Endpoints @@ -48,6 +64,7 @@ def get_clean(source, key, value=None, _type=None): @api_trait_bp.route('/query', methods=['POST']) @cache.memoize(60*60) +@query_traits_error_handler @jsonify_request def query(): ''' From 9c30fbb21e097e082466d545e087158eef5a61a6 Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Fri, 5 Apr 2024 14:43:42 -0500 Subject: [PATCH 09/73] Adapt Zhang traits endpoint to API query Instead of splitting on dataset ('caendr' and 'zhang'), there are now two endpoints for different return formats: SQL-style pagination and JS DataTable-style pagination. Both endpoints accept a `dataset` URL var that filters the query to that dataset. --- src/modules/site-v2/base/views/api/trait.py | 55 +++++++++++++++++-- .../base/views/tools/phenotype_database.py | 42 -------------- .../_includes/phenotype-database-table.html | 6 +- 3 files changed, 54 insertions(+), 49 deletions(-) diff --git a/src/modules/site-v2/base/views/api/trait.py b/src/modules/site-v2/base/views/api/trait.py index cfbda87e3..3a4b8f1d1 100644 --- a/src/modules/site-v2/base/views/api/trait.py +++ b/src/modules/site-v2/base/views/api/trait.py @@ -10,6 +10,7 @@ from caendr.models.datastore import TraitFile, Species from caendr.models.error import NotFoundError +from caendr.models.sql import PhenotypeMetadata from caendr.utils.json import jsonify_request @@ -62,19 +63,19 @@ def inner(*args, **kwargs): # -@api_trait_bp.route('/query', methods=['POST']) +@api_trait_bp.route('/query/sql', methods=['POST']) @cache.memoize(60*60) @query_traits_error_handler @jsonify_request -def query(): +def query_sql(): ''' - Query all trait files, optionally split into different lists based on species. + Query the trait database, and return results with SQL-style pagination. ''' # Get query filters (search parameters) selected_tags = get_clean(request.json, 'selected_tags', []) search_val = get_clean(request.json, 'search_val', '').lower() - filter_dataset = get_clean(request.json, 'dataset') + filter_dataset = get_clean(request.args, 'dataset') # Get query pagination values page = get_clean(request.json, 'page', 1, _type=int) @@ -107,6 +108,52 @@ def query(): } + +@api_trait_bp.route('/query/datatable', methods=['GET']) +@cache.memoize(60*60) +@query_traits_error_handler +@jsonify_request +def query_datatable(): + ''' + Query the trait database, and return results with DataTable-style pagination. + ''' + + # Get query filters (search parameters) + search_value = get_clean(request.args, 'search[value]', '').lower() + filter_dataset = get_clean(request.args, 'dataset') + + # Get query pagination values + draw = get_clean(request.args, 'draw', _type=int) + start = get_clean(request.args, 'start', _type=int) + length = get_clean(request.args, 'length', _type=int) + + # Create the initial query + query = query_phenotype_metadata(dataset=filter_dataset) + total_records = query.count() + + # Filter by search values, if provided + query = filter_trait_query(query, search_val=search_value) + + # Query PhenotypeMetadata (include phenotype values for each trait) + with rollback_on_error_handler(): + data = query.offset(start).limit(length).from_self().\ + join(PhenotypeMetadata.phenotype_values).all() + + # Count how many rows matched the filters + filtered_records = query.count() + + # Format return data + return { + 'data': [ + trait.to_json_with_values() for trait in data + ], + 'draw': draw, + 'recordsTotal': total_records, + 'recordsFiltered': filtered_records, + } + + + @api_trait_bp.route('/', methods=['GET']) @cache.memoize(60*60) @jsonify_request diff --git a/src/modules/site-v2/base/views/tools/phenotype_database.py b/src/modules/site-v2/base/views/tools/phenotype_database.py index f47b8a967..ea476c904 100644 --- a/src/modules/site-v2/base/views/tools/phenotype_database.py +++ b/src/modules/site-v2/base/views/tools/phenotype_database.py @@ -101,48 +101,6 @@ def phenotype_database(): }) -@phenotype_database_bp.route('/traits-zhang') -@cache.memoize(60*60) -@compress.compressed() -def get_zhang_traits_json(): - """ - Phenotype Database table (bulk) - Fetch table content by request for each page and render the data for Datatables() - """ - try: - # get parameters for query - draw = request.args.get('draw', type=int) - start = request.args.get('start', type=int) - length = request.args.get('length', type=int) - search_value = bleach.clean(request.args.get('search[value]', '')).lower() - - query = query_phenotype_metadata(is_bulk_file=True) - total_records = query.count() - - # Filter by search value, if provided - query = filter_trait_query_by_text(query, search_value) - - # Query PhenotypeMetadata (include phenotype values for each trait) - with rollback_on_error_handler(): - data = query.offset(start).limit(length).from_self().\ - join(PhenotypeMetadata.phenotype_values).all() - - json_data = [ trait.to_json_with_values() for trait in data ] - - filtered_records = query.count() - - response_data = { - "draw": draw, - "recordsTotal": total_records, - "recordsFiltered": filtered_records, - "data": json_data - } - - except Exception as ex: - logger.error(f'Failed to retrieve the list of traits: {ex}') - response_data = [] - return jsonify(response_data) - @phenotype_database_bp.route('/traits-list', methods=['POST']) @cache.memoize(60*60) @compress.compressed() diff --git a/src/modules/site-v2/templates/_includes/phenotype-database-table.html b/src/modules/site-v2/templates/_includes/phenotype-database-table.html index 12559812d..892832423 100644 --- a/src/modules/site-v2/templates/_includes/phenotype-database-table.html +++ b/src/modules/site-v2/templates/_includes/phenotype-database-table.html @@ -133,7 +133,7 @@

const zhangTable = $('#zhangTraits').DataTable( { "processing": true, "serverSide": true, - "ajax": "/tools/phenotype-database/traits-zhang", + "ajax": "{{ url_for('api_trait.query_datatable', dataset='zhang') }}", "ordering": false, "columns": [ { @@ -280,13 +280,13 @@

} async function sendPostRequest(data) { - const resp = await fetch("{{ url_for('api_trait.query') }}", { + const resp = await fetch("{{ url_for('api_trait.query_sql', dataset='caendr') }}", { method: "POST", headers: { "Content-Type": "application/json", 'X-CSRF-TOKEN': data.csrf_token }, - body: JSON.stringify({...data, 'dataset': 'caendr'}), + body: JSON.stringify({...data}), }) if (resp.ok) { From ca9cdbb0dec4cf016ea26310ebdc6c791fd5364c Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Fri, 5 Apr 2024 15:28:03 -0500 Subject: [PATCH 10/73] Accept species & submitting user filters in trait query endpoints --- src/modules/site-v2/base/views/api/trait.py | 8 ++++-- src/pkg/caendr/caendr/api/phenotype.py | 32 +++++++++++++++++++-- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/src/modules/site-v2/base/views/api/trait.py b/src/modules/site-v2/base/views/api/trait.py index 3a4b8f1d1..e57dcc5d5 100644 --- a/src/modules/site-v2/base/views/api/trait.py +++ b/src/modules/site-v2/base/views/api/trait.py @@ -76,6 +76,8 @@ def query_sql(): selected_tags = get_clean(request.json, 'selected_tags', []) search_val = get_clean(request.json, 'search_val', '').lower() filter_dataset = get_clean(request.args, 'dataset') + filter_user = get_clean(request.json, 'user') + filter_species = get_clean(request.json, 'species') # Get query pagination values page = get_clean(request.json, 'page', 1, _type=int) @@ -83,7 +85,7 @@ def query_sql(): per_page = 10 # Create the initial query - query = query_phenotype_metadata(dataset=filter_dataset) + query = query_phenotype_metadata(dataset=filter_dataset, species=filter_species, user=filter_user) # Filter by search values, if provided query = filter_trait_query(query, search_val=search_val, tags=selected_tags) @@ -121,6 +123,8 @@ def query_datatable(): # Get query filters (search parameters) search_value = get_clean(request.args, 'search[value]', '').lower() filter_dataset = get_clean(request.args, 'dataset') + filter_user = get_clean(request.args, 'user') + filter_species = get_clean(request.args, 'species') # Get query pagination values draw = get_clean(request.args, 'draw', _type=int) @@ -128,7 +132,7 @@ def query_datatable(): length = get_clean(request.args, 'length', _type=int) # Create the initial query - query = query_phenotype_metadata(dataset=filter_dataset) + query = query_phenotype_metadata(dataset=filter_dataset, species=filter_species, user=filter_user) total_records = query.count() # Filter by search values, if provided diff --git a/src/pkg/caendr/caendr/api/phenotype.py b/src/pkg/caendr/caendr/api/phenotype.py index 627e2ab02..058c788fc 100644 --- a/src/pkg/caendr/caendr/api/phenotype.py +++ b/src/pkg/caendr/caendr/api/phenotype.py @@ -4,7 +4,7 @@ from sqlalchemy import or_, func -from caendr.models.datastore import Species +from caendr.models.datastore import Species, User from caendr.models.sql import PhenotypeMetadata from caendr.services.cloud.postgresql import rollback_on_error @@ -12,8 +12,9 @@ def query_phenotype_metadata( include_values = False, is_bulk_file: Optional[bool] = None, - species: Optional[Union[Species, str]] = None, dataset: Optional[str] = None, + species: Optional[Union[Species, str]] = None, + user: Optional[Union[User, str]] = None, ): """ Returns the list of traits with the corresponding metadata. @@ -39,6 +40,7 @@ def query_phenotype_metadata( # Optionally query by species # None values handled in function query = filter_trait_query_by_species(query, species) + query = filter_trait_query_by_user(query, user) # Include phenotype values for traits if include_values: @@ -71,6 +73,7 @@ def filter_trait_query( search_val: Optional[str] = None, tags: Optional[Iterable[str]] = None, species: Optional[Union[Species, str]] = None, + user: Optional[Union[User, str]] = None, ): ''' Combined filtering function. @@ -78,6 +81,7 @@ def filter_trait_query( query = filter_trait_query_by_text(query, search_val) query = filter_trait_query_by_tags(query, tags) query = filter_trait_query_by_species(query, species) + query = filter_trait_query_by_user(query, user) return query @@ -113,6 +117,28 @@ def filter_trait_query_by_tags(query, tags: Optional[Iterable[str]]): return query +def filter_trait_query_by_user(query, user: Optional[Union[User, str]]): + ''' + Filter by submitting user. + + If username is invalid, passes error raised by `User` class. + ''' + if user: + + # Cast string value to User object using unique datastore ID + if isinstance(user, str): + user = User.get_ds(user) + + # Validate user type + if not isinstance(user, User): + raise ValueError(f'Expected user, got {user}') + + # Filter by the username + query = query.filter_by(submitted_by=user.full_name) + + return query + + def filter_trait_query_by_species(query, species: Optional[Union[Species, str]]): ''' Filter by species. @@ -121,7 +147,7 @@ def filter_trait_query_by_species(query, species: Optional[Union[Species, str]]) ''' if species is not None: - # Cast string values to Species object + # Cast string value to Species object if isinstance(species, str): species = Species.from_name(species) From b732d8d7876db02431ce52b64c0b4568f8211c7d Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Fri, 5 Apr 2024 15:37:58 -0500 Subject: [PATCH 11/73] Move trait metadata query to traits API --- src/modules/site-v2/base/views/api/trait.py | 31 +++++++++++++++++-- .../base/views/tools/phenotype_database.py | 20 ------------ .../_includes/phenotype-database-table.html | 2 +- 3 files changed, 30 insertions(+), 23 deletions(-) diff --git a/src/modules/site-v2/base/views/api/trait.py b/src/modules/site-v2/base/views/api/trait.py index e57dcc5d5..8731c53fc 100644 --- a/src/modules/site-v2/base/views/api/trait.py +++ b/src/modules/site-v2/base/views/api/trait.py @@ -1,9 +1,9 @@ import bleach from functools import wraps -from flask import request, Blueprint, abort +from flask import request, Blueprint, abort, jsonify from caendr.services.logger import logger -from extensions import cache +from extensions import cache, compress from caendr.api.phenotype import query_phenotype_metadata, get_trait, filter_trait_query from caendr.services.cloud.postgresql import rollback_on_error_handler @@ -178,3 +178,30 @@ def query_species(species_name): for tf in TraitFile.query_ds(ignore_errs=True, filters=['species', '=', species.name]) if filter_trait_files(tf) ] + + + +# +# Query single trait data +# + + +@api_trait_bp.route('/metadata', methods=['POST']) +@cache.memoize(60*60) +@compress.compressed() +def get_trait_metadata(): + """ + Get traits data for non-bulk files in JSON format (include phenotype values) + """ + trait_name = get_clean(request.json, 'trait_name') + err_msg = f'Failed to retrieve metadata for trait {trait_name}' + + if trait_name: + try: + trait = get_trait(trait_name).to_json_with_values() + return jsonify(trait) + + except Exception as ex: + logger.error(f'{err_msg}: {ex}') + + return jsonify({ 'message': err_msg }), 404 diff --git a/src/modules/site-v2/base/views/tools/phenotype_database.py b/src/modules/site-v2/base/views/tools/phenotype_database.py index ea476c904..8d72b4480 100644 --- a/src/modules/site-v2/base/views/tools/phenotype_database.py +++ b/src/modules/site-v2/base/views/tools/phenotype_database.py @@ -101,26 +101,6 @@ def phenotype_database(): }) -@phenotype_database_bp.route('/traits-list', methods=['POST']) -@cache.memoize(60*60) -@compress.compressed() -def get_traits_json(): - """ - Get traits data for non-bulk files in JSON format (include phenotype values) - """ - trait_name = request.json.get('trait_name') - err_msg = f'Failed to retrieve metadata for trait {trait_name}' - - if trait_name: - try: - trait = get_trait(trait_name).to_json_with_values() - return jsonify(trait) - - except Exception as ex: - logger.error(f'{err_msg}: {ex}') - - return jsonify({ 'message': err_msg }), 404 - # # Submission Flow diff --git a/src/modules/site-v2/templates/_includes/phenotype-database-table.html b/src/modules/site-v2/templates/_includes/phenotype-database-table.html index 892832423..01dff2816 100644 --- a/src/modules/site-v2/templates/_includes/phenotype-database-table.html +++ b/src/modules/site-v2/templates/_includes/phenotype-database-table.html @@ -188,7 +188,7 @@

const data = {csrf_token: $('#csrf_token').val(), trait_name: $(this).data('value')} $.ajax({ type: "POST", - url: "{{ url_for('phenotype_database.get_traits_json') }}", + url: "{{ url_for('api_trait.get_trait_metadata') }}", data: JSON.stringify(data), contentType: "application/json", dataType: "json", From 83805340ea91f11e6e772121f2063e257e6787ea Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Fri, 5 Apr 2024 16:08:12 -0500 Subject: [PATCH 12/73] Rewrite metadata endpoint w helpers, better error handling --- src/modules/site-v2/base/views/api/trait.py | 64 ++++++++++++++------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/src/modules/site-v2/base/views/api/trait.py b/src/modules/site-v2/base/views/api/trait.py index 8731c53fc..603cc68f6 100644 --- a/src/modules/site-v2/base/views/api/trait.py +++ b/src/modules/site-v2/base/views/api/trait.py @@ -42,19 +42,38 @@ def get_clean(source, key, value=None, _type=None): return v -def query_traits_error_handler(f): +def query_traits_error_handler(err_msg): ''' Wrapper for trait query endpoints. - If query raises an error, returns an empty response and a `500` error. + + If the wrapped function aborts with an error code, that code will be used + by the response. Otherwise, based on the error type, either a `404` or a + `500` will be returned. ''' - @wraps(f) - def inner(*args, **kwargs): - try: - return f(*args, **kwargs) - except Exception as ex: - logger.error(f'Failed to retrieve the list of traits: {ex}') - return {}, 500 - return inner + + def decorator(f): + @wraps(f) + def inner(*args, **kwargs): + try: + return f(*args, **kwargs) + + # Error handling + except Exception as ex: + + # Choose error code based on error type + if hasattr(ex, 'code'): + err_code = ex.code + elif isinstance(ex, NotFoundError): + err_code = 404 + else: + err_code = 500 + + # Log the full error, and return the response with an abridged message + logger.error(f'{err_msg}: {ex}') + return {'message': f'{err_msg}'}, err_code + + return inner + return decorator @@ -65,7 +84,7 @@ def inner(*args, **kwargs): @api_trait_bp.route('/query/sql', methods=['POST']) @cache.memoize(60*60) -@query_traits_error_handler +@query_traits_error_handler('Failed to retrieve the list of traits') @jsonify_request def query_sql(): ''' @@ -113,7 +132,7 @@ def query_sql(): @api_trait_bp.route('/query/datatable', methods=['GET']) @cache.memoize(60*60) -@query_traits_error_handler +@query_traits_error_handler('Failed to retrieve the list of traits') @jsonify_request def query_datatable(): ''' @@ -189,19 +208,22 @@ def query_species(species_name): @api_trait_bp.route('/metadata', methods=['POST']) @cache.memoize(60*60) @compress.compressed() +@query_traits_error_handler('Failed to retrieve trait metadata') +@jsonify_request def get_trait_metadata(): """ Get traits data for non-bulk files in JSON format (include phenotype values) """ - trait_name = get_clean(request.json, 'trait_name') - err_msg = f'Failed to retrieve metadata for trait {trait_name}' - if trait_name: - try: - trait = get_trait(trait_name).to_json_with_values() - return jsonify(trait) + # Get the trait name from the request + trait_name = get_clean(request.json, 'trait_name') + if not trait_name: + abort(400, description='No trait name provided.') - except Exception as ex: - logger.error(f'{err_msg}: {ex}') + # Try getting the trait from the database + trait = get_trait(trait_name) + if trait is None: + abort(404, description=f'Invalid trait name {trait_name}') - return jsonify({ 'message': err_msg }), 404 + # Return the full trait metadata + return trait.to_json_with_values() From e34ba69b4194243dbd479abe440de8d0ec22df65 Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Mon, 8 Apr 2024 12:34:03 -0500 Subject: [PATCH 13/73] Modularize trait offcanvas sidebar JS code --- .../_includes/phenotype-database-table.html | 109 +++------------ .../phenotype-offcanvas-content.html | 125 ++++++++++++++++-- .../site-v2/templates/_scripts/utils.js | 11 ++ 3 files changed, 142 insertions(+), 103 deletions(-) diff --git a/src/modules/site-v2/templates/_includes/phenotype-database-table.html b/src/modules/site-v2/templates/_includes/phenotype-database-table.html index 01dff2816..307af3918 100644 --- a/src/modules/site-v2/templates/_includes/phenotype-database-table.html +++ b/src/modules/site-v2/templates/_includes/phenotype-database-table.html @@ -21,6 +21,10 @@ {% endblock %} + +{% set offcanvas_id = 'phenotype-offcanvas' %} + +
+
+{% endblock %} + + + +{% block script %} + +{% endblock %} From d6ada4b97f12866cf1b4c8bf6b9083d3d205df56 Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Tue, 23 Apr 2024 15:40:19 -0500 Subject: [PATCH 41/73] Fix create/edit page title --- src/modules/site-v2/base/views/admin/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/site-v2/base/views/admin/admin.py b/src/modules/site-v2/base/views/admin/admin.py index 743f6a461..0ac16817f 100755 --- a/src/modules/site-v2/base/views/admin/admin.py +++ b/src/modules/site-v2/base/views/admin/admin.py @@ -67,7 +67,7 @@ def announcements_edit(entity_id=None): announcement = None return render_template('admin/announcements/edit.html', **{ - 'title': 'Edit Announcement', + 'title': ('Edit' if announcement else 'Create') + ' Announcement', 'form': AnnouncementForm(), 'announcement': announcement, From 226b707615cfc9bebd4a30d23ad0ecc4856bfea7 Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Tue, 23 Apr 2024 15:59:40 -0500 Subject: [PATCH 42/73] Pre-populate form values from entity --- src/modules/site-v2/base/views/admin/admin.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/modules/site-v2/base/views/admin/admin.py b/src/modules/site-v2/base/views/admin/admin.py index 0ac16817f..6172b732d 100755 --- a/src/modules/site-v2/base/views/admin/admin.py +++ b/src/modules/site-v2/base/views/admin/admin.py @@ -66,9 +66,12 @@ def announcements_edit(entity_id=None): else: announcement = None + # Initialize the form with the existing object (or None if creating new) + form = AnnouncementForm(obj=announcement) + return render_template('admin/announcements/edit.html', **{ 'title': ('Edit' if announcement else 'Create') + ' Announcement', - 'form': AnnouncementForm(), + 'form': form, 'announcement': announcement, }) From 7d37194b3b120181342abacfd19cf995b7e3cc33 Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Tue, 23 Apr 2024 16:00:16 -0500 Subject: [PATCH 43/73] Fix form formatting --- .../site-v2/templates/admin/announcements/edit.html | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/modules/site-v2/templates/admin/announcements/edit.html b/src/modules/site-v2/templates/admin/announcements/edit.html index 87ecec104..01d0829d3 100644 --- a/src/modules/site-v2/templates/admin/announcements/edit.html +++ b/src/modules/site-v2/templates/admin/announcements/edit.html @@ -15,8 +15,7 @@
-
-
+
- {{ render_field(form.active) }} - {{ render_field(form.url_list) }} + {#
{{ render_field(form.active) }}
#} +
{{ render_field(form.url_list) }}
+ +
From 90dbc5ccf76440637efcf716d258779394a642bd Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Tue, 23 Apr 2024 16:12:15 -0500 Subject: [PATCH 44/73] Flash error messages on request failures --- .../templates/admin/announcements/edit.html | 13 ++++++++++--- .../templates/admin/announcements/list.html | 15 ++++++++++++++- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/modules/site-v2/templates/admin/announcements/edit.html b/src/modules/site-v2/templates/admin/announcements/edit.html index 01d0829d3..91531a523 100644 --- a/src/modules/site-v2/templates/admin/announcements/edit.html +++ b/src/modules/site-v2/templates/admin/announcements/edit.html @@ -39,6 +39,8 @@ {% block script %} From 2c4a097b9a2ee3e0bd6d6253bdefd2584856932a Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Thu, 25 Apr 2024 11:59:50 -0500 Subject: [PATCH 47/73] Change whether announcement is active in edit page --- src/modules/site-v2/base/forms/forms.py | 2 + .../site-v2/base/views/api/notifications.py | 13 +++++-- .../site-v2/templates/_includes/macros.html | 2 + .../templates/admin/announcements/edit.html | 37 +++++++++++++------ .../templates/admin/announcements/list.html | 3 +- 5 files changed, 42 insertions(+), 15 deletions(-) diff --git a/src/modules/site-v2/base/forms/forms.py b/src/modules/site-v2/base/forms/forms.py index 55d5bde1a..de465b8b5 100755 --- a/src/modules/site-v2/base/forms/forms.py +++ b/src/modules/site-v2/base/forms/forms.py @@ -27,6 +27,7 @@ Optional, ValidationError) from wtforms.fields.html5 import EmailField +from wtforms.widgets import CheckboxInput from constants import PRICES, SECTOR_OPTIONS, SHIPPING_OPTIONS, PAYMENT_OPTIONS, TOOL_INPUT_DATA_VALID_FILE_EXTENSIONS @@ -123,6 +124,7 @@ class RecoverUserForm(FlaskForm): class AnnouncementForm(FlaskForm): """ Edit form for site announcements """ + # active = BooleanField('Active', widget=CheckboxInput()) active = BooleanField('Active') content = StringField('Content', [Optional()]) url_list = TextAreaField('URL Patterns', [Optional()]) diff --git a/src/modules/site-v2/base/views/api/notifications.py b/src/modules/site-v2/base/views/api/notifications.py index be9150730..ac8b92abc 100644 --- a/src/modules/site-v2/base/views/api/notifications.py +++ b/src/modules/site-v2/base/views/api/notifications.py @@ -163,11 +163,18 @@ def announcement(entity_id: str = None): if not form.validate(): return 400 + # Extract the new property values from the form, casting "active" to a bool + new_values = { + prop: request.form.get(prop) + for prop in Announcement.get_props_set() + if request.form.get(prop) is not None + } + if request.form.get('active') is not None: + new_values['active'] = request.form.get('active') == 'true' + # Update the announcement object announcement = get_announcement(entity_id) - announcement.set_properties(**{ - prop: request.form.get(prop, announcement[prop]) for prop in Announcement.get_props_set() - }) + announcement.set_properties(**new_values) announcement.save() return { 'id': announcement.name } diff --git a/src/modules/site-v2/templates/_includes/macros.html b/src/modules/site-v2/templates/_includes/macros.html index d7b3909a1..1ae36d6b5 100755 --- a/src/modules/site-v2/templates/_includes/macros.html +++ b/src/modules/site-v2/templates/_includes/macros.html @@ -35,6 +35,8 @@ {% if form_prefix %}{{ form_prefix }}{% endif %} {% if field.type == 'RadioField' %} {{ field(**kwargs) }} + {% elif field.type == 'BooleanField' %} + {{ field(**kwargs) }} {% elif field.type == 'FileField' %}
{{ field(class_="form-control", required=kwargs.get('required', field.flags.required), **kwargs) }} diff --git a/src/modules/site-v2/templates/admin/announcements/edit.html b/src/modules/site-v2/templates/admin/announcements/edit.html index c65cff2ad..4aab47e95 100644 --- a/src/modules/site-v2/templates/admin/announcements/edit.html +++ b/src/modules/site-v2/templates/admin/announcements/edit.html @@ -27,8 +27,8 @@
- {#
{{ render_field(form.active) }}
#} -
{{ render_field(form.style) }}
+
{{ render_field(form.active, class_='form-check-input d-block') }}
+
{{ render_field(form.style) }}
{{ render_field(form.url_list) }}
@@ -46,6 +46,10 @@ {% include '_scripts/utils.js' %} {#/* defines: flash_message */#} + // Declare the MD editor at the "global" scope, so we can get its contents from anywhere + let editor; + + function disableButton() { const btn = document.getElementById('save-btn'); btn.disabled = true; @@ -59,11 +63,29 @@ } + function getFormData() { + // Get the fields from the form + const data = new FormData($('form#form-submit')[0]); + + // Make the "active" checkmark a boolean + // Without this step, checked => "y" and unchecked isn't included + const active = $('form#form-submit #active')[0]; + data.set('active', active.checked); + + // Append the Markdown content to the form object + const md_content = editor.getMarkdown(); + data.append('content', md_content); + + // Return the form data + return data; + } + + $(document).ready(function () { // Markdown editor const Editor = toastui.Editor; - const editor = new Editor({ + editor = new Editor({ el: document.querySelector('#editor'), height: '200px', initialEditType: 'markdown', @@ -80,13 +102,6 @@ e.preventDefault(); disableButton(); - // Get the fields from the form - const data = new FormData($('form#form-submit')[0]); - - // Append the Markdown content to the form object - const md_content = editor.getMarkdown(); - data.append('content', md_content); - // Send the appropriate AJAX request $.ajax({ {% if announcement %} @@ -102,7 +117,7 @@ processData: false, contentType: false, dataType: 'json', - data: data, + data: getFormData(), // On success, redirect to list page success: function(result) { diff --git a/src/modules/site-v2/templates/admin/announcements/list.html b/src/modules/site-v2/templates/admin/announcements/list.html index af295d7dd..60af26231 100644 --- a/src/modules/site-v2/templates/admin/announcements/list.html +++ b/src/modules/site-v2/templates/admin/announcements/list.html @@ -75,8 +75,9 @@ { "data": "active", "render": function(data, type, row) { - return ``; + return data ? `` : ""; }, + "sClass": "optionsToolbar", 'sWidth': '10%', }, { From 25dd3779d8bfe483630f7194caa1cce319733767 Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Thu, 25 Apr 2024 13:04:26 -0500 Subject: [PATCH 48/73] Add preview to edit page --- .../site-v2/templates/_includes/alert.html | 11 ++++- .../templates/admin/announcements/edit.html | 44 +++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/src/modules/site-v2/templates/_includes/alert.html b/src/modules/site-v2/templates/_includes/alert.html index cf5d26a91..585dcb5ce 100644 --- a/src/modules/site-v2/templates/_includes/alert.html +++ b/src/modules/site-v2/templates/_includes/alert.html @@ -1,4 +1,13 @@ -
+{% if not dismissable is defined %} + {% set dismissable = true %} +{% endif %} + +

{{ msg }}

+ {%- if dismissable %} + {%- endif %}
\ No newline at end of file diff --git a/src/modules/site-v2/templates/admin/announcements/edit.html b/src/modules/site-v2/templates/admin/announcements/edit.html index 4aab47e95..e21986eaa 100644 --- a/src/modules/site-v2/templates/admin/announcements/edit.html +++ b/src/modules/site-v2/templates/admin/announcements/edit.html @@ -10,6 +10,16 @@ {% from "_includes/macros.html" import render_field %} + +
+ + {% with dismissable = false %} + {% with alert_id = 'preview-alert' %} + {% include '_includes/alert.html' %} + {% endwith %} + {% endwith %} +
+
{{ form.csrf_token }} @@ -49,6 +59,12 @@ // Declare the MD editor at the "global" scope, so we can get its contents from anywhere let editor; + const alertClassMap = { + {%- for x in AnnouncementType %} + "{{ x.name }}": "{{ x.get_bootstrap_color() }}", + {%- endfor %} + }; + function disableButton() { const btn = document.getElementById('save-btn'); @@ -81,6 +97,23 @@ } + function updatePreview() { + const data = getFormData(); + const preview = document.getElementById('preview-alert'); + + // Update the preview alert class + preview.classList.forEach((foo) => { + if (foo.startsWith('alert-')) { + preview.classList.remove(foo); + } + }); + preview.classList.add(`alert-${ alertClassMap[ data.get('style') ] }`); + + // Update the content + preview.innerHTML = editor.getHTML(); + } + + $(document).ready(function () { // Markdown editor @@ -94,9 +127,20 @@ toolbarItems: [ ['bold', 'italic', 'link'], ], + events: { + // Update preview when content changes + change: () => updatePreview(), + }, }); + // Initialize preview, then update when form changes + updatePreview(); + $("#form-submit").on("change", function(e) { + updatePreview(); + }) + + // Form Submission $("#save-btn").on("click", function(e) { e.preventDefault(); From 78ac2f44aa0829647bfe47151f1258931b341404 Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Thu, 25 Apr 2024 13:05:41 -0500 Subject: [PATCH 49/73] Prevent enter from submitting edit form inside MD editor --- .../site-v2/templates/admin/announcements/edit.html | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/modules/site-v2/templates/admin/announcements/edit.html b/src/modules/site-v2/templates/admin/announcements/edit.html index e21986eaa..eca7d6e53 100644 --- a/src/modules/site-v2/templates/admin/announcements/edit.html +++ b/src/modules/site-v2/templates/admin/announcements/edit.html @@ -189,6 +189,15 @@ window.location = "{{ url_for('admin.announcements') }}"; }); + + // Prevent enter key from submitting form inside Markdown editor + // Primarily affects hyperlink insert popup + $('#editor').keydown(function(event) { + if (event.keyCode == 13) { + event.preventDefault(); + return false; + } + }); }); From f8ac0c997a462d75f4f20cbd5fa20229660f9618 Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Thu, 25 Apr 2024 14:33:45 -0500 Subject: [PATCH 50/73] Fix: Include AnnouncementType in edit page --- src/modules/site-v2/base/views/admin/admin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/modules/site-v2/base/views/admin/admin.py b/src/modules/site-v2/base/views/admin/admin.py index 430c449a8..b6be256df 100755 --- a/src/modules/site-v2/base/views/admin/admin.py +++ b/src/modules/site-v2/base/views/admin/admin.py @@ -79,4 +79,5 @@ def announcements_edit(entity_id=None): 'form': form, 'announcement': announcement, + 'AnnouncementType': AnnouncementType, }) From 9631791bd30f512dd362ad10e8ab97a66dc98225 Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Thu, 25 Apr 2024 14:34:36 -0500 Subject: [PATCH 51/73] Fix: abort with errors instead of returning --- src/modules/site-v2/base/views/api/notifications.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/modules/site-v2/base/views/api/notifications.py b/src/modules/site-v2/base/views/api/notifications.py index ac8b92abc..485224a68 100644 --- a/src/modules/site-v2/base/views/api/notifications.py +++ b/src/modules/site-v2/base/views/api/notifications.py @@ -145,7 +145,7 @@ def announcement(entity_id: str = None): # TODO: Clean / validate values form = AnnouncementForm(request.form) if not form.validate(): - return 400 + abort(400) new_announcement = Announcement(**{ prop: request.form.get(prop) for prop in Announcement.get_props_set() @@ -161,7 +161,7 @@ def announcement(entity_id: str = None): # TODO: Clean / validate values form = AnnouncementForm(request.form) if not form.validate(): - return 400 + abort(400) # Extract the new property values from the form, casting "active" to a bool new_values = { @@ -180,5 +180,4 @@ def announcement(entity_id: str = None): # If somehow the method didn't match any of the above, # return a Method Not Allowed error - return 405 - + abort(405) From 01c004f07873339542f9bf1fdf2e36cfa0b9ae1f Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Thu, 25 Apr 2024 14:40:56 -0500 Subject: [PATCH 52/73] Make announcements deletable --- .../site-v2/base/views/api/notifications.py | 13 ++- .../templates/admin/announcements/list.html | 93 ++++++++++++++++++- .../caendr/models/datastore/announcement.py | 4 +- .../models/datastore/deletable_entity.py | 13 ++- 4 files changed, 116 insertions(+), 7 deletions(-) diff --git a/src/modules/site-v2/base/views/api/notifications.py b/src/modules/site-v2/base/views/api/notifications.py index 485224a68..def8e0b74 100644 --- a/src/modules/site-v2/base/views/api/notifications.py +++ b/src/modules/site-v2/base/views/api/notifications.py @@ -114,12 +114,12 @@ def announcement_list(): Get the list of all site announcements. ''' return { - 'data': [ e.serialize(include_name=True) for e in Announcement.query_ds() ] + 'data': [ e.serialize(include_name=True) for e in Announcement.query_ds(deleted=False) ] } @api_notifications_bp.route('/announcement', methods=['POST']) -@api_notifications_bp.route('/announcement/', methods=['GET', 'PATCH']) +@api_notifications_bp.route('/announcement/', methods=['GET', 'PATCH', 'DELETE']) @admin_required() @jsonify_request def announcement(entity_id: str = None): @@ -130,6 +130,7 @@ def announcement(entity_id: str = None): GET: Get the data for the given announcement ID. POST: Create a new announcement. PATCH: Update an existing announcement. + DELETE: Delete an existing announcement. ''' # GET Request @@ -178,6 +179,14 @@ def announcement(entity_id: str = None): announcement.save() return { 'id': announcement.name } + # DELETE Request + # Lookup the desired announcement and soft delete it + if request.method == 'DELETE': + announcement = get_announcement(entity_id) + announcement.soft_delete() + announcement.save() + return {}, 200 + # If somehow the method didn't match any of the above, # return a Method Not Allowed error abort(405) diff --git a/src/modules/site-v2/templates/admin/announcements/list.html b/src/modules/site-v2/templates/admin/announcements/list.html index 60af26231..e3a65d25a 100644 --- a/src/modules/site-v2/templates/admin/announcements/list.html +++ b/src/modules/site-v2/templates/admin/announcements/list.html @@ -29,6 +29,24 @@
+ + {% endblock %} @@ -37,6 +55,14 @@ + + + + + + diff --git a/src/modules/site-v2/templates/_scripts/utils.js b/src/modules/site-v2/templates/_scripts/utils.js index 7b8ad66d7..98dd2220b 100644 --- a/src/modules/site-v2/templates/_scripts/utils.js +++ b/src/modules/site-v2/templates/_scripts/utils.js @@ -105,6 +105,17 @@ function create_node(html) { } +// Convert text from Markdown to HTML using Toast UI +// This protects against code injection +function markdown_to_html(text) { + const editor = new toastui.Editor({ + el: document.createElement('div'), + initialValue: text, + }); + return editor.getHTML(); +} + + function save_svg(selector, filename=null) { const svg_el = document.querySelector(selector); const data = (new XMLSerializer()).serializeToString(svg_el); @@ -132,10 +143,9 @@ function flash_message(message, full_msg_link=null, full_msg_body=null) { const raw_html = `{% include '_includes/alert.html' %}`; {%- endwith %} - // Create as a new DOM node, and insert the desired message as text - // Inserting as text protects against code injection + // Create as a new DOM node, and insert the desired message as formatted (cleaned) HTML const node = create_node(raw_html); - node.firstElementChild.innerText = message; + node.firstElementChild.innerHTML = markdown_to_html(message); // If both full message fields are provided, add as a link & collapse dropdown if (full_msg_link && full_msg_body) { diff --git a/src/modules/site-v2/templates/admin/announcements/edit.html b/src/modules/site-v2/templates/admin/announcements/edit.html index fc64c15fe..9f263c32e 100644 --- a/src/modules/site-v2/templates/admin/announcements/edit.html +++ b/src/modules/site-v2/templates/admin/announcements/edit.html @@ -1,14 +1,8 @@ {% extends "_layouts/default.html" %} -{% block custom_head %} - -{% endblock %} - - {% block content %} {% from "_includes/macros.html" import render_field %} -
diff --git a/src/modules/site-v2/templates/admin/announcements/list.html b/src/modules/site-v2/templates/admin/announcements/list.html index 635b4a700..5c1c9361c 100644 --- a/src/modules/site-v2/templates/admin/announcements/list.html +++ b/src/modules/site-v2/templates/admin/announcements/list.html @@ -169,6 +169,10 @@

Delete Announcement { "data": "content", 'sWidth': '35%', + "render": function(data, type, row) { + // Protect against code injection + return $('
').text(data).html().replace('\n', '
'); + }, }, { "data": "url_list", From 69546dcdb8ddd195b794151ae6b7b16519f56be9 Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Thu, 25 Apr 2024 16:44:25 -0500 Subject: [PATCH 57/73] Fix: Handle content with multiple lines --- src/modules/site-v2/templates/admin/announcements/edit.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/site-v2/templates/admin/announcements/edit.html b/src/modules/site-v2/templates/admin/announcements/edit.html index 9f263c32e..1047a56a5 100644 --- a/src/modules/site-v2/templates/admin/announcements/edit.html +++ b/src/modules/site-v2/templates/admin/announcements/edit.html @@ -119,7 +119,7 @@ height: '200px', initialEditType: 'markdown', previewStyle: 'vertical', - initialValue: "{{ announcement['content'] }}", + initialValue: `{{ announcement['content'] }}`, toolbarItems: [ ['bold', 'italic', 'link'], ], From 099d98289c08c1a2d0e687699d4997d6780af546 Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Thu, 25 Apr 2024 16:44:55 -0500 Subject: [PATCH 58/73] Set max content length for display in table --- src/modules/site-v2/templates/admin/announcements/list.html | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/modules/site-v2/templates/admin/announcements/list.html b/src/modules/site-v2/templates/admin/announcements/list.html index 5c1c9361c..313cf30b5 100644 --- a/src/modules/site-v2/templates/admin/announcements/list.html +++ b/src/modules/site-v2/templates/admin/announcements/list.html @@ -56,6 +56,8 @@

Delete Announcement {% include '_scripts/utils.js' %} {#/* defines: flash_message */#} + const tableTextMaxLength = 256; + // Declare object to store announcements returned from AJAX call let announcements = {}; let currentAnnouncement = null; @@ -170,6 +172,10 @@

Delete Announcement "data": "content", 'sWidth': '35%', "render": function(data, type, row) { + // Cap text length + if (data.length > tableTextMaxLength) { + data = data.slice(0, tableTextMaxLength - 3) + '...'; + } // Protect against code injection return $('
').text(data).html().replace('\n', '
'); }, From 86e8ecb2aec0b0744cdd8d92f4801f7c625abbda Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Thu, 25 Apr 2024 17:31:37 -0500 Subject: [PATCH 59/73] Check active announcements after creating all requests Uses `@app.after_request` callback to loop through active Announcements and flash any that apply. Only applies to non-redirecting GET requests that return HTML, excluding static files and specific endpoints that set the `block_site_announcements` flag. --- src/modules/site-v2/application.py | 36 +++++++- .../site-v2/base/utils/announcements.py | 27 ++++++ src/modules/site-v2/base/views/auth/auth.py | 6 ++ .../base/views/tools/genetic_mapping.py | 2 + .../caendr/models/datastore/announcement.py | 83 ++++++++++++++++++- 5 files changed, 152 insertions(+), 2 deletions(-) create mode 100644 src/modules/site-v2/base/utils/announcements.py diff --git a/src/modules/site-v2/application.py b/src/modules/site-v2/application.py index ee069872b..b63ceee1d 100755 --- a/src/modules/site-v2/application.py +++ b/src/modules/site-v2/application.py @@ -4,7 +4,7 @@ from datetime import datetime from caendr.services.logger import logger -from flask import Flask, render_template, request, redirect +from flask import Flask, render_template, request, redirect, flash, g, session from flask_wtf.csrf import CSRFProtect from flask_httpauth import HTTPBasicAuth from werkzeug.security import generate_password_hash, check_password_hash @@ -128,6 +128,7 @@ def create_app(config=config): configure_jinja(app) configure_ssl(app) + register_announcement_handlers(app) password_protect_site(app) # app.teardown_request(close_active_connections) @@ -315,6 +316,39 @@ def _jinja2_filter_markdown(text): return render_markdown_inline(text) +def register_announcement_handlers(app): + ''' + Add site-wide announcements to relevant requests. + ''' + @app.after_request + def flash_announcements(response): + from caendr.models.datastore import Announcement + + # Announcements only possible for a non-redirecting GET request that returns HTML (excluding static files) + # Additionally, specific endpoints can block announcements using the block_announcements decorator + # or the block_announcements_from_bp function + can_show_announcements_conditions = [ + request.endpoint != 'static', + request.method == 'GET', + not (response.status_code >= 300 and response.status_code < 400), + response.mimetype == 'text/html', + not getattr(g, 'block_site_announcements', False), + ] + if all(can_show_announcements_conditions): + + # Loop through all active announcements, flashing the ones that match the current path + for a in Announcement.query_ds(filters=[("active", "=", True)], deleted=False): + if a.matches_path(request.path): + flash(a['content'], a['style'].get_bootstrap_color()) + + # As a final check, remove identical messages from the session + if len(session.get('_flashes', [])): + session.update({'_flashes': list(set(session.get('_flashes', [])))}) + + # Return the response + return response + + def register_errorhandlers(app): def render_error(e="generic"): try: diff --git a/src/modules/site-v2/base/utils/announcements.py b/src/modules/site-v2/base/utils/announcements.py new file mode 100644 index 000000000..723a18671 --- /dev/null +++ b/src/modules/site-v2/base/utils/announcements.py @@ -0,0 +1,27 @@ +from functools import wraps + +from flask import g + + + +def block_announcements(f): + ''' + Prevent an endpoint from flashing site-wide announcements. + ''' + @wraps(f) + def decorator(*args, **kwargs): + try: + g.block_site_announcements = True + except: + pass + return f(*args, **kwargs) + return decorator + + +def block_announcements_from_bp(bp): + ''' + Prevent all endpoints in the given blueprint from flashing site-wide announcements. + ''' + @bp.before_request + def _block_announcements(): + g.block_site_announcements = True diff --git a/src/modules/site-v2/base/views/auth/auth.py b/src/modules/site-v2/base/views/auth/auth.py index 6b176dbdc..353c28bcc 100644 --- a/src/modules/site-v2/base/views/auth/auth.py +++ b/src/modules/site-v2/base/views/auth/auth.py @@ -23,10 +23,16 @@ from caendr.services.cloud.secret import get_secret from base.views.auth.oauth import transfer_cart +from base.utils.announcements import block_announcements_from_bp + + PASSWORD_PEPPER = get_secret('PASSWORD_PEPPER') auth_bp = Blueprint('auth', __name__, template_folder='templates') +block_announcements_from_bp(auth_bp) + + @auth_bp.route('/') def auth(): return redirect(url_for('auth.choose_login')) diff --git a/src/modules/site-v2/base/views/tools/genetic_mapping.py b/src/modules/site-v2/base/views/tools/genetic_mapping.py index 5c8e3fc76..9e973fa25 100755 --- a/src/modules/site-v2/base/views/tools/genetic_mapping.py +++ b/src/modules/site-v2/base/views/tools/genetic_mapping.py @@ -6,6 +6,7 @@ from flask import jsonify from base.forms import MappingForm +from base.utils.announcements import block_announcements from base.utils.auth import get_jwt, jwt_required, admin_required, get_current_user, user_is_admin from base.utils.tools import get_upload_err_msg, lookup_report, list_reports, try_submit from constants import TOOL_INPUT_DATA_VALID_FILE_EXTENSIONS @@ -188,6 +189,7 @@ def report(id): @genetic_mapping_bp.route('/report//fullscreen', methods=['GET']) @jwt_required() +@block_announcements def report_fullscreen(id): # Fetch requested mapping report diff --git a/src/pkg/caendr/caendr/models/datastore/announcement.py b/src/pkg/caendr/caendr/models/datastore/announcement.py index 266e98694..17f596cb5 100644 --- a/src/pkg/caendr/caendr/models/datastore/announcement.py +++ b/src/pkg/caendr/caendr/models/datastore/announcement.py @@ -1,4 +1,5 @@ -from enum import Enum +from enum import Enum +import re from caendr.models.datastore import DeletableEntity from caendr.utils.data import unique_id @@ -67,3 +68,83 @@ def style(self): @style.setter def style(self, val): return self._set_enum_prop(AnnouncementType, 'style', val) + + + @property + def url_list(self): + return self._get_raw_prop('url_list', '') + + @url_list.setter + def url_list(self, val): + ''' + Validate the list of URL patterns before saving. + ''' + + # Accept newline-separated string or iterable of patterns + # Convert to a list for validating + if isinstance(val, str): + val = [ pattern.strip() for pattern in val.split('\n') ] + if not isinstance(val, list): + try: + val = [*val] + except: + raise ValueError('URL list must be an iterable or a newline-separated string') + + # Validate each pattern + for idx, path in enumerate(val): + if not self.validate_path(path): + raise ValueError(f'Invalid pattern on line {idx}: "{path}"') + + # If validation succeeded, convert to newline-separated string and save + return self._set_raw_prop('url_list', '\n'.join(val)) + + + + # + # URL Paths + # + + @classmethod + def validate_path(cls, path: str) -> bool: + ''' + Each path must meet the following constraints: + - The first character is a forward slash + - If there is an asterisk (wildcard), it must: + - be the final character + - be preceded by a forward slash + + Sample valid paths: + / + /* + /foo/bar/baz/* + + Sample invalid paths: + foo/bar + /foo* + /foo*/bar + ''' + # Validate the above rules using Regex: + # ^\/ First character is a forward slash '/' + # [^\*]* String of any length that doesn't include a '*' character + # ((?<=\/)\*)?$ Optionally, last character may be '*', and if so previous character must be '/' + return bool(re.match('^\/[^\*]*((?<=\/)\*)?$', path)) + + + def matches_path(self, path: str) -> bool: + + # Loop through each line in the URL list + for pattern in self['url_list'].split('\n'): + + # Turn our wildcard into a Regex wildcard, if given + if pattern.endswith('/*'): + pattern = pattern[:-2] + suffix = '(/.*)?' + else: + suffix = '' + + # Check this pattern + if re.match(f'^{ pattern }{ suffix }$', path): + return True + + # If no patterns matched, path does not match + return False From 4569c373cdd5998b41185e773f3141d16dcc5254 Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Fri, 26 Apr 2024 15:38:41 -0500 Subject: [PATCH 60/73] Extra layer of input sanitization in Announcement class --- .../caendr/models/datastore/announcement.py | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/src/pkg/caendr/caendr/models/datastore/announcement.py b/src/pkg/caendr/caendr/models/datastore/announcement.py index 17f596cb5..5f8fce6db 100644 --- a/src/pkg/caendr/caendr/models/datastore/announcement.py +++ b/src/pkg/caendr/caendr/models/datastore/announcement.py @@ -1,6 +1,10 @@ +import bleach from enum import Enum +import markdown import re +from flask import Markup + from caendr.models.datastore import DeletableEntity from caendr.utils.data import unique_id @@ -39,7 +43,10 @@ def __init__(self, name_or_obj = None, *args, **kwargs): super().__init__(name_or_obj, *args, **kwargs) - ## Props ## + + # + # Props + # @classmethod def get_props_set(cls): @@ -61,6 +68,17 @@ def active(self, val): self._set_raw_prop('active', bool(val)) + @property + def content(self): + return bleach.clean( self._get_raw_prop('content', '') ) + + @content.setter + def content(self, val): + if not isinstance(val, str): + raise ValueError(f'Prop "content" must be a string, got: {val}') + return self._set_raw_prop( 'content', bleach.clean( val ) ) + + @property def style(self): return self._get_enum_prop(AnnouncementType, 'style', None) @@ -100,6 +118,25 @@ def url_list(self, val): + # + # Extra Props + # + + @property + def content_html(self): + ''' + The content of this announcement as HTML. + ''' + return Markup(markdown.markdown( self['content'] )) + + + def serialize(self, **kwargs): + props = super().serialize(**kwargs) + props['content_html'] = self.content_html + return props + + + # # URL Paths # From 64a74294996f691058dfdce1588f7a1d8d9b2e05 Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Mon, 29 Apr 2024 10:34:15 -0500 Subject: [PATCH 61/73] Fix announcement polling / flashing with context processor Check for announcements in a Jinja template context processor, which means we will only flash messages when rendering a Jinja template. This automatically handles responses that don't return templates (i.e. ones that don't clear the flashed message queue), so messages shouldn't bleed over into the next request. This also fixes an offset bug - we have to call `flash` before the template is rendered, otherwise announcements won't be queued in time and will be applied to the *next* page load. --- src/modules/site-v2/application.py | 85 ++++++++++++++++++++++-------- 1 file changed, 64 insertions(+), 21 deletions(-) diff --git a/src/modules/site-v2/application.py b/src/modules/site-v2/application.py index b63ceee1d..c23b6f809 100755 --- a/src/modules/site-v2/application.py +++ b/src/modules/site-v2/application.py @@ -320,33 +320,76 @@ def register_announcement_handlers(app): ''' Add site-wide announcements to relevant requests. ''' - @app.after_request - def flash_announcements(response): - from caendr.models.datastore import Announcement - - # Announcements only possible for a non-redirecting GET request that returns HTML (excluding static files) - # Additionally, specific endpoints can block announcements using the block_announcements decorator - # or the block_announcements_from_bp function - can_show_announcements_conditions = [ + from caendr.models.datastore import Announcement + + def can_show_announcements(request, response=None, check_session_flags=False) -> bool: + ''' + Helper function to determine whether a request should flash the active site-wide announcements. + + Announcements only possible for a non-redirecting GET request that returns HTML, excluding static files. + Additionally, specific endpoints can block announcements using the block_announcements decorator + or the block_announcements_from_bp function. + ''' + + # Request conditions: Must be a GET request to a non-static path + conditions = [ request.endpoint != 'static', request.method == 'GET', - not (response.status_code >= 300 and response.status_code < 400), - response.mimetype == 'text/html', - not getattr(g, 'block_site_announcements', False), ] - if all(can_show_announcements_conditions): - # Loop through all active announcements, flashing the ones that match the current path - for a in Announcement.query_ds(filters=[("active", "=", True)], deleted=False): - if a.matches_path(request.path): - flash(a['content'], a['style'].get_bootstrap_color()) + # Response conditions: Must return HTML, and must not be redirecting + if response: + conditions += [ + response.mimetype == 'text/html', + not (response.status_code >= 300 and response.status_code < 400), + ] + + # Sessions flag(s) + if check_session_flags: + conditions += [ + not getattr(g, 'block_site_announcements', False), + ] + + # Check all conditions + return all(conditions) - # As a final check, remove identical messages from the session - if len(session.get('_flashes', [])): - session.update({'_flashes': list(set(session.get('_flashes', [])))}) + @app.context_processor + def inject_announcements(): + ''' + Before rendering any Jinja template, queue up and flash all the site-wide announcements + that apply to this URL. + + This has to happen before the template is rendered, otherwise announcements will be queued + for the *next* page load. + + This function is registered as a Jinja template context processor, so it will only be run + before a Jinja template is rendered. This prevents announcements from bleeding over to the + next request if e.g. a redirect or a static HTML file is returned instead. + ''' + + # Make sure this request can show announcements + if not can_show_announcements(request, check_session_flags=True): + return - # Return the response - return response + # Make sure site announcements set exists + # Store as a set to help prevent flashing duplicate announcements + if not hasattr(g, 'site_announcements'): + g.site_announcements = set() + + # Queue up all the active announcements that apply to this path + for a in Announcement.query_ds(filters=[("active", "=", True)], deleted=False): + if a.matches_path(request.path): + g.site_announcements.add(a) + + # Flash all the announcements that apply to this path + # We have to do this before the template is rendered, otherwise they'll be queued for the *next* page + for a in g.site_announcements: + flash(a['content'], a['style'].get_bootstrap_color()) + + # Since this is a context processor, we have to return a dict of variables to add to the + # Jinja template rendering context + # In this case, we don't need to add any + return {} def register_errorhandlers(app): From 5984d082cd3d289911b65fa4df9512a7ffd4c7f6 Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Mon, 29 Apr 2024 14:45:48 -0500 Subject: [PATCH 62/73] Use IDs for offcanvas header & body --- .../_includes/phenotype-database-table.html | 18 ++++++++++++++---- .../_includes/phenotype-offcanvas-content.html | 4 ++-- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/modules/site-v2/templates/_includes/phenotype-database-table.html b/src/modules/site-v2/templates/_includes/phenotype-database-table.html index 159de0d65..6f884d82a 100644 --- a/src/modules/site-v2/templates/_includes/phenotype-database-table.html +++ b/src/modules/site-v2/templates/_includes/phenotype-database-table.html @@ -164,14 +164,24 @@ // Fetch data for a trait and fill out side bar for non-bulk traits $(document).on('click','#traitDetails', function() { - $('.offcanvas-header, .offcanvas-body').hide() - $('#{{ offcanvas_id }}-spinner').removeClass('d-none') + const offcanvas_content = $('#{{ offcanvas_id }}-header, #{{ offcanvas_id }}-body'); + const offcanvas_spinner = $('#{{ offcanvas_id }}-spinner'); + + // Show the loader + offcanvas_content.hide() + offcanvas_spinner.removeClass('d-none') + + // Query data for the given trait queryTraitByName($(this).data('value'), $('#csrf_token').val()) + + // On success, fill out and show the offcanvas content .then((response) => { - $('#{{ offcanvas_id }}-spinner').addClass('d-none') - $('.offcanvas-header, .offcanvas-body').show() + offcanvas_spinner.addClass('d-none') + offcanvas_content.show() fillOutModal(response) }) + + // On failure, close the offcanvas item and flash the error (if given) .fail((error) => { console.error(error) $('#{{ offcanvas_id }} .btn-close').click() diff --git a/src/modules/site-v2/templates/_includes/phenotype-offcanvas-content.html b/src/modules/site-v2/templates/_includes/phenotype-offcanvas-content.html index fe066bf1d..58c21e5b1 100644 --- a/src/modules/site-v2/templates/_includes/phenotype-offcanvas-content.html +++ b/src/modules/site-v2/templates/_includes/phenotype-offcanvas-content.html @@ -6,7 +6,7 @@
-
+

@@ -17,7 +17,7 @@

-
+

Capture Date

From 89d4d35fabedc506cf4ab41d70c3400ff993d3a1 Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Mon, 29 Apr 2024 16:11:53 -0500 Subject: [PATCH 63/73] Track & lookup traits by ID in table & submission flow --- .../base/views/tools/phenotype_database.py | 7 +++---- .../_includes/phenotype-database-table.html | 11 +++++------ src/pkg/caendr/caendr/models/trait.py | 18 ++++++++++++++++++ 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/src/modules/site-v2/base/views/tools/phenotype_database.py b/src/modules/site-v2/base/views/tools/phenotype_database.py index 26eabfbdf..c8d40efb0 100644 --- a/src/modules/site-v2/base/views/tools/phenotype_database.py +++ b/src/modules/site-v2/base/views/tools/phenotype_database.py @@ -98,13 +98,12 @@ def submit_traits(): # Check for URL vars specifying an initial trait # These will be inherited from submit_start - initial_trait_name = request.args.get('trait') - initial_trait_set = request.args.get('dataset') + initial_trait_id = request.args.get('trait') # Try looking up the specified trait - if initial_trait_name: + if initial_trait_id: try: - initial_trait = Trait(dataset=initial_trait_set, trait_name=initial_trait_name) + initial_trait = Trait.from_id(initial_trait_id) except NotFoundError: flash('That trait could not be found.', 'danger') initial_trait = None diff --git a/src/modules/site-v2/templates/_includes/phenotype-database-table.html b/src/modules/site-v2/templates/_includes/phenotype-database-table.html index 6f884d82a..79ce1a025 100644 --- a/src/modules/site-v2/templates/_includes/phenotype-database-table.html +++ b/src/modules/site-v2/templates/_includes/phenotype-database-table.html @@ -142,12 +142,11 @@ } }, { - "data": "Analyze", + "data": "id", "render": function(data, type, row) { - const trait_name = row.trait_name_caendr; - const url = '{{ analyze_link | safe }}'.replace("TRAIT_NAME", trait_name).replace("TRAIT_SET", 'zhang'); + const url = '{{ analyze_link | safe }}'.replace("TRAIT_NAME", data).replace("TRAIT_SET", 'zhang'); return ` -
+ ` }, @@ -243,7 +242,7 @@ function filterTable(arr) { for (let i = 0; i < arr.length; i++) { const speciesName = first_letter_caps(arr[i].species_name).replace('_', '. ').italics(); - const traitName = arr[i].trait_name_caendr + const traitName = arr[i].id let url = '{{ analyze_link | safe }}'.replace("TRAIT_NAME", traitName).replace("TRAIT_SET", 'caendr') const createdOn = formatDate(arr[i].created_on) let tagsHTML = '' @@ -269,7 +268,7 @@ ${createdOn} ${tagsHTML} - diff --git a/src/pkg/caendr/caendr/models/trait.py b/src/pkg/caendr/caendr/models/trait.py index 4a351c630..8d819f9d9 100644 --- a/src/pkg/caendr/caendr/models/trait.py +++ b/src/pkg/caendr/caendr/models/trait.py @@ -66,6 +66,24 @@ def __init__(self, dataset = None, trait_name = None, trait_file_id = None, trai # Constructors # + @staticmethod + def from_id(trait_id: str) -> 'Trait': + ''' + Instantiate a `Trait` object from a unique trait ID. + The given ID must exist in the PhenotypeMetadata SQL table, otherwise a `ValueError` will be raised. + ''' + + # Get the SQL row with the given trait ID + sql_row = PhenotypeMetadata.query.get(trait_id) + if sql_row is None: + raise ValueError(f'Invalid trait ID {trait_id}') + + # Construct a Trait object using the data in the SQL row + return Trait( + trait_name = sql_row.trait_name_caendr, + dataset = sql_row.dataset, + ) + @staticmethod def from_dataset(dataset: str, trait_name: Optional[str] = None) -> 'Trait': return Trait( dataset = dataset, trait_name = trait_name ) From 91480e8788ab0d6d6871b216640166eb5d1f5408 Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Mon, 29 Apr 2024 16:19:59 -0500 Subject: [PATCH 64/73] Remove trait dataset param from table selection Using the unique `trait_id` is cleaner --- .../templates/_includes/phenotype-database-table.html | 11 ++++++----- .../tools/phenotype_database/phenotypedb.html | 2 +- .../tools/phenotype_database/submit-traits.html | 6 +++--- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/modules/site-v2/templates/_includes/phenotype-database-table.html b/src/modules/site-v2/templates/_includes/phenotype-database-table.html index 79ce1a025..3d73454ea 100644 --- a/src/modules/site-v2/templates/_includes/phenotype-database-table.html +++ b/src/modules/site-v2/templates/_includes/phenotype-database-table.html @@ -144,7 +144,7 @@ { "data": "id", "render": function(data, type, row) { - const url = '{{ analyze_link | safe }}'.replace("TRAIT_NAME", data).replace("TRAIT_SET", 'zhang'); + const url = '{{ analyze_link | safe }}'.replace("TRAIT_NAME", data); return `
Analyze @@ -242,8 +242,7 @@ function filterTable(arr) { for (let i = 0; i < arr.length; i++) { const speciesName = first_letter_caps(arr[i].species_name).replace('_', '. ').italics(); - const traitName = arr[i].id - let url = '{{ analyze_link | safe }}'.replace("TRAIT_NAME", traitName).replace("TRAIT_SET", 'caendr') + const traitID = arr[i].id const createdOn = formatDate(arr[i].created_on) let tagsHTML = '' if (arr[i].tags !== null && arr[i].tags.length) { @@ -252,6 +251,8 @@ tagsHTML += `
${tags[j].replaceAll('/', ' • ')}
` } } + + const url = '{{ analyze_link | safe }}'.replace("TRAIT_NAME", traitID); let html = ` @@ -268,7 +269,7 @@ ${createdOn} ${tagsHTML} - @@ -280,7 +281,7 @@

Full Description

${arr[i].description_long}

- View Full Details
diff --git a/src/modules/site-v2/templates/tools/phenotype_database/phenotypedb.html b/src/modules/site-v2/templates/tools/phenotype_database/phenotypedb.html index 637afe5e2..1a1bfe97c 100644 --- a/src/modules/site-v2/templates/tools/phenotype_database/phenotypedb.html +++ b/src/modules/site-v2/templates/tools/phenotype_database/phenotypedb.html @@ -8,7 +8,7 @@ class="bi bi-arrow-right-circle-fill me-1" aria-hidden="true">Start an Analysis
- {% with analyze_link = url_for("phenotype_database.submit_start", trait="TRAIT_NAME", dataset="TRAIT_SET") %} + {% with analyze_link = url_for("phenotype_database.submit_start", trait="TRAIT_NAME") %} {% include "_includes/phenotype-database-table.html" %} {% endwith %}
diff --git a/src/modules/site-v2/templates/tools/phenotype_database/submit-traits.html b/src/modules/site-v2/templates/tools/phenotype_database/submit-traits.html index 12ed7eb6c..23a9c729a 100644 --- a/src/modules/site-v2/templates/tools/phenotype_database/submit-traits.html +++ b/src/modules/site-v2/templates/tools/phenotype_database/submit-traits.html @@ -38,7 +38,7 @@
- {% with analyze_link = 'javascript:select_trait(null, \"TRAIT_NAME\", \"TRAIT_SET\")' %} + {% with analyze_link = 'javascript:select_trait(null, \"TRAIT_NAME\")' %} {% include "_includes/phenotype-database-table.html" %} {% endwith %}
@@ -129,7 +129,7 @@ } - function select_trait(id=null, trait=null, dataset=null) { + function select_trait(id=null, trait=null) { // Unless a selector is specifically named, // default to last selector to open the phenotype db modal @@ -142,7 +142,7 @@ // Response check prevents infinite recursion .then((response) => { if (response !== null && typeof response === 'string') throw Error(`Invalid response for trait ${id}: "${response}"`); - select_trait(id, response, dataset); + select_trait(id, response); }) .fail((error) => { // If an error message was received, display it, otherwise use a fallback message From 24ad04be6690e219a73da508d8e5f7bbd6b0c4cb Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Mon, 29 Apr 2024 16:36:44 -0500 Subject: [PATCH 65/73] Handle trait ID lookup error --- src/modules/site-v2/base/views/tools/phenotype_database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/site-v2/base/views/tools/phenotype_database.py b/src/modules/site-v2/base/views/tools/phenotype_database.py index c8d40efb0..66ac16073 100644 --- a/src/modules/site-v2/base/views/tools/phenotype_database.py +++ b/src/modules/site-v2/base/views/tools/phenotype_database.py @@ -104,7 +104,7 @@ def submit_traits(): if initial_trait_id: try: initial_trait = Trait.from_id(initial_trait_id) - except NotFoundError: + except (NotFoundError, ValueError): flash('That trait could not be found.', 'danger') initial_trait = None else: From 76a569635c14be054188ed7cf1f5b7950863a924 Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Mon, 29 Apr 2024 16:39:19 -0500 Subject: [PATCH 66/73] Clean up Datatable column definitions a bit --- .../templates/_includes/phenotype-database-table.html | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/modules/site-v2/templates/_includes/phenotype-database-table.html b/src/modules/site-v2/templates/_includes/phenotype-database-table.html index 3d73454ea..9d4b85645 100644 --- a/src/modules/site-v2/templates/_includes/phenotype-database-table.html +++ b/src/modules/site-v2/templates/_includes/phenotype-database-table.html @@ -124,13 +124,12 @@ {"data": "trait_name_caendr"}, {"data": "description_long"}, { - "data": "Upload Date", + "data": "created_on", "render": function(data, type, row) { - return formatDate(row.created_on) + return formatDate(data) } }, { - "data": "View Full Details", "render": function(data, type, row) { const d = JSON.stringify(row) return ` From 2a7f27ed41f6947a72fbcca333b8742024008c96 Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Mon, 29 Apr 2024 16:41:04 -0500 Subject: [PATCH 67/73] Move flashing error message to JS util function A number of endpoints (i.e. tool submissions) have a standard set of error fields they might return, so this pulls that all together. Also means tool submissions now scroll to the top when an error message appears, which is nice. --- .../_includes/phenotype-database-table.html | 8 +------ .../_includes/trait-library-table.html | 8 +------ .../site-v2/templates/_scripts/utils.js | 21 +++++++++++++++++++ .../tools/genetic_mapping/mapping.html | 7 ++----- .../heritability-calculator.html | 7 ++----- .../phenotype_database/submit-traits.html | 12 ++--------- 6 files changed, 29 insertions(+), 34 deletions(-) diff --git a/src/modules/site-v2/templates/_includes/phenotype-database-table.html b/src/modules/site-v2/templates/_includes/phenotype-database-table.html index 9d4b85645..8b36d22a8 100644 --- a/src/modules/site-v2/templates/_includes/phenotype-database-table.html +++ b/src/modules/site-v2/templates/_includes/phenotype-database-table.html @@ -183,13 +183,7 @@ .fail((error) => { console.error(error) $('#{{ offcanvas_id }} .btn-close').click() - $('html').animate({ - scrollTop: $('body').offset().top - }, 100); - if (error.responseJSON && error.responseJSON.message) { - $('.alert').remove() - flash_message(error.responseJSON.message) - } + flashErrorResponse(error.responseJSON, 'There was a problem retrieving this trait. Please try again later.') }) }) diff --git a/src/modules/site-v2/templates/_includes/trait-library-table.html b/src/modules/site-v2/templates/_includes/trait-library-table.html index e722a187f..f686a9da1 100644 --- a/src/modules/site-v2/templates/_includes/trait-library-table.html +++ b/src/modules/site-v2/templates/_includes/trait-library-table.html @@ -227,13 +227,7 @@

Full Description

.fail((error) => { console.error(error) $('#{{ offcanvas_id }} .btn-close').click() - $('html').animate({ - scrollTop: $('body').offset().top - }, 100); - if (error.responseJSON && error.responseJSON.message) { - $('.alert').remove() - flash_message(error.responseJSON.message) - } + flashErrorResponse(error.responseJSON, 'There was a problem retrieving this trait. Please try again later.') }) }) diff --git a/src/modules/site-v2/templates/_scripts/utils.js b/src/modules/site-v2/templates/_scripts/utils.js index 8ac403d9e..936299c10 100644 --- a/src/modules/site-v2/templates/_scripts/utils.js +++ b/src/modules/site-v2/templates/_scripts/utils.js @@ -167,6 +167,27 @@ function flash_message(message, full_msg_link=null, full_msg_body=null) { } +function flashErrorResponse(responseJSON, backupMessage, scrollToTop = true) { + + // Optionally scroll to the top of the page to highlight the error message + if (scrollToTop) { + $('html').animate({ + scrollTop: $('body').offset().top, + }, 100); + } + + // If the response JSON contains a message, flash all the details it has + if (responseJSON && responseJSON.message) { + flash_message(responseJSON.message, responseJSON.full_msg_link, responseJSON.full_msg_body); + } + + // Otherwise, flash a backup message + else { + flash_message(backupMessage); + } +} + + /* Format a date as YYYY-MM-DD. */ function formatDate(d) { diff --git a/src/modules/site-v2/templates/tools/genetic_mapping/mapping.html b/src/modules/site-v2/templates/tools/genetic_mapping/mapping.html index d37f9dfd0..faa4ccab8 100644 --- a/src/modules/site-v2/templates/tools/genetic_mapping/mapping.html +++ b/src/modules/site-v2/templates/tools/genetic_mapping/mapping.html @@ -133,7 +133,7 @@

Example Data

{% from "_scripts/submit-job.js" import def_submit_job %} {% from "_includes/macros.html" import update_species_field %} From 804358877c53816a00f42a13bc5cc256b6028ded Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Mon, 6 May 2024 14:17:58 -0500 Subject: [PATCH 68/73] Fix: Initialize announcement style properly --- src/modules/site-v2/base/views/admin/admin.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/modules/site-v2/base/views/admin/admin.py b/src/modules/site-v2/base/views/admin/admin.py index b6be256df..78a08bffd 100755 --- a/src/modules/site-v2/base/views/admin/admin.py +++ b/src/modules/site-v2/base/views/admin/admin.py @@ -73,6 +73,8 @@ def announcements_edit(entity_id=None): # Initialize the form with the existing object (or None if creating new) form = AnnouncementForm(obj=announcement) + if announcement: + form.style.data = announcement['style'].name return render_template('admin/announcements/edit.html', **{ 'title': ('Edit' if announcement else 'Create') + ' Announcement', From c80c1579dc7a0067b28dbd1c88b058158285697a Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Mon, 6 May 2024 15:52:53 -0500 Subject: [PATCH 69/73] Look up announcement objects using view decorator --- .../site-v2/base/utils/view_decorators.py | 49 ++++++++++++++++++- src/modules/site-v2/base/views/admin/admin.py | 11 ++--- .../site-v2/base/views/api/notifications.py | 23 ++------- 3 files changed, 56 insertions(+), 27 deletions(-) diff --git a/src/modules/site-v2/base/utils/view_decorators.py b/src/modules/site-v2/base/utils/view_decorators.py index 5174d26da..115a6e817 100644 --- a/src/modules/site-v2/base/utils/view_decorators.py +++ b/src/modules/site-v2/base/utils/view_decorators.py @@ -9,7 +9,7 @@ from base.utils.tools import lookup_report, get_upload_err_msg from constants import TOOL_INPUT_DATA_VALID_FILE_EXTENSIONS -from caendr.models.datastore import DatasetRelease, Species +from caendr.models.datastore import DatasetRelease, Species, Entity from caendr.models.error import NotFoundError, ReportLookupError, EmptyReportDataError, EmptyReportResultsError, FileUploadError, DataValidationError from caendr.models.job_pipeline import JobPipeline from caendr.services.logger import logger @@ -147,6 +147,53 @@ def decorator(*args, report_id, **kwargs): +def parse_entity_id(entity_class: Type[Entity], required: bool = True, kw_name_id: str = 'entity_id', kw_name_entity: str = 'entity'): + ''' + Given an entity ID as a keyword argument, lookup and inject the entity with that ID. + ''' + def wrapper(f): + @wraps(f) + def decorator(*args, **kwargs): + + # Extract the entity ID from the keywords using the given name + entity_id = kwargs.get(kw_name_id) + kwargs = { key: val for key, val in kwargs.items() if key != kw_name_id } + + # If no ID given, optionally raise error + if entity_id is None: + if required: + abort(404) + else: + e = None + + # If ID given, try retrieving entity from datastore + else: + try: + e = entity_class.get_ds(entity_id) + + # If not found, abort with 404 + except NotFoundError as ex: + logger.error(f'Could not find {entity_class.kind} with ID {entity_id}: {ex}') + abort(404, description = f'Could not find an {entity_class.kind} object with the given ID.') + + # General error: include default message + except Exception as ex: + logger.error(f'Error retrieving {entity_class.kind} with ID {entity_id}: {ex}') + abort(500, description = 'Something went wrong') + + # If entity does not exist, abort with 404 + if e is None: + logger.error(f'Could not find {entity_class.kind} with ID {entity_id}') + abort(404, description = f'Could not find an {entity_class.kind} object with the given ID.') + + # Inject retrieved entity into function call + return f(*args, **{kw_name_entity: e}, **kwargs) + + return decorator + return wrapper + + + def validate_form(form_class: Type[FlaskForm], from_json: bool = False, err_msg: str = None, flash_err_msg: bool = True): ''' Parse the request form into the given form type, validate the fields, and inject the data as a dict. diff --git a/src/modules/site-v2/base/views/admin/admin.py b/src/modules/site-v2/base/views/admin/admin.py index 78a08bffd..03b123595 100755 --- a/src/modules/site-v2/base/views/admin/admin.py +++ b/src/modules/site-v2/base/views/admin/admin.py @@ -3,6 +3,7 @@ from config import config from base.forms import AnnouncementForm from base.utils.auth import admin_required +from base.utils.view_decorators import parse_entity_id from caendr.models.datastore import Announcement from caendr.services.cloud.secret import get_secret @@ -59,17 +60,11 @@ def announcements(): @admin_bp.route('/announcements/create', methods=['GET']) @admin_bp.route('/announcements/edit/', methods=['GET']) @admin_required() -def announcements_edit(entity_id=None): +@parse_entity_id(Announcement, required=False, kw_name_id='entity_id', kw_name_entity='announcement') +def announcements_edit(announcement: Announcement = None): ''' Manage the site announcements. ''' - if entity_id: - try: - announcement = Announcement.get_ds(entity_id) - except: - abort(500) - else: - announcement = None # Initialize the form with the existing object (or None if creating new) form = AnnouncementForm(obj=announcement) diff --git a/src/modules/site-v2/base/views/api/notifications.py b/src/modules/site-v2/base/views/api/notifications.py index 25e6a92d2..d10fca34e 100644 --- a/src/modules/site-v2/base/views/api/notifications.py +++ b/src/modules/site-v2/base/views/api/notifications.py @@ -7,6 +7,8 @@ from base.utils.tools import lookup_report from base.views.tools import pairwise_indel_finder_bp, genetic_mapping_bp, heritability_calculator_bp +from base.utils.view_decorators import parse_entity_id + from caendr.models.datastore import NemascanReport, HeritabilityReport, IndelPrimerReport, Announcement from caendr.models.error import ReportLookupError, NotFoundError from caendr.models.status import JobStatus @@ -88,20 +90,6 @@ def job_finish(kind, id, status): # -def get_announcement(entity_id): - ''' - Helper function to handle looking up announcement entity. - TODO: This can be done with new decorator(s) in another branch. - ''' - try: - return Announcement.get_ds(entity_id) - except NotFoundError: - abort(404) - except Exception as ex: - logger.error(f'Error retrieving announcement {entity_id}: {ex}') - abort(500) - - @api_notifications_bp.route('/announcements', methods=['GET']) @admin_required() @jsonify_request @@ -117,8 +105,9 @@ def announcement_list(): @api_notifications_bp.route('/announcement', methods=['POST']) @api_notifications_bp.route('/announcement/', methods=['GET', 'PATCH', 'DELETE']) @admin_required() +@parse_entity_id(Announcement, required=False, kw_name_id='entity_id', kw_name_entity='announcement') @jsonify_request -def announcement(entity_id: str = None): +def announcement(announcement: Announcement = None): ''' Manage one of the site announcements. @@ -132,7 +121,7 @@ def announcement(entity_id: str = None): # GET Request # Return the list of announcement entities if request.method == 'GET': - return get_announcement(entity_id).serialize() + return announcement.serialize() # POST Request # Create a new announcement, and return its unique ID @@ -170,7 +159,6 @@ def announcement(entity_id: str = None): new_values['active'] = request.form.get('active') == 'true' # Update the announcement object - announcement = get_announcement(entity_id) announcement.set_properties(**new_values) announcement.save() return { 'id': announcement.name } @@ -178,7 +166,6 @@ def announcement(entity_id: str = None): # DELETE Request # Lookup the desired announcement and soft delete it if request.method == 'DELETE': - announcement = get_announcement(entity_id) announcement.soft_delete() announcement.save() return {}, 200 From 28ff2fd7001f14d530d1c3ed0bc06b21fc8f1f67 Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Mon, 6 May 2024 16:10:21 -0500 Subject: [PATCH 70/73] Parse announcement form using `validate_form` decorator Includes validation code for string fields. Part of unifying form parsing into one decorator that handles all cases / exceptions in the same way. --- .../site-v2/base/utils/view_decorators.py | 9 +++++- .../site-v2/base/views/api/notifications.py | 28 ++++++------------- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/src/modules/site-v2/base/utils/view_decorators.py b/src/modules/site-v2/base/utils/view_decorators.py index 115a6e817..4e6f094ba 100644 --- a/src/modules/site-v2/base/utils/view_decorators.py +++ b/src/modules/site-v2/base/utils/view_decorators.py @@ -194,7 +194,7 @@ def decorator(*args, **kwargs): -def validate_form(form_class: Type[FlaskForm], from_json: bool = False, err_msg: str = None, flash_err_msg: bool = True): +def validate_form(form_class: Type[FlaskForm], from_json: bool = False, err_msg: str = None, flash_err_msg: bool = True, methods = None): ''' Parse the request form into the given form type, validate the fields, and inject the data as a dict. @@ -211,6 +211,8 @@ def validate_form(form_class: Type[FlaskForm], from_json: bool = False, err_msg: - `from_json`: If `True`, use the request `.get_json()` as the fields instead. - `err_msg`: An error message to add to the response if validation fails. - `flash_err_msg`: If `True`, flashes the `err_msg` in addition to returning it. + - `methods`: A list of request methods to expect a form from. If `None`, checks for a form in all requests; + otherwise, does not try to extract a form from any request method not in the list. ''' def wrapper(f): @@ -228,6 +230,11 @@ def decorator(*args, **kwargs): # If user is admin, allow them to bypass cache with URL variable no_cache = bool(user_is_admin() and request.args.get("nocache", False)) + # If a list of methods is provided, make sure the current request method is in it + # If it's not, then there should not be any form data for this request + if methods is not None and request.method not in methods: + return f(*args, form_data=None, no_cache=no_cache, **kwargs) + # Pull the raw data from either the form or the JSON body raw_data = request.get_json() if from_json else request.form diff --git a/src/modules/site-v2/base/views/api/notifications.py b/src/modules/site-v2/base/views/api/notifications.py index d10fca34e..50455ae2e 100644 --- a/src/modules/site-v2/base/views/api/notifications.py +++ b/src/modules/site-v2/base/views/api/notifications.py @@ -7,7 +7,7 @@ from base.utils.tools import lookup_report from base.views.tools import pairwise_indel_finder_bp, genetic_mapping_bp, heritability_calculator_bp -from base.utils.view_decorators import parse_entity_id +from base.utils.view_decorators import parse_entity_id, validate_form from caendr.models.datastore import NemascanReport, HeritabilityReport, IndelPrimerReport, Announcement from caendr.models.error import ReportLookupError, NotFoundError @@ -106,8 +106,9 @@ def announcement_list(): @api_notifications_bp.route('/announcement/', methods=['GET', 'PATCH', 'DELETE']) @admin_required() @parse_entity_id(Announcement, required=False, kw_name_id='entity_id', kw_name_entity='announcement') +@validate_form(AnnouncementForm, methods=['POST', 'PATCH']) @jsonify_request -def announcement(announcement: Announcement = None): +def announcement(announcement: Announcement = None, form_data = None, no_cache: bool = False): ''' Manage one of the site announcements. @@ -126,15 +127,8 @@ def announcement(announcement: Announcement = None): # POST Request # Create a new announcement, and return its unique ID if request.method == 'POST': - - # Validate form - # TODO: Clean / validate values - form = AnnouncementForm(request.form) - if not form.validate(): - abort(400) - new_announcement = Announcement(**{ - prop: request.form.get(prop) for prop in Announcement.get_props_set() + prop: form_data.get(prop) for prop in Announcement.get_props_set() }) new_announcement.save() return { 'id': new_announcement.name } @@ -143,20 +137,14 @@ def announcement(announcement: Announcement = None): # Lookup the desired announcement and update its properties if request.method == 'PATCH': - # Validate form - # TODO: Clean / validate values - form = AnnouncementForm(request.form) - if not form.validate(): - abort(400) - # Extract the new property values from the form, casting "active" to a bool new_values = { - prop: request.form.get(prop) + prop: form_data.get(prop) for prop in Announcement.get_props_set() - if request.form.get(prop) is not None + if form_data.get(prop) is not None } - if request.form.get('active') is not None: - new_values['active'] = request.form.get('active') == 'true' + if form_data.get('active') is not None: + new_values['active'] = form_data.get('active') == 'true' # Update the announcement object announcement.set_properties(**new_values) From 77f0d9dc34f0a3f995556009d2521b60f813328b Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Tue, 7 May 2024 11:28:16 -0500 Subject: [PATCH 71/73] Bugfix: Correct return value for templates that can't show announcements --- src/modules/site-v2/application.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/modules/site-v2/application.py b/src/modules/site-v2/application.py index c23b6f809..210700880 100755 --- a/src/modules/site-v2/application.py +++ b/src/modules/site-v2/application.py @@ -368,8 +368,9 @@ def inject_announcements(): ''' # Make sure this request can show announcements + # If not, we have to return an empty object, since this is a context processor if not can_show_announcements(request, check_session_flags=True): - return + return {} # Make sure site announcements set exists # Store as a set to help prevent flashing duplicate announcements From 9d7e3083f565f1072f0f7c709e6a544d0b3187bb Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Tue, 7 May 2024 13:25:16 -0500 Subject: [PATCH 72/73] Make style optional in form --- src/modules/site-v2/base/forms/forms.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/modules/site-v2/base/forms/forms.py b/src/modules/site-v2/base/forms/forms.py index 2e7e403b6..79b37fd00 100755 --- a/src/modules/site-v2/base/forms/forms.py +++ b/src/modules/site-v2/base/forms/forms.py @@ -124,11 +124,10 @@ class RecoverUserForm(FlaskForm): class AnnouncementForm(FlaskForm): """ Edit form for site announcements """ - # active = BooleanField('Active', widget=CheckboxInput()) active = BooleanField('Active') content = StringField('Content', [Optional()]) url_list = TextAreaField('URL Patterns', [Optional()]) - style = SelectField('Style', choices=[(x.name, x.value) for x in AnnouncementType]) + style = SelectField('Style', [Optional()], choices=[(x.name, x.value) for x in AnnouncementType]) class MarkdownForm(FlaskForm): """ markdown editing form """ From 7863dc9e6dbb8faef2dbffd0ec9bdbefee492981 Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Tue, 7 May 2024 15:23:20 -0500 Subject: [PATCH 73/73] Fix `markdown_to_html` to `markdownToHTML` --- .../templates/_includes/phenotype-database-table.html | 8 +++++++- src/modules/site-v2/templates/_scripts/utils.js | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/modules/site-v2/templates/_includes/phenotype-database-table.html b/src/modules/site-v2/templates/_includes/phenotype-database-table.html index d8f659dbc..10115e88b 100644 --- a/src/modules/site-v2/templates/_includes/phenotype-database-table.html +++ b/src/modules/site-v2/templates/_includes/phenotype-database-table.html @@ -153,7 +153,13 @@

} }, {"data": "trait_name_caendr"}, - {"data": "description_long"}, + { + "data": "description_long", + // TODO: Should this be Markdown? + // "render": function(data, type, row) { + // return markdownToHTML(data); + // }, + }, { "data": "Upload Date", "render": function(data, type, row) { diff --git a/src/modules/site-v2/templates/_scripts/utils.js b/src/modules/site-v2/templates/_scripts/utils.js index 98dd2220b..65a17303a 100644 --- a/src/modules/site-v2/templates/_scripts/utils.js +++ b/src/modules/site-v2/templates/_scripts/utils.js @@ -107,7 +107,7 @@ function create_node(html) { // Convert text from Markdown to HTML using Toast UI // This protects against code injection -function markdown_to_html(text) { +function markdownToHTML(text) { const editor = new toastui.Editor({ el: document.createElement('div'), initialValue: text, @@ -145,7 +145,7 @@ function flash_message(message, full_msg_link=null, full_msg_body=null) { // Create as a new DOM node, and insert the desired message as formatted (cleaned) HTML const node = create_node(raw_html); - node.firstElementChild.innerHTML = markdown_to_html(message); + node.firstElementChild.innerHTML = markdownToHTML(message); // If both full message fields are provided, add as a link & collapse dropdown if (full_msg_link && full_msg_body) {