From 1dc03b4ab05408e55d313568b70f03522a2736ec Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Thu, 4 Apr 2024 13:45:36 -0500 Subject: [PATCH 01/44] Move trait querying to API --- src/modules/site-v2/base/views/api/trait.py | 46 +++++++++++++++---- .../base/views/tools/phenotype_database.py | 30 +----------- .../_includes/phenotype-database-table.html | 2 +- 3 files changed, 38 insertions(+), 40 deletions(-) diff --git a/src/modules/site-v2/base/views/api/trait.py b/src/modules/site-v2/base/views/api/trait.py index ccc3080eb..b043f7510 100644 --- a/src/modules/site-v2/base/views/api/trait.py +++ b/src/modules/site-v2/base/views/api/trait.py @@ -2,6 +2,9 @@ from caendr.services.logger import logger from extensions import cache +from caendr.api.phenotype import query_phenotype_metadata, get_trait, filter_trait_query_by_text, filter_trait_query_by_tags +from caendr.services.cloud.postgresql import rollback_on_error_handler + from caendr.models.datastore import TraitFile, Species from caendr.models.error import NotFoundError from caendr.utils.json import jsonify_request @@ -17,23 +20,46 @@ def filter_trait_files(tf): return tf.is_public and not tf.is_bulk_file -@api_trait_bp.route('/all', methods=['GET']) +@api_trait_bp.route('/query', methods=['POST']) @cache.memoize(60*60) @jsonify_request -def query_all(): +def query(): ''' Query all trait files, optionally split into different lists based on species. ''' - # Optionally split into an object w species names for keys - if request.args.get('split_by_species', False): - return { - species: [ tf.serialize() for tf in tf_list ] - for species, tf_list in TraitFile.query_ds_split_species(filter=filter_trait_files).items() - } + # Get query filters + selected_tags = request.json.get('selected_tags', []) + search_val = request.json.get('search_val', '') + + # Get query pagination values + page = int(request.json.get('page', 1)) + current_page = int(request.json.get('current_page', 1)) + per_page = 10 + + # Filter by search value and tags, if provided + query = query_phenotype_metadata() + query = filter_trait_query_by_text(query, search_val) + query = filter_trait_query_by_tags(query, selected_tags) + + # Paginate the query, rolling back on error + with rollback_on_error_handler(): + pagination = query.paginate(page=page, per_page=per_page) - # Otherwise, return all trait files in one list - return [ tf.serialize() for tf in TraitFile.query_ds(ignore_errs=True) if filter_trait_files(tf) ] + # Format return data + return { + 'data': [ + tr.to_json() for tr in pagination.items + ], + 'pagination': { + 'has_next': pagination.has_next, + 'has_prev': pagination.has_prev, + 'prev_num': pagination.prev_num, + 'next_num': pagination.next_num, + 'total_pages': pagination.pages, + 'current_page': current_page + }, + } @api_trait_bp.route('/', methods=['GET']) diff --git a/src/modules/site-v2/base/views/tools/phenotype_database.py b/src/modules/site-v2/base/views/tools/phenotype_database.py index cead747ad..f47b8a967 100644 --- a/src/modules/site-v2/base/views/tools/phenotype_database.py +++ b/src/modules/site-v2/base/views/tools/phenotype_database.py @@ -79,40 +79,12 @@ def phenotype_database(): """ form = EmptyForm() - # Get the list of traits for non-bulk files + # Get the list of unique tags try: query = query_phenotype_metadata() - - # Get the list of unique tags tags = [ tr.tags.split(', ') for tr in query if tr.tags ] tags_list = [tg for tr_tag in tags for tg in tr_tag] unique_tags = list(set(tags_list)) - - if request.method == 'POST': - selected_tags = request.json.get('selected_tags', []) - search_val = request.json.get('search_val', '') - page = int(request.json.get('page', 1)) - current_page = int(request.json.get('current_page', 1)) - per_page = 10 - - # Filter by search value and tags, if provided - query = filter_trait_query_by_text(query, search_val) - query = filter_trait_query_by_tags(query, selected_tags) - - # Paginate the query, rolling back on error - with rollback_on_error_handler(): - pagination = query.paginate(page=page, per_page=per_page) - - json_data = [ tr.to_json() for tr in pagination.items ] - pagination_data = { - 'has_next': pagination.has_next, - 'has_prev': pagination.has_prev, - 'prev_num': pagination.prev_num, - 'next_num': pagination.next_num, - 'total_pages': pagination.pages, - 'current_page': current_page - } - return jsonify({'data': json_data, 'pagination': pagination_data }) except Exception as ex: logger.error(f'Failed to retrieve the list of traits: {ex}') diff --git a/src/modules/site-v2/templates/_includes/phenotype-database-table.html b/src/modules/site-v2/templates/_includes/phenotype-database-table.html index cefbe1512..60db7023c 100644 --- a/src/modules/site-v2/templates/_includes/phenotype-database-table.html +++ b/src/modules/site-v2/templates/_includes/phenotype-database-table.html @@ -280,7 +280,7 @@

} async function sendPostRequest(data) { - const resp = await fetch("{{ url_for('phenotype_database.phenotype_database') }}", { + const resp = await fetch("{{ url_for('api_trait.query') }}", { method: "POST", headers: { "Content-Type": "application/json", From 29f4c825a029bb9151648e5ff8a94b045fffbad9 Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Thu, 4 Apr 2024 16:12:29 -0500 Subject: [PATCH 02/44] Filter non-Zhang traits by "dataset" rather than bulk_file directly --- src/modules/site-v2/base/views/api/trait.py | 7 +++-- .../_includes/phenotype-database-table.html | 2 +- src/pkg/caendr/caendr/api/phenotype.py | 27 ++++++++++++------- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/modules/site-v2/base/views/api/trait.py b/src/modules/site-v2/base/views/api/trait.py index b043f7510..424369694 100644 --- a/src/modules/site-v2/base/views/api/trait.py +++ b/src/modules/site-v2/base/views/api/trait.py @@ -31,14 +31,17 @@ def query(): # Get query filters selected_tags = request.json.get('selected_tags', []) search_val = request.json.get('search_val', '') + filter_dataset = request.json.get('dataset', None) # Get query pagination values page = int(request.json.get('page', 1)) current_page = int(request.json.get('current_page', 1)) per_page = 10 - # Filter by search value and tags, if provided - query = query_phenotype_metadata() + # Create the initial query + query = query_phenotype_metadata(dataset=filter_dataset) + + # Filter by search values, if provided query = filter_trait_query_by_text(query, search_val) query = filter_trait_query_by_tags(query, selected_tags) diff --git a/src/modules/site-v2/templates/_includes/phenotype-database-table.html b/src/modules/site-v2/templates/_includes/phenotype-database-table.html index 60db7023c..12559812d 100644 --- a/src/modules/site-v2/templates/_includes/phenotype-database-table.html +++ b/src/modules/site-v2/templates/_includes/phenotype-database-table.html @@ -286,7 +286,7 @@

"Content-Type": "application/json", 'X-CSRF-TOKEN': data.csrf_token }, - body: JSON.stringify(data) + body: JSON.stringify({...data, 'dataset': 'caendr'}), }) if (resp.ok) { diff --git a/src/pkg/caendr/caendr/api/phenotype.py b/src/pkg/caendr/caendr/api/phenotype.py index dd766d488..71e2069fa 100644 --- a/src/pkg/caendr/caendr/api/phenotype.py +++ b/src/pkg/caendr/caendr/api/phenotype.py @@ -1,5 +1,6 @@ import bleach import os +from typing import Optional, Union from sqlalchemy import or_, func @@ -9,7 +10,12 @@ from caendr.services.cloud.postgresql import rollback_on_error -def query_phenotype_metadata(is_bulk_file=False, include_values=False, species: str = None): +def query_phenotype_metadata( + include_values = False, + is_bulk_file: Optional[bool] = None, + species: Optional[str] = None, + dataset: Optional[str] = None, +): """ Returns the list of traits with the corresponding metadata. @@ -19,16 +25,19 @@ def query_phenotype_metadata(is_bulk_file=False, include_values=False, species: - phenotype_values: if True, include phenotype values for each trait - species: filters by species """ + + # Create the initial query query = PhenotypeMetadata.query - # Get traits for bulk file - if is_bulk_file: - query = query.filter_by(is_bulk_file=True) - else: - # Get traits for non-bulk files - query = query.filter_by(is_bulk_file=False) + # Optionally query by bulk file + if is_bulk_file is not None: + query = query.filter_by(is_bulk_file=bool(is_bulk_file)) + + # Optionally query by dataset + if dataset is not None: + query = query.filter_by(dataset=dataset) - # Query by species + # Optionally query by species if species is not None: if species in Species.all().keys(): query = query.filter_by(species_name=species) @@ -37,7 +46,7 @@ def query_phenotype_metadata(is_bulk_file=False, include_values=False, species: # Include phenotype values for traits if include_values: - query = query.join(PhenotypeMetadata.phenotype_values) + query = query.join(PhenotypeMetadata.phenotype_values) return query From 33fe4fdc53987fb5072602b25982f3475956294a Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Thu, 4 Apr 2024 16:16:43 -0500 Subject: [PATCH 03/44] Move species filter to helper func w better validation --- src/pkg/caendr/caendr/api/phenotype.py | 33 ++++++++++++++++++++------ 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/src/pkg/caendr/caendr/api/phenotype.py b/src/pkg/caendr/caendr/api/phenotype.py index 71e2069fa..68dba2eaa 100644 --- a/src/pkg/caendr/caendr/api/phenotype.py +++ b/src/pkg/caendr/caendr/api/phenotype.py @@ -5,7 +5,6 @@ from sqlalchemy import or_, func from caendr.models.datastore import Species -from caendr.models.error import BadRequestError from caendr.models.sql import PhenotypeMetadata from caendr.services.cloud.postgresql import rollback_on_error @@ -13,7 +12,7 @@ def query_phenotype_metadata( include_values = False, is_bulk_file: Optional[bool] = None, - species: Optional[str] = None, + species: Optional[Union[Species, str]] = None, dataset: Optional[str] = None, ): """ @@ -38,11 +37,8 @@ def query_phenotype_metadata( query = query.filter_by(dataset=dataset) # Optionally query by species - if species is not None: - if species in Species.all().keys(): - query = query.filter_by(species_name=species) - else: - raise BadRequestError(f'Unrecognized species ID "{species}".') + # None values handled in function + filter_trait_query_by_species(query, species) # Include phenotype values for traits if include_values: @@ -85,3 +81,26 @@ def filter_trait_query_by_tags(query, tags): PhenotypeMetadata.tags.ilike(f"%{bleach.clean(tag)}%") for tag in tags )) return query + + +def filter_trait_query_by_species(query, species: Optional[Union[Species, str]]): + ''' + Filter by species. + + If species is invalid, passes error raised by `Species` class. + ''' + if species is not None: + + # Cast string values to Species object + if isinstance(species, str): + species = Species.from_name(species) + + # Validate species type + if not isinstance(species, Species): + raise ValueError(f'Expected species identifier, got {species}') + + # Filter by the species name + query = query.filter_by(species_name=species.name) + + # Return the (possibly filtered) query + return query From c344052b42c2e6f7b830b6f08169516f1e9e5245 Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Thu, 4 Apr 2024 16:23:11 -0500 Subject: [PATCH 04/44] Add section & docstring comments, typing --- src/modules/site-v2/base/views/api/trait.py | 13 ++++++++++++- src/pkg/caendr/caendr/api/phenotype.py | 21 ++++++++++++++++++--- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/modules/site-v2/base/views/api/trait.py b/src/modules/site-v2/base/views/api/trait.py index 424369694..8d0344bc8 100644 --- a/src/modules/site-v2/base/views/api/trait.py +++ b/src/modules/site-v2/base/views/api/trait.py @@ -16,10 +16,21 @@ +# +# Helper Functions +# + + def filter_trait_files(tf): return tf.is_public and not tf.is_bulk_file + +# +# Query Endpoints +# + + @api_trait_bp.route('/query', methods=['POST']) @cache.memoize(60*60) @jsonify_request @@ -28,7 +39,7 @@ def query(): Query all trait files, optionally split into different lists based on species. ''' - # Get query filters + # Get query filters (search parameters) selected_tags = request.json.get('selected_tags', []) search_val = request.json.get('search_val', '') filter_dataset = request.json.get('dataset', None) diff --git a/src/pkg/caendr/caendr/api/phenotype.py b/src/pkg/caendr/caendr/api/phenotype.py index 68dba2eaa..5f14c1077 100644 --- a/src/pkg/caendr/caendr/api/phenotype.py +++ b/src/pkg/caendr/caendr/api/phenotype.py @@ -1,6 +1,6 @@ import bleach import os -from typing import Optional, Union +from typing import Optional, Union, Iterable from sqlalchemy import or_, func @@ -58,7 +58,19 @@ def get_trait(trait_name): return PhenotypeMetadata.query.get(trait_name) -def filter_trait_query_by_text(query, search_val): + +# +# Query Filters +# +# Conditionally add common filter types to a query object +# + + +def filter_trait_query_by_text(query, search_val: Optional[str]): + ''' + Filter by a text search value on the text fields. + Generic "search" functionality. + ''' print(search_val) if search_val and len(search_val): query = query.filter( @@ -75,7 +87,10 @@ def filter_trait_query_by_text(query, search_val): return query -def filter_trait_query_by_tags(query, tags): +def filter_trait_query_by_tags(query, tags: Optional[Iterable[str]]): + ''' + Filter by trait tags. + ''' if len(tags): query = query.filter(or_( PhenotypeMetadata.tags.ilike(f"%{bleach.clean(tag)}%") for tag in tags From 4a7eab060e7a3b7a49c84f2910031008dec2a228 Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Fri, 5 Apr 2024 13:23:03 -0500 Subject: [PATCH 05/44] Use bleach to clean request params --- src/modules/site-v2/base/views/api/trait.py | 25 ++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/modules/site-v2/base/views/api/trait.py b/src/modules/site-v2/base/views/api/trait.py index 8d0344bc8..03a8165d5 100644 --- a/src/modules/site-v2/base/views/api/trait.py +++ b/src/modules/site-v2/base/views/api/trait.py @@ -1,3 +1,5 @@ +import bleach + from flask import request, Blueprint, abort from caendr.services.logger import logger from extensions import cache @@ -25,6 +27,19 @@ def filter_trait_files(tf): return tf.is_public and not tf.is_bulk_file +def get_clean(source, key, value=None, _type=None): + v = source.get(key, value) + + # Clean value + if isinstance(v, str): v = bleach.clean(v) + elif isinstance(v, list): v = [ bleach.clean(x) for x in v ] + + # Optional typecasting + if _type: v = _type(v) + + return v + + # # Query Endpoints @@ -40,13 +55,13 @@ def query(): ''' # Get query filters (search parameters) - selected_tags = request.json.get('selected_tags', []) - search_val = request.json.get('search_val', '') - filter_dataset = request.json.get('dataset', None) + selected_tags = get_clean(request.json, 'selected_tags', []) + search_val = get_clean(request.json, 'search_val', '').lower() + filter_dataset = get_clean(request.json, 'dataset') # Get query pagination values - page = int(request.json.get('page', 1)) - current_page = int(request.json.get('current_page', 1)) + page = get_clean(request.json, 'page', 1, _type=int) + current_page = get_clean(request.json, 'current_page', 1, _type=int) per_page = 10 # Create the initial query From 18f8acc824ddf50823526edae275e49f2fc150c8 Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Fri, 5 Apr 2024 14:28:39 -0500 Subject: [PATCH 06/44] Minor query filtering bugfixes - Actually add the species filter to the `query` - Make sure `tags` is defined before trying to take length --- src/pkg/caendr/caendr/api/phenotype.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pkg/caendr/caendr/api/phenotype.py b/src/pkg/caendr/caendr/api/phenotype.py index 5f14c1077..83d256d5c 100644 --- a/src/pkg/caendr/caendr/api/phenotype.py +++ b/src/pkg/caendr/caendr/api/phenotype.py @@ -38,7 +38,7 @@ def query_phenotype_metadata( # Optionally query by species # None values handled in function - filter_trait_query_by_species(query, species) + query = filter_trait_query_by_species(query, species) # Include phenotype values for traits if include_values: @@ -91,7 +91,7 @@ def filter_trait_query_by_tags(query, tags: Optional[Iterable[str]]): ''' Filter by trait tags. ''' - if len(tags): + if tags and len(tags): query = query.filter(or_( PhenotypeMetadata.tags.ilike(f"%{bleach.clean(tag)}%") for tag in tags )) From cdc252aca1312b5aad8e149b96c74608802a68cc Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Fri, 5 Apr 2024 14:31:11 -0500 Subject: [PATCH 07/44] Convenience func to apply multiple query filters at once --- src/modules/site-v2/base/views/api/trait.py | 5 ++--- src/pkg/caendr/caendr/api/phenotype.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/modules/site-v2/base/views/api/trait.py b/src/modules/site-v2/base/views/api/trait.py index 03a8165d5..7b3d82ca6 100644 --- a/src/modules/site-v2/base/views/api/trait.py +++ b/src/modules/site-v2/base/views/api/trait.py @@ -4,7 +4,7 @@ from caendr.services.logger import logger from extensions import cache -from caendr.api.phenotype import query_phenotype_metadata, get_trait, filter_trait_query_by_text, filter_trait_query_by_tags +from caendr.api.phenotype import query_phenotype_metadata, get_trait, filter_trait_query from caendr.services.cloud.postgresql import rollback_on_error_handler from caendr.models.datastore import TraitFile, Species @@ -68,8 +68,7 @@ def query(): query = query_phenotype_metadata(dataset=filter_dataset) # Filter by search values, if provided - query = filter_trait_query_by_text(query, search_val) - query = filter_trait_query_by_tags(query, selected_tags) + query = filter_trait_query(query, search_val=search_val, tags=selected_tags) # Paginate the query, rolling back on error with rollback_on_error_handler(): diff --git a/src/pkg/caendr/caendr/api/phenotype.py b/src/pkg/caendr/caendr/api/phenotype.py index 83d256d5c..627e2ab02 100644 --- a/src/pkg/caendr/caendr/api/phenotype.py +++ b/src/pkg/caendr/caendr/api/phenotype.py @@ -66,6 +66,21 @@ def get_trait(trait_name): # +def filter_trait_query( + query, + search_val: Optional[str] = None, + tags: Optional[Iterable[str]] = None, + species: Optional[Union[Species, str]] = None, + ): + ''' + Combined filtering function. + ''' + query = filter_trait_query_by_text(query, search_val) + query = filter_trait_query_by_tags(query, tags) + query = filter_trait_query_by_species(query, species) + return query + + def filter_trait_query_by_text(query, search_val: Optional[str]): ''' Filter by a text search value on the text fields. From 705b19b527ef765494fb206ccef1e3d18dd6527e Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Fri, 5 Apr 2024 14:38:15 -0500 Subject: [PATCH 08/44] Return better errors from API endpoint --- src/modules/site-v2/base/views/api/trait.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/modules/site-v2/base/views/api/trait.py b/src/modules/site-v2/base/views/api/trait.py index 7b3d82ca6..cfbda87e3 100644 --- a/src/modules/site-v2/base/views/api/trait.py +++ b/src/modules/site-v2/base/views/api/trait.py @@ -1,4 +1,5 @@ import bleach +from functools import wraps from flask import request, Blueprint, abort from caendr.services.logger import logger @@ -40,6 +41,21 @@ def get_clean(source, key, value=None, _type=None): return v +def query_traits_error_handler(f): + ''' + Wrapper for trait query endpoints. + If query raises an error, returns an empty response and a `500` error. + ''' + @wraps(f) + def inner(*args, **kwargs): + try: + return f(*args, **kwargs) + except Exception as ex: + logger.error(f'Failed to retrieve the list of traits: {ex}') + return {}, 500 + return inner + + # # Query Endpoints @@ -48,6 +64,7 @@ def get_clean(source, key, value=None, _type=None): @api_trait_bp.route('/query', methods=['POST']) @cache.memoize(60*60) +@query_traits_error_handler @jsonify_request def query(): ''' From 9c30fbb21e097e082466d545e087158eef5a61a6 Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Fri, 5 Apr 2024 14:43:42 -0500 Subject: [PATCH 09/44] Adapt Zhang traits endpoint to API query Instead of splitting on dataset ('caendr' and 'zhang'), there are now two endpoints for different return formats: SQL-style pagination and JS DataTable-style pagination. Both endpoints accept a `dataset` URL var that filters the query to that dataset. --- src/modules/site-v2/base/views/api/trait.py | 55 +++++++++++++++++-- .../base/views/tools/phenotype_database.py | 42 -------------- .../_includes/phenotype-database-table.html | 6 +- 3 files changed, 54 insertions(+), 49 deletions(-) diff --git a/src/modules/site-v2/base/views/api/trait.py b/src/modules/site-v2/base/views/api/trait.py index cfbda87e3..3a4b8f1d1 100644 --- a/src/modules/site-v2/base/views/api/trait.py +++ b/src/modules/site-v2/base/views/api/trait.py @@ -10,6 +10,7 @@ from caendr.models.datastore import TraitFile, Species from caendr.models.error import NotFoundError +from caendr.models.sql import PhenotypeMetadata from caendr.utils.json import jsonify_request @@ -62,19 +63,19 @@ def inner(*args, **kwargs): # -@api_trait_bp.route('/query', methods=['POST']) +@api_trait_bp.route('/query/sql', methods=['POST']) @cache.memoize(60*60) @query_traits_error_handler @jsonify_request -def query(): +def query_sql(): ''' - Query all trait files, optionally split into different lists based on species. + Query the trait database, and return results with SQL-style pagination. ''' # Get query filters (search parameters) selected_tags = get_clean(request.json, 'selected_tags', []) search_val = get_clean(request.json, 'search_val', '').lower() - filter_dataset = get_clean(request.json, 'dataset') + filter_dataset = get_clean(request.args, 'dataset') # Get query pagination values page = get_clean(request.json, 'page', 1, _type=int) @@ -107,6 +108,52 @@ def query(): } + +@api_trait_bp.route('/query/datatable', methods=['GET']) +@cache.memoize(60*60) +@query_traits_error_handler +@jsonify_request +def query_datatable(): + ''' + Query the trait database, and return results with DataTable-style pagination. + ''' + + # Get query filters (search parameters) + search_value = get_clean(request.args, 'search[value]', '').lower() + filter_dataset = get_clean(request.args, 'dataset') + + # Get query pagination values + draw = get_clean(request.args, 'draw', _type=int) + start = get_clean(request.args, 'start', _type=int) + length = get_clean(request.args, 'length', _type=int) + + # Create the initial query + query = query_phenotype_metadata(dataset=filter_dataset) + total_records = query.count() + + # Filter by search values, if provided + query = filter_trait_query(query, search_val=search_value) + + # Query PhenotypeMetadata (include phenotype values for each trait) + with rollback_on_error_handler(): + data = query.offset(start).limit(length).from_self().\ + join(PhenotypeMetadata.phenotype_values).all() + + # Count how many rows matched the filters + filtered_records = query.count() + + # Format return data + return { + 'data': [ + trait.to_json_with_values() for trait in data + ], + 'draw': draw, + 'recordsTotal': total_records, + 'recordsFiltered': filtered_records, + } + + + @api_trait_bp.route('/', methods=['GET']) @cache.memoize(60*60) @jsonify_request diff --git a/src/modules/site-v2/base/views/tools/phenotype_database.py b/src/modules/site-v2/base/views/tools/phenotype_database.py index f47b8a967..ea476c904 100644 --- a/src/modules/site-v2/base/views/tools/phenotype_database.py +++ b/src/modules/site-v2/base/views/tools/phenotype_database.py @@ -101,48 +101,6 @@ def phenotype_database(): }) -@phenotype_database_bp.route('/traits-zhang') -@cache.memoize(60*60) -@compress.compressed() -def get_zhang_traits_json(): - """ - Phenotype Database table (bulk) - Fetch table content by request for each page and render the data for Datatables() - """ - try: - # get parameters for query - draw = request.args.get('draw', type=int) - start = request.args.get('start', type=int) - length = request.args.get('length', type=int) - search_value = bleach.clean(request.args.get('search[value]', '')).lower() - - query = query_phenotype_metadata(is_bulk_file=True) - total_records = query.count() - - # Filter by search value, if provided - query = filter_trait_query_by_text(query, search_value) - - # Query PhenotypeMetadata (include phenotype values for each trait) - with rollback_on_error_handler(): - data = query.offset(start).limit(length).from_self().\ - join(PhenotypeMetadata.phenotype_values).all() - - json_data = [ trait.to_json_with_values() for trait in data ] - - filtered_records = query.count() - - response_data = { - "draw": draw, - "recordsTotal": total_records, - "recordsFiltered": filtered_records, - "data": json_data - } - - except Exception as ex: - logger.error(f'Failed to retrieve the list of traits: {ex}') - response_data = [] - return jsonify(response_data) - @phenotype_database_bp.route('/traits-list', methods=['POST']) @cache.memoize(60*60) @compress.compressed() diff --git a/src/modules/site-v2/templates/_includes/phenotype-database-table.html b/src/modules/site-v2/templates/_includes/phenotype-database-table.html index 12559812d..892832423 100644 --- a/src/modules/site-v2/templates/_includes/phenotype-database-table.html +++ b/src/modules/site-v2/templates/_includes/phenotype-database-table.html @@ -133,7 +133,7 @@

const zhangTable = $('#zhangTraits').DataTable( { "processing": true, "serverSide": true, - "ajax": "/tools/phenotype-database/traits-zhang", + "ajax": "{{ url_for('api_trait.query_datatable', dataset='zhang') }}", "ordering": false, "columns": [ { @@ -280,13 +280,13 @@

} async function sendPostRequest(data) { - const resp = await fetch("{{ url_for('api_trait.query') }}", { + const resp = await fetch("{{ url_for('api_trait.query_sql', dataset='caendr') }}", { method: "POST", headers: { "Content-Type": "application/json", 'X-CSRF-TOKEN': data.csrf_token }, - body: JSON.stringify({...data, 'dataset': 'caendr'}), + body: JSON.stringify({...data}), }) if (resp.ok) { From ca9cdbb0dec4cf016ea26310ebdc6c791fd5364c Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Fri, 5 Apr 2024 15:28:03 -0500 Subject: [PATCH 10/44] Accept species & submitting user filters in trait query endpoints --- src/modules/site-v2/base/views/api/trait.py | 8 ++++-- src/pkg/caendr/caendr/api/phenotype.py | 32 +++++++++++++++++++-- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/src/modules/site-v2/base/views/api/trait.py b/src/modules/site-v2/base/views/api/trait.py index 3a4b8f1d1..e57dcc5d5 100644 --- a/src/modules/site-v2/base/views/api/trait.py +++ b/src/modules/site-v2/base/views/api/trait.py @@ -76,6 +76,8 @@ def query_sql(): selected_tags = get_clean(request.json, 'selected_tags', []) search_val = get_clean(request.json, 'search_val', '').lower() filter_dataset = get_clean(request.args, 'dataset') + filter_user = get_clean(request.json, 'user') + filter_species = get_clean(request.json, 'species') # Get query pagination values page = get_clean(request.json, 'page', 1, _type=int) @@ -83,7 +85,7 @@ def query_sql(): per_page = 10 # Create the initial query - query = query_phenotype_metadata(dataset=filter_dataset) + query = query_phenotype_metadata(dataset=filter_dataset, species=filter_species, user=filter_user) # Filter by search values, if provided query = filter_trait_query(query, search_val=search_val, tags=selected_tags) @@ -121,6 +123,8 @@ def query_datatable(): # Get query filters (search parameters) search_value = get_clean(request.args, 'search[value]', '').lower() filter_dataset = get_clean(request.args, 'dataset') + filter_user = get_clean(request.args, 'user') + filter_species = get_clean(request.args, 'species') # Get query pagination values draw = get_clean(request.args, 'draw', _type=int) @@ -128,7 +132,7 @@ def query_datatable(): length = get_clean(request.args, 'length', _type=int) # Create the initial query - query = query_phenotype_metadata(dataset=filter_dataset) + query = query_phenotype_metadata(dataset=filter_dataset, species=filter_species, user=filter_user) total_records = query.count() # Filter by search values, if provided diff --git a/src/pkg/caendr/caendr/api/phenotype.py b/src/pkg/caendr/caendr/api/phenotype.py index 627e2ab02..058c788fc 100644 --- a/src/pkg/caendr/caendr/api/phenotype.py +++ b/src/pkg/caendr/caendr/api/phenotype.py @@ -4,7 +4,7 @@ from sqlalchemy import or_, func -from caendr.models.datastore import Species +from caendr.models.datastore import Species, User from caendr.models.sql import PhenotypeMetadata from caendr.services.cloud.postgresql import rollback_on_error @@ -12,8 +12,9 @@ def query_phenotype_metadata( include_values = False, is_bulk_file: Optional[bool] = None, - species: Optional[Union[Species, str]] = None, dataset: Optional[str] = None, + species: Optional[Union[Species, str]] = None, + user: Optional[Union[User, str]] = None, ): """ Returns the list of traits with the corresponding metadata. @@ -39,6 +40,7 @@ def query_phenotype_metadata( # Optionally query by species # None values handled in function query = filter_trait_query_by_species(query, species) + query = filter_trait_query_by_user(query, user) # Include phenotype values for traits if include_values: @@ -71,6 +73,7 @@ def filter_trait_query( search_val: Optional[str] = None, tags: Optional[Iterable[str]] = None, species: Optional[Union[Species, str]] = None, + user: Optional[Union[User, str]] = None, ): ''' Combined filtering function. @@ -78,6 +81,7 @@ def filter_trait_query( query = filter_trait_query_by_text(query, search_val) query = filter_trait_query_by_tags(query, tags) query = filter_trait_query_by_species(query, species) + query = filter_trait_query_by_user(query, user) return query @@ -113,6 +117,28 @@ def filter_trait_query_by_tags(query, tags: Optional[Iterable[str]]): return query +def filter_trait_query_by_user(query, user: Optional[Union[User, str]]): + ''' + Filter by submitting user. + + If username is invalid, passes error raised by `User` class. + ''' + if user: + + # Cast string value to User object using unique datastore ID + if isinstance(user, str): + user = User.get_ds(user) + + # Validate user type + if not isinstance(user, User): + raise ValueError(f'Expected user, got {user}') + + # Filter by the username + query = query.filter_by(submitted_by=user.full_name) + + return query + + def filter_trait_query_by_species(query, species: Optional[Union[Species, str]]): ''' Filter by species. @@ -121,7 +147,7 @@ def filter_trait_query_by_species(query, species: Optional[Union[Species, str]]) ''' if species is not None: - # Cast string values to Species object + # Cast string value to Species object if isinstance(species, str): species = Species.from_name(species) From b732d8d7876db02431ce52b64c0b4568f8211c7d Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Fri, 5 Apr 2024 15:37:58 -0500 Subject: [PATCH 11/44] Move trait metadata query to traits API --- src/modules/site-v2/base/views/api/trait.py | 31 +++++++++++++++++-- .../base/views/tools/phenotype_database.py | 20 ------------ .../_includes/phenotype-database-table.html | 2 +- 3 files changed, 30 insertions(+), 23 deletions(-) diff --git a/src/modules/site-v2/base/views/api/trait.py b/src/modules/site-v2/base/views/api/trait.py index e57dcc5d5..8731c53fc 100644 --- a/src/modules/site-v2/base/views/api/trait.py +++ b/src/modules/site-v2/base/views/api/trait.py @@ -1,9 +1,9 @@ import bleach from functools import wraps -from flask import request, Blueprint, abort +from flask import request, Blueprint, abort, jsonify from caendr.services.logger import logger -from extensions import cache +from extensions import cache, compress from caendr.api.phenotype import query_phenotype_metadata, get_trait, filter_trait_query from caendr.services.cloud.postgresql import rollback_on_error_handler @@ -178,3 +178,30 @@ def query_species(species_name): for tf in TraitFile.query_ds(ignore_errs=True, filters=['species', '=', species.name]) if filter_trait_files(tf) ] + + + +# +# Query single trait data +# + + +@api_trait_bp.route('/metadata', methods=['POST']) +@cache.memoize(60*60) +@compress.compressed() +def get_trait_metadata(): + """ + Get traits data for non-bulk files in JSON format (include phenotype values) + """ + trait_name = get_clean(request.json, 'trait_name') + err_msg = f'Failed to retrieve metadata for trait {trait_name}' + + if trait_name: + try: + trait = get_trait(trait_name).to_json_with_values() + return jsonify(trait) + + except Exception as ex: + logger.error(f'{err_msg}: {ex}') + + return jsonify({ 'message': err_msg }), 404 diff --git a/src/modules/site-v2/base/views/tools/phenotype_database.py b/src/modules/site-v2/base/views/tools/phenotype_database.py index ea476c904..8d72b4480 100644 --- a/src/modules/site-v2/base/views/tools/phenotype_database.py +++ b/src/modules/site-v2/base/views/tools/phenotype_database.py @@ -101,26 +101,6 @@ def phenotype_database(): }) -@phenotype_database_bp.route('/traits-list', methods=['POST']) -@cache.memoize(60*60) -@compress.compressed() -def get_traits_json(): - """ - Get traits data for non-bulk files in JSON format (include phenotype values) - """ - trait_name = request.json.get('trait_name') - err_msg = f'Failed to retrieve metadata for trait {trait_name}' - - if trait_name: - try: - trait = get_trait(trait_name).to_json_with_values() - return jsonify(trait) - - except Exception as ex: - logger.error(f'{err_msg}: {ex}') - - return jsonify({ 'message': err_msg }), 404 - # # Submission Flow diff --git a/src/modules/site-v2/templates/_includes/phenotype-database-table.html b/src/modules/site-v2/templates/_includes/phenotype-database-table.html index 892832423..01dff2816 100644 --- a/src/modules/site-v2/templates/_includes/phenotype-database-table.html +++ b/src/modules/site-v2/templates/_includes/phenotype-database-table.html @@ -188,7 +188,7 @@

const data = {csrf_token: $('#csrf_token').val(), trait_name: $(this).data('value')} $.ajax({ type: "POST", - url: "{{ url_for('phenotype_database.get_traits_json') }}", + url: "{{ url_for('api_trait.get_trait_metadata') }}", data: JSON.stringify(data), contentType: "application/json", dataType: "json", From 83805340ea91f11e6e772121f2063e257e6787ea Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Fri, 5 Apr 2024 16:08:12 -0500 Subject: [PATCH 12/44] Rewrite metadata endpoint w helpers, better error handling --- src/modules/site-v2/base/views/api/trait.py | 64 ++++++++++++++------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/src/modules/site-v2/base/views/api/trait.py b/src/modules/site-v2/base/views/api/trait.py index 8731c53fc..603cc68f6 100644 --- a/src/modules/site-v2/base/views/api/trait.py +++ b/src/modules/site-v2/base/views/api/trait.py @@ -42,19 +42,38 @@ def get_clean(source, key, value=None, _type=None): return v -def query_traits_error_handler(f): +def query_traits_error_handler(err_msg): ''' Wrapper for trait query endpoints. - If query raises an error, returns an empty response and a `500` error. + + If the wrapped function aborts with an error code, that code will be used + by the response. Otherwise, based on the error type, either a `404` or a + `500` will be returned. ''' - @wraps(f) - def inner(*args, **kwargs): - try: - return f(*args, **kwargs) - except Exception as ex: - logger.error(f'Failed to retrieve the list of traits: {ex}') - return {}, 500 - return inner + + def decorator(f): + @wraps(f) + def inner(*args, **kwargs): + try: + return f(*args, **kwargs) + + # Error handling + except Exception as ex: + + # Choose error code based on error type + if hasattr(ex, 'code'): + err_code = ex.code + elif isinstance(ex, NotFoundError): + err_code = 404 + else: + err_code = 500 + + # Log the full error, and return the response with an abridged message + logger.error(f'{err_msg}: {ex}') + return {'message': f'{err_msg}'}, err_code + + return inner + return decorator @@ -65,7 +84,7 @@ def inner(*args, **kwargs): @api_trait_bp.route('/query/sql', methods=['POST']) @cache.memoize(60*60) -@query_traits_error_handler +@query_traits_error_handler('Failed to retrieve the list of traits') @jsonify_request def query_sql(): ''' @@ -113,7 +132,7 @@ def query_sql(): @api_trait_bp.route('/query/datatable', methods=['GET']) @cache.memoize(60*60) -@query_traits_error_handler +@query_traits_error_handler('Failed to retrieve the list of traits') @jsonify_request def query_datatable(): ''' @@ -189,19 +208,22 @@ def query_species(species_name): @api_trait_bp.route('/metadata', methods=['POST']) @cache.memoize(60*60) @compress.compressed() +@query_traits_error_handler('Failed to retrieve trait metadata') +@jsonify_request def get_trait_metadata(): """ Get traits data for non-bulk files in JSON format (include phenotype values) """ - trait_name = get_clean(request.json, 'trait_name') - err_msg = f'Failed to retrieve metadata for trait {trait_name}' - if trait_name: - try: - trait = get_trait(trait_name).to_json_with_values() - return jsonify(trait) + # Get the trait name from the request + trait_name = get_clean(request.json, 'trait_name') + if not trait_name: + abort(400, description='No trait name provided.') - except Exception as ex: - logger.error(f'{err_msg}: {ex}') + # Try getting the trait from the database + trait = get_trait(trait_name) + if trait is None: + abort(404, description=f'Invalid trait name {trait_name}') - return jsonify({ 'message': err_msg }), 404 + # Return the full trait metadata + return trait.to_json_with_values() From e34ba69b4194243dbd479abe440de8d0ec22df65 Mon Sep 17 00:00:00 2001 From: Vincent LaGrassa Date: Mon, 8 Apr 2024 12:34:03 -0500 Subject: [PATCH 13/44] Modularize trait offcanvas sidebar JS code --- .../_includes/phenotype-database-table.html | 109 +++------------ .../phenotype-offcanvas-content.html | 125 ++++++++++++++++-- .../site-v2/templates/_scripts/utils.js | 11 ++ 3 files changed, 142 insertions(+), 103 deletions(-) diff --git a/src/modules/site-v2/templates/_includes/phenotype-database-table.html b/src/modules/site-v2/templates/_includes/phenotype-database-table.html index 01dff2816..307af3918 100644 --- a/src/modules/site-v2/templates/_includes/phenotype-database-table.html +++ b/src/modules/site-v2/templates/_includes/phenotype-database-table.html @@ -21,6 +21,10 @@ {% endblock %} + +{% set offcanvas_id = 'phenotype-offcanvas' %} + +