Merge pull request #485 from AndersenLab/feature/moderation-queue

Feature/moderation queue

src/modules/site-v2/base/forms/forms.py

@@ -350,23 +350,18 @@ class StrainListForm(Form):
 
 class TraitSubmissionForm(FlaskForm):
   """ The trait submission form """
-  file = FileField('Select file', render_kw={'accept': ','.join({ f'.{ext}' for ext in TOOL_INPUT_DATA_VALID_FILE_EXTENSIONS})}, validators=[Required()])
+  file = FileField('Select file', render_kw={'accept': ','.join({ f'.{ext}' for ext in TOOL_INPUT_DATA_VALID_FILE_EXTENSIONS})})
   species = SpeciesSelectField(validators=[Required()])
-  trait_name_user = StringField('Internal Trait Name', validators=[Required(), Length(min=3, max=50)])
+  trait_name_user = StringField('Internal Trait Name', validators=[Required(), Length(min=3, max=100)])
   trait_name_display_1 = StringField('Display Name 1', validators=[Required(), Length(min=3, max=50)])
   trait_name_display_2 = StringField('Display Name 2')
   trait_name_display_3 = StringField('Display Name 3')
   description_short = TextAreaField('Short Description', validators=[Required(), Length(min=10, max=200)])
   description_long = TextAreaField('Long Description', validators=[Required(), Length(min=10)])
   units = StringField('Unit of Measurement', validators=[Length(min=0, max=10)])
   tags = MultiCheckboxField("Categories", choices=TRAIT_CATEGORY_OPTIONS, validators=[Required()])
-  username = StringField('Username', validators=[Required(), Length(min=3, max=50)])
+  email = StringField('Email', validators=[Email(), Required(), Length(min=3, max=50)])
   institution = StringField('Institution', validators=[Required(), Length(min=3, max=50)])
   source_lab = StringField('Source Lab', validators=[Required(), Length(min=1, max=4)])
   protocols = TextAreaField('Protocols', validators=[Length(min=0, max=200)])
-  publication = TextAreaField('Publications', validators=[Length(min=0, max=200)])
-
-
-
-
-
+  publication = TextAreaField('Publications', validators=[Length(min=0, max=200)])

src/modules/site-v2/base/utils/trait.py

@@ -136,7 +136,7 @@ def add_trait(form_data, user):
     logger.error(f'Failed to upload the file data to the database: {ex}')
     return {'message': 'Failed to submit a form. Please try again later.'}, 500
 
-  return {'message': 'Trait submitted successfully.'}, 200
+  return {'message': 'Trait submitted successfully.', 'trait_id': tf.name}, 200
 
 
 def rollback_submission_on_error(trait_id, blob_name=None):
@@ -168,4 +168,46 @@ def rollback_submission_on_error(trait_id, blob_name=None):
 
     # Delete the file data from Phenotype Database SQL table
     PhenotypeDatabase.delete_by_metadata_id(tf.name)
+
+
+def update_trait_metadata(id, form_data):
+  """ Update Trait metadata in datastore and SQL table """
+  try:
+    tf = TraitFile.get_ds(id)
+    props_to_update = {
+      'trait_name_user':      bleach.clean(form_data['trait_name_user']),
+      'trait_name_display_1': bleach.clean(form_data['trait_name_display_1']),
+      'trait_name_display_2': bleach.clean(form_data['trait_name_display_2']),
+      'trait_name_display_3': bleach.clean(form_data['trait_name_display_3']),
+      'description_short':    bleach.clean(form_data['description_short']),
+      'description_long':     bleach.clean(form_data['description_long']),
+      'units':                bleach.clean(form_data['units']),
+      'tags':                 [ bleach.clean(tag) for tag in form_data['tags'] ],
+      'institution':          bleach.clean(form_data['institution']),
+      'source_lab':           bleach.clean(form_data['source_lab']),
+      'protocols':            bleach.clean(form_data['protocols']),        
+      'publication':          bleach.clean(form_data['publication']),
+    }
+    tf.set_properties(**props_to_update)
+    tf.save()
+  except Exception as ex:
+    logger.error(f'Failed to update the trait file {tf.name}: {ex}')
+    return {'message': 'Failed to update the trait. Please try again later.'}, 500
+
+  try:
+    trait_sql = get_trait(id)
+    if trait_sql:
+      trait_sql.update(**props_to_update)
+  except Exception as ex:
+    # TODO: Rollback the changes in Datastore
+    logger.error(f'Failed to update the trait metadata in SQL table: {ex}')
+    return {'message': 'Failed to update the trait. Please try again later.'}, 500
+
+  return {'message': 'Trait updated successfully.'}, 200
+
+
+def user_is_trait_owner(trait, user) -> bool:
+  """ Check if the user is the owner of the trait """
+  return user.email == trait['submitter_email']
+
 

src/modules/site-v2/base/views/api/trait.py

@@ -16,6 +16,7 @@
 from caendr.models.error     import NotFoundError
 from caendr.models.sql       import PhenotypeMetadata
 from caendr.utils.json       import jsonify_request
+from base.utils.auth         import jwt_required
 
 
 api_trait_bp = Blueprint(
@@ -428,3 +429,13 @@ def query_trait_categories():
     Get list of trait categories.
   """
   return get_trait_categories()
+
+
+
+
+# TODO: Merge this with Vince's code
+@api_trait_bp.route('/review/<string:trait_id>/submit', methods=['POST'])
+@cache.memoize(60*60)
+@jwt_required()
+def submit_trait(trait_id):
+  pass

src/modules/site-v2/base/views/data/data.py

@@ -1,21 +1,28 @@
 import yaml
 import csv
+import os
+from datetime import datetime
 
-from flask      import render_template, Blueprint, redirect, url_for, request, flash, jsonify
+from flask      import render_template, Blueprint, redirect, url_for, request, flash, jsonify, abort, send_file
 from extensions import cache
 from config     import config
 
 from caendr.models.error           import EnvVarError, FileUploadError
-from caendr.models.datastore       import Species
-from caendr.services.cloud.storage import get_blob
+from caendr.models.datastore       import Species, TraitFile
+from caendr.api.phenotype          import get_phenotype_values_for_trait
+from caendr.services.cloud.storage import get_blob, download_blob_to_file
 from caendr.services.logger        import logger
 from caendr.services.validate      import validate_file, StrainValidator, NumberValidator
 from caendr.utils.local_files      import LocalUploadFile
-from base.utils.auth               import jwt_required, get_current_user
-from base.utils.trait              import add_trait
-from base.forms                    import TraitSubmissionForm
+from caendr.utils.env              import get_env_var
+from caendr.utils.data             import get_file_format
+from base.utils.auth               import jwt_required, get_current_user, user_is_admin
+from base.utils.trait              import add_trait, update_trait_metadata, user_is_trait_owner
+from base.forms                    import TraitSubmissionForm, EmptyForm
 from constants                     import TOOL_INPUT_DATA_VALID_FILE_EXTENSIONS
 
+MODULE_DB_OPERATIONS_BUCKET_NAME = get_env_var('MODULE_DB_OPERATIONS_BUCKET_NAME')
+UPLOADS_DIR = os.path.join('.', 'uploads')
 
 
 data_bp = Blueprint(
@@ -72,6 +79,7 @@ def protocols():
 # Submit Trait
 #
 @data_bp.route('/trait/start-submit')
+@cache.memoize(60*60)
 @jwt_required()
 def submit_trait_start():
   """ Submit Trait start page """
@@ -84,14 +92,15 @@ def submit_trait_start():
 # Submit Trait Form
 #
 @data_bp.route('/trait/create', methods=['GET', 'POST'])
+@cache.memoize(60*60)
 @jwt_required()
 def submit_trait_form():
   """ Trait Submission Form """
   form = TraitSubmissionForm()
   user = get_current_user()
 
-  if hasattr(user, 'username') and not form.username.data:
-    form.username.data = user.username
+  if hasattr(user, 'email') and not form.email.data:
+    form.email.data = user.email
 
   # Handle form submission
   if request.method == 'POST':
@@ -100,7 +109,7 @@ def submit_trait_form():
 
     # Validate form fields
     if not form.validate_on_submit():
-      flash('Please fill out all required fields.', 'warning')
+      flash(f'Please fill out all required fields: {form.errors}', 'warning')
 
     else:
       # Add the trait to the database
@@ -109,47 +118,156 @@ def submit_trait_form():
         flash(resp['message'], 'danger')
       else:
         flash('Trait submitted successfully.', 'success')
-        return redirect(url_for('data.submit_trait_start'))
+        return redirect(url_for('data.trait', id=resp['trait_id']))
 
   return render_template('data/submit-trait-form.html', **{
     # Page Info
-    'title': 'Phenotype Database Trait Submission',
+    'title':                      'Phenotype Database Trait Submission',
     'tool_alt_parent_breadcrumb': {"title": "Submit Trait", "url": url_for('data.submit_trait_start')},
+    'new_submission':             True,
+
+    # Data
+    'form':             form,
+    'phenotype_values': None,
+  })
+
+
+@data_bp.route('/trait/<string:id>')
+@cache.memoize(60*60)
+@jwt_required()
+def trait(id):
+  """ Trait Page """
+  trait_ds = TraitFile.get_ds(id)
+  trait_name = ' '.join(trait_ds.display_name)
+  phenotype_values = get_phenotype_values_for_trait(id)
+  return render_template('data/trait.html', **{
+    # Page Info}
+    'title':                      trait_ds['trait_name_display_1'],
+    'tool_alt_parent_breadcrumb': {"title": "MTL", "url": url_for('data.my_trait_library')},
+
+    # Data
+    'trait_name':    trait_name,
+    'trait':         trait_ds.serialize(),
+    'file_content':  phenotype_values,
+    'form':          EmptyForm(),
+    'user_is_owner': user_is_trait_owner(trait_ds.serialize(), get_current_user()),
+
+  })
+
+@data_bp.route('/trait/<string:id>/edit', methods=['GET', 'PUT'])
+@cache.memoize(60*60)
+@jwt_required()
+def edit_trait(id):
+  """ Edit Trait Page"""
+  user = get_current_user()
+  trait_ds = TraitFile.get_ds(id).serialize()
+  if user_is_trait_owner(trait_ds, user) and not user_is_admin():
+    return abort(401)
+
+  # Handle Trait Update
+  if request.method == 'PUT':
+    form = TraitSubmissionForm(request.form)
+    form.species.data = trait_ds['species']
+    form.email.data = trait_ds['submitter_email']
+
+    # Validate form fields
+    if not form.validate_on_submit():
+      return jsonify({'message': f'Please fill out all required fields: {form.errors}'}), 500
+
+    # Update the trait metadata
+    else:
+      resp, code = update_trait_metadata(id, form.data)
+      if code != 200:
+        flash(resp['message'], 'danger')
+      else:
+        flash(resp['message'], 'success')
+        return jsonify( resp ), code
+
+  form_data = {
+    'species':              trait_ds.get('species'),
+    'trait_name_user':      trait_ds.get('trait_name_caendr') if trait_ds['from_caendr'] else trait_ds.get('trait_name_user'),
+    'trait_name_display_1': trait_ds.get('trait_name_display_1'),
+    'trait_name_display_2': trait_ds.get('trait_name_display_2'),
+    'trait_name_display_3': trait_ds.get('trait_name_display_3'),
+    'description_short':    trait_ds.get('description_short'),
+    'description_long':     trait_ds.get('description_long'),
+    'unit':                 trait_ds.get('unit'),
+    'tags':                 trait_ds.get('tags'),
+    'email':                trait_ds.get('submitter_email'),
+    'institution':          trait_ds.get('institution'),
+    'source_lab':           trait_ds.get('source_lab'),
+    'protocols':            trait_ds.get('protocols'),
+    'publication':          trait_ds.get('publication')
+  }
+  form = TraitSubmissionForm(data=form_data)
+  return render_template('data/submit-trait-form.html', **{
+    # Page Info
+    'title':                      'Edit Trait',
+    'tool_alt_parent_breadcrumb': { "title": trait_ds['trait_name_display_1'], "url": url_for('data.trait', id=trait_ds['name']) },
+    'new_submission':             False,
 
     # Data
-    'form': form,
+    'form':             form,
+    'phenotype_values': [ v.to_json() for v in get_phenotype_values_for_trait(id) ],
+    'file':             {
+                          'name':       trait_ds['filename'],
+                          'created_on': datetime.strftime(trait_ds['created_on'], "%Y-%m-%d"),
+                        },
+    'trait_id':         id,
+    'user_is_admin':    user_is_admin(),
   })
 
+
 #
 # File Upload
 #
 @data_bp.route('/trait/parse-file', methods=['POST'])
 @jwt_required()
-def parse_trait_file():
+def validate_and_parse_trait_file():
   """ Parse the trait file and return the data """
   try:
     with LocalUploadFile(request.files.get('file'), valid_file_extensions=TOOL_INPUT_DATA_VALID_FILE_EXTENSIONS) as file:
       # Validate the file
       try:
         species = Species.from_name(request.form.get('species'))
         validate_file(file, [
-                              StrainValidator( 'strain', species=species, force_unique=True, force_unique_msgs={} ),
+                              StrainValidator( 'strain', species=species, force_unique=True, force_unique_msgs={}, strain_issues=None ),
                               NumberValidator( None, accept_float=True, accept_na=True ),
                             ])
       except Exception as ex:
         return jsonify({ 'message': ex.msg }), 500
 
       # Parse the file
-      with open(file) as f:
-        file_content = []
-        for idx, row in enumerate( csv.reader(f, delimiter='\t') ):
-          file_content.append({'col_1': row[0], 'col_2': row[1]})
-        return jsonify(file_content), 200
+      file_content = parse_trait_file(file)
+      return jsonify(file_content), 200
 
   except FileUploadError as ex:
     return jsonify({ 'message': ex.description }), ex.code
 
   except Exception as ex:
     logger.error(f'Failed to parse the file: {ex}')
     return jsonify({ 'message': 'Failed to parse the file. Please try again later.' }), 500
+
+
+@data_bp.route('/trait/<string:id>/download-file')
+@cache.memoize(60*60)
+@jwt_required()
+def download_trait_file(id):
+  """ Download the trait file """
+  user = get_current_user()
+  trait_ds = TraitFile.get_ds(id)
+  if user_is_trait_owner(trait_ds.serialize(), user) and not user_is_admin():
+    return abort(401)
+
+  file = download_blob_to_file(MODULE_DB_OPERATIONS_BUCKET_NAME, trait_ds.get_filepath()[1], destination=UPLOADS_DIR)
+  mimetype = get_file_format('tsv')['mimetype']
+  return send_file(file, mimetype=mimetype, as_attachment=True, attachment_filename=trait_ds['filename'].raw_string)
+
 
+def parse_trait_file(file):
+  """ Parse a trait file into a list of dictionaries """
+  with open(file) as f:
+    file_content = []
+    for idx, row in enumerate( csv.reader(f, delimiter='\t') ):
+      file_content.append({'col_1': row[0], 'col_2': row[1]})
+    return file_content

src/modules/site-v2/constants.py

@@ -39,12 +39,12 @@ class PRICES:
 TOOL_INPUT_DATA_VALID_FILE_EXTENSIONS = { 'tsv' }
 
 TRAIT_CATEGORY_OPTIONS = [
-  ('Growth/Physiology', 'Growth/Physiology'), 
-  ('Morphology/Development/Lineage/Cell type', 'Morphology/Development/Lineage/Cell type'), 
+  ('Growth/Physiology', 'Growth • Physiology'), 
+  ('Morphology/Development/Lineage/Cell type', 'Morphology • Development • Lineage • Cell type'), 
   ('Behavior', 'Behavior'), 
   ('Molecular', 'Molecular'), 
   ('Stress response', 'Stress response'), 
-  ('Drug/Compound/Condition/Treatment', 'Drug/Compound/Condition/Treatment'), 
+  ('Drug/Compound/Condition/Treatment', 'Drug • Compound • Condition • Treatment'), 
   ('Ecology', 'Ecology'), 
   ('Genomics', 'Genomics'), 
   ('Reproduction', 'Reproduction')