Merge pull request #477 from AndersenLab/feature/submit-trait

Trait Submission Form

env/development/global.env

@@ -103,6 +103,7 @@ EXTERNAL_DB_BACKUP_PATH=db/external
 MODULE_DB_OPERATIONS_RELEASE_FILEPATH=reference_genome/{SPECIES}/{RELEASE}
 MODULE_DB_OPERATIONS_SVA_FILEPATH=strain_variant_annotation/{SPECIES}
 MODULE_DB_OPERATIONS_PHENOTYPE_FILEPATH=trait_files/{SPECIES}
+MODULE_DB_OPERATIONS_TRAITFILE_PUBLIC_FILEPATH=trait_files/public
 
 GENE_GTF_FILENAME=canonical_geneset.gtf.gz
 GENE_GFF_FILENAME=annotations.gff3.gz

env/main/global.env

@@ -94,6 +94,7 @@ EXTERNAL_DB_BACKUP_PATH=db/external
 
 MODULE_DB_OPERATIONS_RELEASE_FILEPATH=reference_genome/{SPECIES}/{RELEASE}
 MODULE_DB_OPERATIONS_SVA_FILEPATH=strain_variant_annotation/{SPECIES}
+MODULE_DB_OPERATIONS_TRAITFILE_PUBLIC_FILEPATH=trait_files/public
 
 GENE_GTF_FILENAME=canonical_geneset.gtf.gz
 GENE_GFF_FILENAME=annotations.gff3.gz

env/qa/global.env

@@ -103,6 +103,7 @@ EXTERNAL_DB_BACKUP_PATH=db/external
 MODULE_DB_OPERATIONS_RELEASE_FILEPATH=reference_genome/{SPECIES}/{RELEASE}
 MODULE_DB_OPERATIONS_SVA_FILEPATH=strain_variant_annotation/{SPECIES}
 MODULE_DB_OPERATIONS_PHENOTYPE_FILEPATH=trait_files/{SPECIES}
+MODULE_DB_OPERATIONS_TRAITFILE_PUBLIC_FILEPATH=trait_files/public
 
 GENE_GTF_FILENAME=canonical_geneset.gtf.gz
 GENE_GFF_FILENAME=annotations.gff3.gz

src/modules/site-v2/base/forms/forms.py

@@ -29,7 +29,7 @@
 from wtforms.fields.html5 import EmailField
 
 
-from constants import PRICES, SECTOR_OPTIONS, SHIPPING_OPTIONS, PAYMENT_OPTIONS, TOOL_INPUT_DATA_VALID_FILE_EXTENSIONS
+from constants import PRICES, SECTOR_OPTIONS, SHIPPING_OPTIONS, PAYMENT_OPTIONS, TOOL_INPUT_DATA_VALID_FILE_EXTENSIONS, TRAIT_CATEGORY_OPTIONS
 
 from caendr.services.profile import get_profile_role_form_options
 from caendr.services.user import get_user_role_form_options, get_local_user_by_email
@@ -336,3 +336,27 @@ class MappingSubmissionForm(Form):
 
 class StrainListForm(Form):
   species = SpeciesSelectField(validators=[Required()])
+
+
+class TraitSubmissionForm(FlaskForm):
+  """ The trait submission form """
+  file = FileField('Select file', render_kw={'accept': ','.join({ f'.{ext}' for ext in TOOL_INPUT_DATA_VALID_FILE_EXTENSIONS})}, validators=[Required()])
+  species = SpeciesSelectField(validators=[Required()])
+  trait_name_user = StringField('Internal Trait Name', validators=[Required(), Length(min=3, max=50)])
+  trait_name_display_1 = StringField('Display Name 1', validators=[Required(), Length(min=3, max=50)])
+  trait_name_display_2 = StringField('Display Name 2')
+  trait_name_display_3 = StringField('Display Name 3')
+  description_short = TextAreaField('Short Description', validators=[Required(), Length(min=10, max=200)])
+  description_long = TextAreaField('Long Description', validators=[Required(), Length(min=10)])
+  units = StringField('Unit of Measurement', validators=[Length(min=0, max=10)])
+  tags = MultiCheckboxField("Categories", choices=TRAIT_CATEGORY_OPTIONS, validators=[Required()])
+  username = StringField('Username', validators=[Required(), Length(min=3, max=50)])
+  institution = StringField('Institution', validators=[Required(), Length(min=3, max=50)])
+  source_lab = StringField('Source Lab', validators=[Required(), Length(min=1, max=4)])
+  protocols = TextAreaField('Protocols', validators=[Length(min=0, max=200)])
+  publication = TextAreaField('Publications', validators=[Length(min=0, max=200)])
+
+
+
+
+

src/modules/site-v2/base/views/data/data.py

@@ -1,13 +1,32 @@
 import yaml
+import bleach
+import csv
 
-from flask import render_template, Blueprint, redirect, url_for
+from flask      import render_template, Blueprint, redirect, url_for, request, flash, jsonify, abort
 from extensions import cache
-from config import config
+from config     import config
 
-from caendr.models.error import EnvVarError
-from caendr.services.cloud.storage import get_blob
+from caendr.models.error           import EnvVarError, FileUploadError
+from caendr.models.datastore       import TraitFile, Species
+from caendr.models.status          import PublishStatus
+from caendr.services.cloud.storage import get_blob, upload_blob_from_file_object, check_blob_exists
+from caendr.services.logger        import logger
+from caendr.services.validate      import validate_file, StrainValidator, NumberValidator
+from caendr.utils.data             import unique_id
+from base.utils.auth               import jwt_required, get_current_user
+from caendr.utils.env              import get_env_var
+from caendr.utils.local_files      import LocalUploadFile
+from base.forms                    import TraitSubmissionForm
+from constants                     import TOOL_INPUT_DATA_VALID_FILE_EXTENSIONS
+from caendr.models.sql             import PhenotypeMetadata, PhenotypeDatabase
 
 
+
+
+
+MODULE_DB_OPERATIONS_BUCKET_NAME = get_env_var('MODULE_DB_OPERATIONS_BUCKET_NAME')
+MODULE_DB_OPERATIONS_TRAITFILE_PUBLIC_FILEPATH = get_env_var('MODULE_DB_OPERATIONS_TRAITFILE_PUBLIC_FILEPATH')
+
 data_bp = Blueprint(
   'data', __name__, template_folder='templates'
 )
@@ -57,3 +76,186 @@ def protocols():
     }
     return render_template('data/protocols.html', **params)
 
+
+#
+# Submit Trait
+#
+@data_bp.route('/submit-trait')
+@jwt_required()
+def submit_trait_start():
+  """ Submit Trait start page """
+  return render_template('data/submit-trait-start.html', **{
+    'title': 'Submit Trait',
+    'disable_parent_breadcrumb': True
+  })
+
+#
+# Submit Trait Form
+#
+@data_bp.route('/submit-trait/new-submission', methods=['GET', 'POST'])
+@jwt_required()
+def submit_trait_form():
+  """ Trait Submission Form """
+  form = TraitSubmissionForm()
+  user = get_current_user()
+
+  if hasattr(user, 'username') and not form.username.data:
+    form.username.data = user.username
+
+  # Handle form submission
+  if request.method == 'POST':
+    form = TraitSubmissionForm(request.form)
+    form.file.data = request.files.get('file')
+
+    # Validate form fields
+    if not form.validate_on_submit():
+      flash('Please fill out all required fields.', 'warning')
+
+    else:
+      try:
+        # Create a new TraitFile object
+        tf = TraitFile(unique_id())
+
+        # Create a unique filename for file upload
+        hashed_filename = f'{unique_id()}.tsv'
+
+        tf.set_properties(**{
+          # User submitted data
+          'trait_name_user':      bleach.clean(form.trait_name_user.data),
+          'trait_name_display_1': bleach.clean(form.trait_name_display_1.data),
+          'trait_name_display_2': bleach.clean(form.trait_name_display_2.data),
+          'trait_name_display_3': bleach.clean(form.trait_name_display_2.data),
+          'filename':             bleach.clean(form.file.data.filename),
+          'species':              bleach.clean(form.species.data),
+          'description_short':    bleach.clean(form.description_short.data),
+          'description_long':     bleach.clean(form.description_long.data),
+          'units':                bleach.clean(form.units.data),
+          'tags':                 [ bleach.clean(tag) for tag in form.tags.data ],
+          'institution':          bleach.clean(form.institution.data),
+          'source_lab':           bleach.clean(form.source_lab.data),
+          'protocols':            bleach.clean(form.protocols.data),
+          'publication':          bleach.clean(form.publication.data),
+
+          # Internally used data
+          'dataset':           'public',
+          'publish_status':    PublishStatus.UPLOADED,
+          'is_bulk_file':      False,
+          'hashed_filename':   hashed_filename,
+        })
+
+        tf.set_user(user)
+
+        # Save the TraitFile object to Datastore
+        tf.save()
+
+        # Seed to Phenotype Metadata SQL table
+        new_trait = PhenotypeMetadata()
+        new_trait.add_trait(tf)
+
+      except Exception as ex:
+        logger.error(f'Failed to create a trait file {form.trait_name_user.data}: {ex}')
+        flash('Failed to submit a form. Please try again later.', 'danger')
+        abort(500)
+
+      # Save file to GCP bucket
+      species_name = Species.get(form.species.data).name
+      blob_name = f'{MODULE_DB_OPERATIONS_TRAITFILE_PUBLIC_FILEPATH}/{species_name}/{user.name}/{hashed_filename}'
+
+      # Check if the file already exists
+      if check_blob_exists(MODULE_DB_OPERATIONS_BUCKET_NAME, blob_name):
+        flash('File already exists.', 'danger')
+      else:
+        upload_blob_from_file_object(MODULE_DB_OPERATIONS_BUCKET_NAME, form.file.data, blob_name)
+
+      # Reset the file pointer
+      form.file.data.seek(0)
+
+      try:
+        # Seed the file data to Phenotype Database SQL table
+        with LocalUploadFile(form.file.data, valid_file_extensions=TOOL_INPUT_DATA_VALID_FILE_EXTENSIONS) as file:
+
+          # Validate the file
+          try:
+            validate_file(file, [
+                                  StrainValidator( 'strain', species=tf['species'], force_unique=True, force_unique_msgs={} ),
+                                  NumberValidator( None, accept_float=True, accept_na=True ),
+                                ])
+          except Exception as ex:
+            flash(f'Failed to validate the file: {ex.msg}', 'danger')
+            return render_template('data/submit-trait-form.html', **{
+              # Page Info
+              'title': 'Phenotype Database Trait Submission',
+              'tool_alt_parent_breadcrumb': {"title": "Submit Trait", "url": url_for('data.submit_trait_start')},
+
+              # Data
+              'form': form,
+            })
+
+          # Parse the trait file
+          trait_data_list = []
+          with open(file) as f:
+            for idx, row in enumerate( csv.reader(f, delimiter='\t') ):
+              if idx == 0:
+                continue
+              else:
+                trait_data = {
+                  'trait_name':  tf['trait_name_user'],
+                  'strain_name': row[0],
+                  'trait_value': row[1],
+                  'metadata_id': tf.name
+                }
+                trait_data_list.append(trait_data)
+
+        trait_data = PhenotypeDatabase()
+        trait_data.add_trait_data(trait_data_list)
+
+      except FileUploadError as ex:
+        logger.error(f'Failed to upload a file {form.file.data.filename}: {ex}')
+        flash('Failed to submit a form. Please try again later.', 'danger')
+        abort(500)
+
+      flash('Trait submitted successfully.', 'success')
+      # TODO: change the redirect to MTL
+      return redirect(url_for('data.submit_trait_start'))
+
+  return render_template('data/submit-trait-form.html', **{
+    # Page Info
+    'title': 'Phenotype Database Trait Submission',
+    'tool_alt_parent_breadcrumb': {"title": "Submit Trait", "url": url_for('data.submit_trait_start')},
+
+    # Data
+    'form': form,
+  })
+
+#
+# File Upload
+#
+@data_bp.route('/submit-trait/parse-file', methods=['POST'])
+@jwt_required()
+def parse_trait_file():
+  """ Parse the trait file and return the data """
+  try:
+    with LocalUploadFile(request.files.get('file'), valid_file_extensions=TOOL_INPUT_DATA_VALID_FILE_EXTENSIONS) as file:
+      # Validate the file
+      try:
+        species = Species.from_name(request.form.get('species'))
+        validate_file(file, [
+                              StrainValidator( 'strain', species=species, force_unique=True, force_unique_msgs={} ),
+                              NumberValidator( None, accept_float=True, accept_na=True ),
+                            ])
+      except Exception as ex:
+        return jsonify({ 'message': ex.msg }), 500
+
+      # Parse the file
+      with open(file) as f:
+        file_content = []
+        for idx, row in enumerate( csv.reader(f, delimiter='\t') ):
+          file_content.append({'col_1': row[0], 'col_2': row[1]})
+        return jsonify(file_content), 200
+
+  except FileUploadError as ex:
+    return jsonify({ 'message': ex.description }), ex.code
+
+  except Exception as ex:
+    logger.error(f'Failed to parse the file: {ex}')
+    return jsonify({ 'message': 'Failed to parse the file. Please try again later.' }), 500

src/modules/site-v2/base/views/tools/phenotype_database.py

@@ -155,12 +155,12 @@ def get_traits_json():
   """
     Get traits data for non-bulk files in JSON format (include phenotype values)
   """
-  trait_name = request.json.get('trait_name')
-  err_msg = f'Failed to retrieve metadata for trait {trait_name}'
+  trait_id = request.json.get('trait_id')
+  err_msg = f'Failed to retrieve metadata for trait {trait_id}'
 
-  if trait_name:
+  if trait_id:
     try:
-      trait = get_trait(trait_name).to_json_with_values()
+      trait = get_trait(trait_id).to_json_with_values()
       return jsonify(trait)
 
     except Exception as ex:

src/modules/site-v2/constants.py

@@ -37,3 +37,15 @@ class PRICES:
 # TOOL_INPUT_DATA_VALID_FILE_EXTENSIONS = { 'csv', 'tsv' }
 # TOOL_INPUT_DATA_VALID_FILE_EXTENSIONS = { 'csv' }
 TOOL_INPUT_DATA_VALID_FILE_EXTENSIONS = { 'tsv' }
+
+TRAIT_CATEGORY_OPTIONS = [
+  ('Growth/Physiology', 'Growth/Physiology'), 
+  ('Morphology/Development/Lineage/Cell type', 'Morphology/Development/Lineage/Cell type'), 
+  ('Behavior', 'Behavior'), 
+  ('Molecular', 'Molecular'), 
+  ('Stress response', 'Stress response'), 
+  ('Drug/Compound/Condition/Treatment', 'Drug/Compound/Condition/Treatment'), 
+  ('Ecology', 'Ecology'), 
+  ('Genomics', 'Genomics'), 
+  ('Reproduction', 'Reproduction')
+]

src/modules/site-v2/templates/_includes/breadcrumb.html

@@ -11,7 +11,7 @@
             {% elif alt_parent_breadcrumb %}
             <li class="breadcrumb-item">{{ alt_parent_breadcrumb["title"] }}</li>
             {% elif tool_alt_parent_breadcrumb %}
-                {% if tool_alt_parent_breadcrumb["title"] == 'Strain Catalog' %}
+                {% if tool_alt_parent_breadcrumb["title"] == 'Strain Catalog' or  tool_alt_parent_breadcrumb["title"] == 'Submit Trait'%}
                 <li class="breadcrumb-item">{{request.blueprint|replace('_', ' ')|title}}</li>
                 {% endif %}
             <li class="breadcrumb-item"><a href="{{ tool_alt_parent_breadcrumb.url }}">{{ tool_alt_parent_breadcrumb["title"] }}</a></li>

src/modules/site-v2/templates/_includes/macros.html

@@ -26,9 +26,19 @@
   </script>
 {% endmacro %}
 
-{% macro render_field(field, form_prefix=None, form_suffix=None, make_err_container=false) %}
+{% macro render_field(field, form_prefix=None, form_suffix=None, make_err_container=false, has_popover=false, content=None) %}
   <div class="form-group {% if field.errors %}has-error{% endif %}">
-    {{ field.label(class='form-label') }}
+    <div class="label-group d-flex">
+      {{ field.label(class='form-label') }}
+      {% if has_popover %}
+      <!-- Info Popover -->
+      <a class="btn btn-secondary rounded-circle infoButton mx-2" role="button" tabindex="0" data-bs-toggle="popover"
+      data-bs-trigger="focus" data-bs-title="More information"
+      data-bs-content="{{ content }}" aria-label="More information about {{ field.label.text }}"><i class="bi bi-info lh-1"
+          aria-hidden="true"></i></a>
+      <!-- /Info Popover -->
+      {% endif %}
+    </div>
     {% if form_prefix or form_suffix %}
     <div class="input-group">
     {% endif %}
@@ -46,6 +56,7 @@
         {{ field(class="form-control", required=kwargs.get('required', field.flags.required), **kwargs) }}
       {% endif %}
       {% if form_suffix %}<span class="input-group-addon">{{ form_prefix }}</span>{% endif %}
+
     {% if form_prefix or form_suffix %}
     </div>
     {% endif %}
@@ -55,6 +66,7 @@
     {% if make_err_container %}
       <p class='text-danger err-container'></p>
     {% endif %}
+
   </div>
 {% endmacro %}
 

src/modules/site-v2/templates/_includes/navbar.html

@@ -48,6 +48,7 @@
               <li><a class="dropdown-item" href="{{ url_for('data_releases.data_releases') }}"> Data Releases </a></li>
               <li><a class="dropdown-item" href="{{ url_for('data.protocols')     }}"> Protocols </a></li>
               <li><a class="dropdown-item" href="https://docs.google.com/forms/d/e/1FAIpQLSfO0m4UMzTSz79weA2ICkuIHDYnZegqXly4SA15_w3FMuyocQ/viewform" target="_blank"> Submit A Strain <i class="bi bi-box-arrow-right" aria-hidden="true"></i></a><span class="visually-hidden">Link opens in a new tab</span></li>
+              <li><a class="dropdown-item" href="{{ url_for('data.submit_trait_start')     }}"> Submit Trait </a></li>
             </ul>
           </div>
           <div class="nav-item dropdown mx-3">