diff --git a/environment_python_3_5.yml b/environment_python_3_5.yml index 0ab098dff..43b44b4cc 100644 --- a/environment_python_3_5.yml +++ b/environment_python_3_5.yml @@ -8,6 +8,7 @@ dependencies: - bokeh=0.13.0 - crds>=7.2.7 - django=2.1.1 +- inflection=0.3.1 - ipython=6.5.0 - jinja2=2.10 - jwst=0.13.0 diff --git a/environment_python_3_6.yml b/environment_python_3_6.yml index da9fd83ec..3d6eddfc4 100644 --- a/environment_python_3_6.yml +++ b/environment_python_3_6.yml @@ -8,6 +8,7 @@ dependencies: - bokeh=1.2.0 - crds>=7.2.7 - django=2.1.7 +- inflection=0.3.1 - ipython=7.5.0 - jinja2=2.10 - jwst=0.13.1 diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py index 557223b6c..b76a2c3b6 100644 --- a/jwql/database/database_interface.py +++ b/jwql/database/database_interface.py @@ -62,8 +62,7 @@ import socket import pandas as pd -from sqlalchemy import Boolean -from sqlalchemy import Column +from sqlalchemy import Boolean, Column, DateTime, Integer, MetaData, String, Table from sqlalchemy import create_engine from sqlalchemy import Date from sqlalchemy import DateTime @@ -75,11 +74,12 @@ from sqlalchemy import Time from sqlalchemy import UniqueConstraint from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.ext.automap import automap_base from sqlalchemy.orm import sessionmaker from sqlalchemy.orm.query import Query from sqlalchemy.types import ARRAY -from jwql.utils.constants import FILE_SUFFIX_TYPES, JWST_INSTRUMENT_NAMES +from jwql.utils.constants import ANOMALIES, FILE_SUFFIX_TYPES, JWST_INSTRUMENT_NAMES from jwql.utils.utils import get_config @@ -87,6 +87,7 @@ @property def data_frame(self): """Method to return a ``pandas.DataFrame`` of the results""" + return pd.read_sql(self.statement, self.session.bind) Query.data_frame = data_frame @@ -130,11 +131,10 @@ def load_connection(connection_string): base = declarative_base(engine) Session = sessionmaker(bind=engine) session = Session() - meta = MetaData() + meta = MetaData(engine) return session, base, engine, meta - # Import a global session. If running from readthedocs or Jenkins, pass a dummy connection string if 'build' and 'project' in socket.gethostname() or os.path.expanduser('~') == '/home/jenkins': dummy_connection_string = 'postgresql+psycopg2://account:password@hostname:0000/db_name' @@ -144,62 +144,6 @@ def load_connection(connection_string): session, base, engine, meta = load_connection(SETTINGS['connection_string']) -class Anomaly(base): - """ORM for the ``anomalies`` table""" - - # Name the table - __tablename__ = 'anomalies' - - # Define the columns - id = Column(Integer, primary_key=True, nullable=False) - filename = Column(String, nullable=False) - flag_date = Column(DateTime, nullable=False, default=datetime.now()) - bowtie = Column(Boolean, nullable=False, default=False) - snowball = Column(Boolean, nullable=False, default=False) - cosmic_ray_shower = Column(Boolean, nullable=False, default=False) - crosstalk = Column(Boolean, nullable=False, default=False) - cte_correction_error = Column(Boolean, nullable=False, default=False) - data_transfer_error = Column(Boolean, nullable=False, default=False) - detector_ghost = Column(Boolean, nullable=False, default=False) - diamond = Column(Boolean, nullable=False, default=False) - diffraction_spike = Column(Boolean, nullable=False, default=False) - dragon_breath = Column(Boolean, nullable=False, default=False) - earth_limb = Column(Boolean, nullable=False, default=False) - excessive_saturation = Column(Boolean, nullable=False, default=False) - figure8_ghost = Column(Boolean, nullable=False, default=False) - filter_ghost = Column(Boolean, nullable=False, default=False) - fringing = Column(Boolean, nullable=False, default=False) - guidestar_failure = Column(Boolean, nullable=False, default=False) - banding = Column(Boolean, nullable=False, default=False) - persistence = Column(Boolean, nullable=False, default=False) - prominent_blobs = Column(Boolean, nullable=False, default=False) - trail = Column(Boolean, nullable=False, default=False) - scattered_light = Column(Boolean, nullable=False, default=False) - other = Column(Boolean, nullable=False, default=False) - - def __repr__(self): - """Return the canonical string representation of the object""" - - # Get the columns that are True - a_list = [col for col, val in self.__dict__.items() - if val is True and isinstance(val, bool)] - - txt = ('Anomaly {0.id}: {0.filename} flagged at ' - '{0.flag_date} for {1}').format(self, a_list) - - return txt - - @property - def colnames(self): - """A list of all the column names in this table""" - - # Get the columns - a_list = [col for col, val in self.__dict__.items() - if isinstance(val, bool)] - - return a_list - - class FilesystemGeneral(base): """ORM for the general (non instrument specific) filesystem monitor table""" @@ -258,6 +202,40 @@ class Monitor(base): log_file = Column(String(), nullable=False) +def anomaly_orm_factory(class_name): + """Create a ``SQLAlchemy`` ORM Class for an anomaly table. + + Parameters + ---------- + class_name : str + The name of the class to be created + + Returns + ------- + class : obj + The ``SQLAlchemy`` ORM + """ + + # Initialize a dictionary to hold the column metadata + data_dict = {} + data_dict['__tablename__'] = class_name.lower() + + # Define anomaly table column names + data_dict['columns'] = ANOMALIES + data_dict['names'] = [name.replace('_', ' ') for name in data_dict['columns']] + + # Create a table with the appropriate Columns + data_dict['id'] = Column(Integer, primary_key=True, nullable=False) + data_dict['rootname'] = Column(String(), nullable=False) + data_dict['flag_date'] = Column(DateTime, nullable=False) + data_dict['user'] = Column(String(), nullable=False) + + for column in data_dict['columns']: + data_dict[column] = Column(Boolean, nullable=False, default=False) + + return type(class_name, (base,), data_dict) + + def get_monitor_columns(data_dict, table_name): """Read in the corresponding table definition text file to generate ``SQLAlchemy`` columns for the table. @@ -379,6 +357,7 @@ class : obj # Create tables from ORM factory +Anomaly = anomaly_orm_factory('anomaly') NIRCamDarkQueryHistory = monitor_orm_factory('nircam_dark_query_history') NIRCamDarkPixelStats = monitor_orm_factory('nircam_dark_pixel_stats') NIRCamDarkDarkCurrent = monitor_orm_factory('nircam_dark_dark_current') diff --git a/jwql/jwql_monitors/generate_preview_images.py b/jwql/jwql_monitors/generate_preview_images.py index 107b8e190..d5aff6986 100755 --- a/jwql/jwql_monitors/generate_preview_images.py +++ b/jwql/jwql_monitors/generate_preview_images.py @@ -319,7 +319,7 @@ def create_mosaic(filenames): elif datadim == 3: full_array = np.zeros((datashape[0], full_ydim, full_xdim)) * np.nan else: - raise ValueError(('Difference image for {} must be either 2D or 3D.'.format(filenames[0]))) + raise ValueError('Difference image for {} must be either 2D or 3D.'.format(filenames[0])) # Place the data from the individual detectors in the appropriate # places in the final image diff --git a/jwql/tests/test_database_interface.py b/jwql/tests/test_database_interface.py new file mode 100755 index 000000000..96bb884d0 --- /dev/null +++ b/jwql/tests/test_database_interface.py @@ -0,0 +1,51 @@ +#! /usr/bin/env python + +"""Tests for the ``database_interface.py`` module. + +Authors +------- + + - Joe Filippazzo + - Matthew Bourque + +Use +--- + + These tests can be run via the command line (omit the ``-s`` to + suppress verbose output to stdout): + :: + + pytest -s database_interface.py +""" + +import datetime +import os +import pytest +import random +import string + +from jwql.database import database_interface as di + +# Determine if tests are being run on jenkins +ON_JENKINS = os.path.expanduser('~') == '/home/jenkins' + + +@pytest.mark.skipif(ON_JENKINS, reason='Requires access to development database server.') +def test_anomaly_table(): + """Test to see that the database has an anomalies table""" + + assert 'anomaly' in di.engine.table_names() + + +@pytest.mark.skipif(ON_JENKINS, reason='Requires access to development database server.') +def test_anomaly_records(): + """Test to see that new records can be entered""" + + # Add some data + random_string = ''.join(random.SystemRandom().choice(string.ascii_lowercase + string.ascii_uppercase + string.digits) for _ in range(10)) + di.session.add(di.Anomaly(rootname=random_string, flag_date=datetime.datetime.today(), user='test', ghost=True)) + di.session.commit() + + # Test the ghosts column + ghosts = di.session.query(di.Anomaly).filter(di.Anomaly.ghost == "True") + assert ghosts.data_frame.iloc[0]['ghost'] == True diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index 3cb8fa7b8..1db166b1a 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -20,11 +20,21 @@ ``utils.py`` """ +import inflection + + # Defines the x and y coordinates of amplifier boundaries AMPLIFIER_BOUNDARIES = {'nircam': {'1': [(0, 0), (512, 2048)], '2': [(512, 0), (1024, 2048)], '3': [(1024, 0), (1536, 2048)], '4': [(1536, 0), (2048, 2048)]} } +# Defines the possible anomalies to flag through the web app +ANOMALIES = ['snowball', 'cosmic_ray_shower', 'crosstalk', 'data_transfer_error', 'diffraction_spike', + 'excessive_saturation', 'ghost', 'guidestar_failure', 'persistence', 'satellite_trail', 'other'] + +# Defines the possible anomalies (with rendered name) to flag through the web app +ANOMALY_CHOICES = [(anomaly, inflection.titleize(anomaly)) for anomaly in ANOMALIES] + # Possible suffix types for nominal files GENERIC_SUFFIX_TYPES = ['uncal', 'cal', 'rateints', 'rate', 'trapsfilled', 'i2d', 'x1dints', 'x1d', 's2d', 's3d', 'dark', 'crfints', diff --git a/jwql/utils/preview_image.py b/jwql/utils/preview_image.py index db0e5dbb5..0369dec36 100755 --- a/jwql/utils/preview_image.py +++ b/jwql/utils/preview_image.py @@ -204,7 +204,8 @@ def get_data(self, filename, ext): else: data = hdulist[ext].data.astype(np.float) else: - raise ValueError(('WARNING: no {} extension in {}!'.format(ext, filename))) + raise ValueError('WARNING: no {} extension in {}!'.format(ext, filename)) + if 'PIXELDQ' in extnames: dq = hdulist['PIXELDQ'].data dq = (dq & dqflags.pixel['NON_SCIENCE'] == 0) @@ -212,7 +213,6 @@ def get_data(self, filename, ext): yd, xd = data.shape[-2:] dq = np.ones((yd, xd), dtype="bool") - # Collect information on aperture location within the # full detector. This is needed for mosaicking NIRCam # detectors later. @@ -225,7 +225,7 @@ def get_data(self, filename, ext): logging.warning('SUBSTR and SUBSIZE header keywords not found') else: - raise FileNotFoundError(('WARNING: {} does not exist!'.format(filename))) + raise FileNotFoundError('WARNING: {} does not exist!'.format(filename)) return data, dq @@ -266,7 +266,7 @@ def make_figure(self, image, integration_number, min_value, max_value, # Check the input scaling if scale not in ['linear', 'log']: - raise ValueError(('WARNING: scaling option {} not supported.'.format(scale))) + raise ValueError('WARNING: scaling option {} not supported.'.format(scale)) # Set the figure size yd, xd = image.shape diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 9184ac10c..039bf07ba 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -44,6 +44,7 @@ from astroquery.mast import Mast from jwedb.edb_interface import mnemonic_inventory +from jwql.database import database_interface as di from jwql.edb.engineering_database import get_mnemonic, get_mnemonic_info from jwql.instrument_monitors.miri_monitors.data_trending import dashboard as miri_dash from jwql.instrument_monitors.nirspec_monitors.data_trending import dashboard as nirspec_dash @@ -146,6 +147,33 @@ def get_all_proposals(): return proposals +def get_current_flagged_anomalies(rootname): + """Return a list of currently flagged anomalies for the given + ``rootname`` + + Parameters + ---------- + rootname : str + The rootname of interest (e.g. + ``jw86600008001_02101_00001_guider2/``) + + Returns + ------- + current_anomalies : list + A list of currently flagged anomalies for the given ``rootname`` + (e.g. ``['snowball', 'crosstalk']``) + """ + + query = di.session.query(di.Anomaly).filter(di.Anomaly.rootname == rootname).order_by(di.Anomaly.flag_date.desc()).limit(1) + all_records = query.data_frame + if not all_records.empty: + current_anomalies = [col for col, val in np.sum(all_records, axis=0).items() if val] + else: + current_anomalies = [] + + return current_anomalies + + def get_dashboard_components(): """Build and return dictionaries containing components and html needed for the dashboard. diff --git a/jwql/website/apps/jwql/forms.py b/jwql/website/apps/jwql/forms.py index c869c0bb6..06e8aed35 100644 --- a/jwql/website/apps/jwql/forms.py +++ b/jwql/website/apps/jwql/forms.py @@ -10,6 +10,7 @@ - Lauren Chambers - Johannes Sahlmann + - Matthew Bourque Use --- @@ -40,20 +41,55 @@ def view_function(request): placed in the ``jwql/utils/`` directory. """ + +import datetime import glob import os from astropy.time import Time, TimeDelta from django import forms from django.shortcuts import redirect - from jwedb.edb_interface import is_valid_mnemonic -from jwql.utils.constants import JWST_INSTRUMENT_NAMES_SHORTHAND + +from jwql.database import database_interface as di +from jwql.utils.constants import ANOMALY_CHOICES, JWST_INSTRUMENT_NAMES_SHORTHAND from jwql.utils.utils import get_config, filename_parser FILESYSTEM_DIR = os.path.join(get_config()['jwql_dir'], 'filesystem') +class AnomalySubmitForm(forms.Form): + """A multiple choice field for specifying flagged anomalies.""" + + # Define anomaly choice field + anomaly_choices = forms.MultipleChoiceField(choices=ANOMALY_CHOICES, widget=forms.CheckboxSelectMultiple()) + + def update_anomaly_table(self, rootname, user, anomaly_choices): + """Updated the ``anomaly`` table of the database with flagged + anomaly information + + Parameters + ---------- + rootname : str + The rootname of the image to flag (e.g. + ``jw86600008001_02101_00001_guider2``) + user : str + The ``ezid`` of the authenticated user that is flagging the + anomaly + anomaly_choices : list + A list of anomalies that are to be flagged (e.g. + ``['snowball', 'crosstalk']``) + """ + + data_dict = {} + data_dict['rootname'] = rootname + data_dict['flag_date'] = datetime.datetime.now() + data_dict['user'] = user + for choice in anomaly_choices: + data_dict[choice] = True + di.engine.execute(di.Anomaly.__table__.insert(), data_dict) + + class FileSearchForm(forms.Form): """Single-field form to search for a proposal or fileroot.""" diff --git a/jwql/website/apps/jwql/static/css/jwql.css b/jwql/website/apps/jwql/static/css/jwql.css index 364dff1c9..b801c8358 100644 --- a/jwql/website/apps/jwql/static/css/jwql.css +++ b/jwql/website/apps/jwql/static/css/jwql.css @@ -1,5 +1,15 @@ +.anomaly_choice { + list-style: none; +} + +.anomaly_form { + position: absolute; + top: 50%; + transform: translateY(-50%); +} + .APT_parameters { - width: 20% + width: 20% } .banner { @@ -182,6 +192,11 @@ display : inline; display: inline; } +#id_anomaly_choices { + list-style: none; + padding-left: 0; +} + /*Don't let the search bar be super long*/ .input-group { width: 250px; @@ -501,3 +516,8 @@ li.dropdown:hover .dropdown-menu { display: block; } +ul.no-bullets { + list-style: none; + padding-left:10px; + line-height:25px; +} diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html index c3d85a21c..8362ab77a 100644 --- a/jwql/website/apps/jwql/templates/view_image.html +++ b/jwql/website/apps/jwql/templates/view_image.html @@ -40,21 +40,55 @@

{{ file_root }}

View Proposal {{ file_root[2:7] }}

- -
- - - Displaying integration 1/1
- {{ file_root }}_cal_integ0.jpg - - -
+
+ + +
+ + + Displaying integration 1/1
+ {{ file_root }}_cal_integ0.jpg + + +
+ + +
+ +
+
Submit Anomaly
+
+ + {% if form.errors %} + {% for field in form %} + {% for error in field.errors %} +
+ {{ error|escape }} +
+ {% endfor %} + {% endfor %} + {% endif %} + + + {{ csrf_input }} + + + {% for field in form %} + {% for subwidget in field.subwidgets %} +
  • {{subwidget}}
    • + {% endfor %} + {% endfor %} + +
    +
    +
    + +

    Download FITS Download JPEG - Submit Anomaly

    @@ -70,42 +104,6 @@

    {{ file_root }}

    Lauren needs to figure out what to do with these: {{suffixes}} {% endif %} - - - - {% endblock %} \ No newline at end of file diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 13b9e1c71..4d743db7e 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -35,6 +35,7 @@ placed in the ``jwql/utils/`` directory. """ +import datetime import os from django.http import JsonResponse @@ -45,15 +46,15 @@ from .data_containers import get_filenames_by_instrument from .data_containers import get_header_info from .data_containers import get_image_info +from .data_containers import get_current_flagged_anomalies from .data_containers import get_proposal_info from .data_containers import random_404_page from .data_containers import thumbnails from .data_containers import thumbnails_ajax from .data_containers import data_trending from .data_containers import nirspec_trending -from .forms import FileSearchForm +from .forms import AnomalySubmitForm, FileSearchForm from .oauth import auth_info, auth_required -import jwql from jwql.utils.constants import JWST_INSTRUMENT_NAMES, MONITORS, JWST_INSTRUMENT_NAMES_MIXEDCASE from jwql.utils.utils import get_base_url, get_config @@ -461,9 +462,11 @@ def view_image(request, user, inst, file_root, rewrite=False): ---------- request : HttpRequest object Incoming request from the webpage + user : dict + A dictionary of user credentials. inst : str Name of JWST instrument - file : str + file_root : str FITS filename of selected image in filesystem rewrite : bool, optional Regenerate the jpg preview of `file` if it already exists? @@ -473,16 +476,32 @@ def view_image(request, user, inst, file_root, rewrite=False): HttpResponse object Outgoing response sent to the webpage """ + # Ensure the instrument is correctly capitalized inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()] template = 'view_image.html' image_info = get_image_info(file_root, rewrite) + + # Determine current flagged anomalies + current_anomalies = get_current_flagged_anomalies(file_root) + + # Create a form instance + form = AnomalySubmitForm(request.POST or None, initial={'anomaly_choices': current_anomalies}) + + # If this is a POST request, process the form data + if request.method == 'POST': + anomaly_choices = dict(request.POST)['anomaly_choices'] + if form.is_valid(): + form.update_anomaly_table(file_root, user['ezid'], anomaly_choices) + + # Build the context context = {'inst': inst, 'file_root': file_root, 'jpg_files': image_info['all_jpegs'], 'fits_files': image_info['all_files'], 'suffixes': image_info['suffixes'], - 'num_ints': image_info['num_ints']} + 'num_ints': image_info['num_ints'], + 'form': form} return render(request, template, context) diff --git a/requirements.txt b/requirements.txt index 2e5e8e1f2..a8b804c75 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ astroquery==0.3.9 authlib==0.11 bokeh==1.2.0 django==2.2.2 +inflection==0.3.1 ipython==7.5.0 jinja2==2.10.1 jwedb>=0.0.3