Merge branch 'develop' into release-0.20.0

environment_python_3_5.yml

@@ -8,6 +8,7 @@ dependencies:
 - bokeh=0.13.0
 - crds>=7.2.7
 - django=2.1.1
+- inflection=0.3.1
 - ipython=6.5.0
 - jinja2=2.10
 - jwst=0.13.0

environment_python_3_6.yml

@@ -8,6 +8,7 @@ dependencies:
 - bokeh=1.2.0
 - crds>=7.2.7
 - django=2.1.7
+- inflection=0.3.1
 - ipython=7.5.0
 - jinja2=2.10
 - jwst=0.13.1

jwql/database/database_interface.py

@@ -62,8 +62,7 @@
 import socket
 
 import pandas as pd
-from sqlalchemy import Boolean
-from sqlalchemy import Column
+from sqlalchemy import Boolean, Column, DateTime, Integer, MetaData, String, Table
 from sqlalchemy import create_engine
 from sqlalchemy import Date
 from sqlalchemy import DateTime
@@ -75,18 +74,20 @@
 from sqlalchemy import Time
 from sqlalchemy import UniqueConstraint
 from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.ext.automap import automap_base
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy.orm.query import Query
 from sqlalchemy.types import ARRAY
 
-from jwql.utils.constants import FILE_SUFFIX_TYPES, JWST_INSTRUMENT_NAMES
+from jwql.utils.constants import ANOMALIES, FILE_SUFFIX_TYPES, JWST_INSTRUMENT_NAMES
 from jwql.utils.utils import get_config
 
 
 # Monkey patch Query with data_frame method
 @property
 def data_frame(self):
     """Method to return a ``pandas.DataFrame`` of the results"""
+
     return pd.read_sql(self.statement, self.session.bind)
 
 Query.data_frame = data_frame
@@ -130,11 +131,10 @@ def load_connection(connection_string):
     base = declarative_base(engine)
     Session = sessionmaker(bind=engine)
     session = Session()
-    meta = MetaData()
+    meta = MetaData(engine)
 
     return session, base, engine, meta
 
-
 # Import a global session.  If running from readthedocs or Jenkins, pass a dummy connection string
 if 'build' and 'project' in socket.gethostname() or os.path.expanduser('~') == '/home/jenkins':
     dummy_connection_string = 'postgresql+psycopg2://account:password@hostname:0000/db_name'
@@ -144,62 +144,6 @@ def load_connection(connection_string):
     session, base, engine, meta = load_connection(SETTINGS['connection_string'])
 
 
-class Anomaly(base):
-    """ORM for the ``anomalies`` table"""
-
-    # Name the table
-    __tablename__ = 'anomalies'
-
-    # Define the columns
-    id = Column(Integer, primary_key=True, nullable=False)
-    filename = Column(String, nullable=False)
-    flag_date = Column(DateTime, nullable=False, default=datetime.now())
-    bowtie = Column(Boolean, nullable=False, default=False)
-    snowball = Column(Boolean, nullable=False, default=False)
-    cosmic_ray_shower = Column(Boolean, nullable=False, default=False)
-    crosstalk = Column(Boolean, nullable=False, default=False)
-    cte_correction_error = Column(Boolean, nullable=False, default=False)
-    data_transfer_error = Column(Boolean, nullable=False, default=False)
-    detector_ghost = Column(Boolean, nullable=False, default=False)
-    diamond = Column(Boolean, nullable=False, default=False)
-    diffraction_spike = Column(Boolean, nullable=False, default=False)
-    dragon_breath = Column(Boolean, nullable=False, default=False)
-    earth_limb = Column(Boolean, nullable=False, default=False)
-    excessive_saturation = Column(Boolean, nullable=False, default=False)
-    figure8_ghost = Column(Boolean, nullable=False, default=False)
-    filter_ghost = Column(Boolean, nullable=False, default=False)
-    fringing = Column(Boolean, nullable=False, default=False)
-    guidestar_failure = Column(Boolean, nullable=False, default=False)
-    banding = Column(Boolean, nullable=False, default=False)
-    persistence = Column(Boolean, nullable=False, default=False)
-    prominent_blobs = Column(Boolean, nullable=False, default=False)
-    trail = Column(Boolean, nullable=False, default=False)
-    scattered_light = Column(Boolean, nullable=False, default=False)
-    other = Column(Boolean, nullable=False, default=False)
-
-    def __repr__(self):
-        """Return the canonical string representation of the object"""
-
-        # Get the columns that are True
-        a_list = [col for col, val in self.__dict__.items()
-                  if val is True and isinstance(val, bool)]
-
-        txt = ('Anomaly {0.id}: {0.filename} flagged at '
-               '{0.flag_date} for {1}').format(self, a_list)
-
-        return txt
-
-    @property
-    def colnames(self):
-        """A list of all the column names in this table"""
-
-        # Get the columns
-        a_list = [col for col, val in self.__dict__.items()
-                  if isinstance(val, bool)]
-
-        return a_list
-
-
 class FilesystemGeneral(base):
     """ORM for the general (non instrument specific) filesystem monitor
     table"""
@@ -258,6 +202,40 @@ class Monitor(base):
     log_file = Column(String(), nullable=False)
 
 
+def anomaly_orm_factory(class_name):
+    """Create a ``SQLAlchemy`` ORM Class for an anomaly table.
+
+    Parameters
+    ----------
+    class_name : str
+        The name of the class to be created
+
+    Returns
+    -------
+    class : obj
+        The ``SQLAlchemy`` ORM
+    """
+
+    # Initialize a dictionary to hold the column metadata
+    data_dict = {}
+    data_dict['__tablename__'] = class_name.lower()
+
+    # Define anomaly table column names
+    data_dict['columns'] = ANOMALIES
+    data_dict['names'] = [name.replace('_', ' ') for name in data_dict['columns']]
+
+    # Create a table with the appropriate Columns
+    data_dict['id'] = Column(Integer, primary_key=True, nullable=False)
+    data_dict['rootname'] = Column(String(), nullable=False)
+    data_dict['flag_date'] = Column(DateTime, nullable=False)
+    data_dict['user'] = Column(String(), nullable=False)
+
+    for column in data_dict['columns']:
+        data_dict[column] = Column(Boolean, nullable=False, default=False)
+
+    return type(class_name, (base,), data_dict)
+
+
 def get_monitor_columns(data_dict, table_name):
     """Read in the corresponding table definition text file to
     generate ``SQLAlchemy`` columns for the table.
@@ -379,6 +357,7 @@ class : obj
 
 
 # Create tables from ORM factory
+Anomaly = anomaly_orm_factory('anomaly')
 NIRCamDarkQueryHistory = monitor_orm_factory('nircam_dark_query_history')
 NIRCamDarkPixelStats = monitor_orm_factory('nircam_dark_pixel_stats')
 NIRCamDarkDarkCurrent = monitor_orm_factory('nircam_dark_dark_current')

jwql/jwql_monitors/generate_preview_images.py

@@ -319,7 +319,7 @@ def create_mosaic(filenames):
     elif datadim == 3:
         full_array = np.zeros((datashape[0], full_ydim, full_xdim)) * np.nan
     else:
-        raise ValueError(('Difference image for {} must be either 2D or 3D.'.format(filenames[0])))
+        raise ValueError('Difference image for {} must be either 2D or 3D.'.format(filenames[0]))
 
     # Place the data from the individual detectors in the appropriate
     # places in the final image

jwql/tests/test_database_interface.py

@@ -0,0 +1,51 @@
+#! /usr/bin/env python
+
+"""Tests for the ``database_interface.py`` module.
+
+Authors
+-------
+
+    - Joe Filippazzo
+    - Matthew Bourque
+
+Use
+---
+
+    These tests can be run via the command line (omit the ``-s`` to
+    suppress verbose output to stdout):
+    ::
+
+        pytest -s database_interface.py
+"""
+
+import datetime
+import os
+import pytest
+import random
+import string
+
+from jwql.database import database_interface as di
+
+# Determine if tests are being run on jenkins
+ON_JENKINS = os.path.expanduser('~') == '/home/jenkins'
+
+
+@pytest.mark.skipif(ON_JENKINS, reason='Requires access to development database server.')
+def test_anomaly_table():
+    """Test to see that the database has an anomalies table"""
+
+    assert 'anomaly' in di.engine.table_names()
+
+
+@pytest.mark.skipif(ON_JENKINS, reason='Requires access to development database server.')
+def test_anomaly_records():
+    """Test to see that new records can be entered"""
+
+    # Add some data
+    random_string = ''.join(random.SystemRandom().choice(string.ascii_lowercase + string.ascii_uppercase + string.digits) for _ in range(10))
+    di.session.add(di.Anomaly(rootname=random_string, flag_date=datetime.datetime.today(), user='test', ghost=True))
+    di.session.commit()
+
+    # Test the ghosts column
+    ghosts = di.session.query(di.Anomaly).filter(di.Anomaly.ghost == "True")
+    assert ghosts.data_frame.iloc[0]['ghost'] == True

jwql/utils/constants.py

@@ -20,11 +20,21 @@
     ``utils.py``
 """
 
+import inflection
+
+
 # Defines the x and y coordinates of amplifier boundaries
 AMPLIFIER_BOUNDARIES = {'nircam': {'1': [(0, 0), (512, 2048)], '2': [(512, 0), (1024, 2048)],
                                    '3': [(1024, 0), (1536, 2048)], '4': [(1536, 0), (2048, 2048)]}
                         }
 
+# Defines the possible anomalies to flag through the web app
+ANOMALIES = ['snowball', 'cosmic_ray_shower', 'crosstalk', 'data_transfer_error', 'diffraction_spike',
+             'excessive_saturation', 'ghost', 'guidestar_failure', 'persistence', 'satellite_trail', 'other']
+
+# Defines the possible anomalies (with rendered name) to flag through the web app
+ANOMALY_CHOICES = [(anomaly, inflection.titleize(anomaly)) for anomaly in ANOMALIES]
+
 # Possible suffix types for nominal files
 GENERIC_SUFFIX_TYPES = ['uncal', 'cal', 'rateints', 'rate', 'trapsfilled', 'i2d',
                         'x1dints', 'x1d', 's2d', 's3d', 'dark', 'crfints',

jwql/utils/preview_image.py

@@ -204,15 +204,15 @@ def get_data(self, filename, ext):
                     else:
                         data = hdulist[ext].data.astype(np.float)
                 else:
-                    raise ValueError(('WARNING: no {} extension in {}!'.format(ext, filename)))
+                    raise ValueError('WARNING: no {} extension in {}!'.format(ext, filename))
+
                 if 'PIXELDQ' in extnames:
                     dq = hdulist['PIXELDQ'].data
                     dq = (dq & dqflags.pixel['NON_SCIENCE'] == 0)
                 else:
                     yd, xd = data.shape[-2:]
                     dq = np.ones((yd, xd), dtype="bool")
 
-
                 # Collect information on aperture location within the
                 # full detector. This is needed for mosaicking NIRCam
                 # detectors later.
@@ -225,7 +225,7 @@ def get_data(self, filename, ext):
                     logging.warning('SUBSTR and SUBSIZE header keywords not found')
 
         else:
-            raise FileNotFoundError(('WARNING: {} does not exist!'.format(filename)))
+            raise FileNotFoundError('WARNING: {} does not exist!'.format(filename))
 
         return data, dq
 
@@ -266,7 +266,7 @@ def make_figure(self, image, integration_number, min_value, max_value,
 
         # Check the input scaling
         if scale not in ['linear', 'log']:
-            raise ValueError(('WARNING: scaling option {} not supported.'.format(scale)))
+            raise ValueError('WARNING: scaling option {} not supported.'.format(scale))
 
         # Set the figure size
         yd, xd = image.shape

jwql/website/apps/jwql/data_containers.py

@@ -44,6 +44,7 @@
 from astroquery.mast import Mast
 from jwedb.edb_interface import mnemonic_inventory
 
+from jwql.database import database_interface as di
 from jwql.edb.engineering_database import get_mnemonic, get_mnemonic_info
 from jwql.instrument_monitors.miri_monitors.data_trending import dashboard as miri_dash
 from jwql.instrument_monitors.nirspec_monitors.data_trending import dashboard as nirspec_dash
@@ -146,6 +147,33 @@ def get_all_proposals():
     return proposals
 
 
+def get_current_flagged_anomalies(rootname):
+    """Return a list of currently flagged anomalies for the given
+    ``rootname``
+
+    Parameters
+    ----------
+    rootname : str
+        The rootname of interest (e.g.
+        ``jw86600008001_02101_00001_guider2/``)
+
+    Returns
+    -------
+    current_anomalies : list
+        A list of currently flagged anomalies for the given ``rootname``
+        (e.g. ``['snowball', 'crosstalk']``)
+    """
+
+    query = di.session.query(di.Anomaly).filter(di.Anomaly.rootname == rootname).order_by(di.Anomaly.flag_date.desc()).limit(1)
+    all_records = query.data_frame
+    if not all_records.empty:
+        current_anomalies = [col for col, val in np.sum(all_records, axis=0).items() if val]
+    else:
+        current_anomalies = []
+
+    return current_anomalies
+
+
 def get_dashboard_components():
     """Build and return dictionaries containing components and html
     needed for the dashboard.

jwql/website/apps/jwql/forms.py

@@ -10,6 +10,7 @@
 
     - Lauren Chambers
     - Johannes Sahlmann
+    - Matthew Bourque
 
 Use
 ---
@@ -40,20 +41,55 @@ def view_function(request):
     placed in the ``jwql/utils/`` directory.
 
 """
+
+import datetime
 import glob
 import os
 
 from astropy.time import Time, TimeDelta
 from django import forms
 from django.shortcuts import redirect
-
 from jwedb.edb_interface import is_valid_mnemonic
-from jwql.utils.constants import JWST_INSTRUMENT_NAMES_SHORTHAND
+
+from jwql.database import database_interface as di
+from jwql.utils.constants import ANOMALY_CHOICES, JWST_INSTRUMENT_NAMES_SHORTHAND
 from jwql.utils.utils import get_config, filename_parser
 
 FILESYSTEM_DIR = os.path.join(get_config()['jwql_dir'], 'filesystem')
 
 
+class AnomalySubmitForm(forms.Form):
+    """A multiple choice field for specifying flagged anomalies."""
+
+    # Define anomaly choice field
+    anomaly_choices = forms.MultipleChoiceField(choices=ANOMALY_CHOICES, widget=forms.CheckboxSelectMultiple())
+
+    def update_anomaly_table(self, rootname, user, anomaly_choices):
+        """Updated the ``anomaly`` table of the database with flagged
+        anomaly information
+
+        Parameters
+        ----------
+        rootname : str
+            The rootname of the image to flag (e.g.
+            ``jw86600008001_02101_00001_guider2``)
+        user : str
+            The ``ezid`` of the authenticated user that is flagging the
+            anomaly
+        anomaly_choices : list
+            A list of anomalies that are to be flagged (e.g.
+            ``['snowball', 'crosstalk']``)
+        """
+
+        data_dict = {}
+        data_dict['rootname'] = rootname
+        data_dict['flag_date'] = datetime.datetime.now()
+        data_dict['user'] = user
+        for choice in anomaly_choices:
+            data_dict[choice] = True
+        di.engine.execute(di.Anomaly.__table__.insert(), data_dict)
+
+
 class FileSearchForm(forms.Form):
     """Single-field form to search for a proposal or fileroot."""