From 2f47bc358d77edc35ec8d6826b2bfbf542e08848 Mon Sep 17 00:00:00 2001
From: Teagan King <55458851+tnking97@users.noreply.github.com>
Date: Thu, 14 Oct 2021 16:00:00 -0400
Subject: [PATCH 01/18] initial grating wheel sensor telemetry monitor pages

---
 docs/source/common_monitors.rst               |   6 +
 docs/source/tests.rst                         |   6 +
 jwql/database/database_interface.py           |   2 +
 .../nirspec/nirspec_grating_query_history.txt |   8 +
 .../nirspec/nirspec_grating_stats.txt         |  11 +
 .../common_monitors/grating_monitor.py        | 467 ++++++++++++++++++
 jwql/tests/test_grating_monitor.py            |  50 ++
 jwql/utils/constants.py                       |   6 +-
 jwql/website/apps/jwql/bokeh_containers.py    |  46 +-
 .../apps/jwql/monitor_pages/__init__.py       |   1 +
 .../monitor_pages/monitor_grating_bokeh.py    | 117 +++++
 .../yaml/monitor_grating_interface.yaml       | 170 +++++++
 jwql/website/apps/jwql/monitor_views.py       |  33 ++
 .../apps/jwql/templates/grating_monitor.html  |  30 ++
 jwql/website/apps/jwql/urls.py                |   1 +
 15 files changed, 952 insertions(+), 2 deletions(-)
 create mode 100644 jwql/database/monitor_table_definitions/nirspec/nirspec_grating_query_history.txt
 create mode 100644 jwql/database/monitor_table_definitions/nirspec/nirspec_grating_stats.txt
 create mode 100755 jwql/instrument_monitors/common_monitors/grating_monitor.py
 create mode 100644 jwql/tests/test_grating_monitor.py
 create mode 100644 jwql/website/apps/jwql/monitor_pages/monitor_grating_bokeh.py
 create mode 100644 jwql/website/apps/jwql/monitor_pages/yaml/monitor_grating_interface.yaml
 create mode 100644 jwql/website/apps/jwql/templates/grating_monitor.html

diff --git a/docs/source/common_monitors.rst b/docs/source/common_monitors.rst
index b67bb5d2e..f1b34d7ca 100644
--- a/docs/source/common_monitors.rst
+++ b/docs/source/common_monitors.rst
@@ -20,6 +20,12 @@ dark_monitor.py
     :members:
     :undoc-members:
 
+grating_monitor.py
+--------------------
+.. automodule:: jwql.instrument_monitors.common_monitors.grating_monitor
+    :members:
+    :undoc-members:
+
 readnoise_monitor.py
 --------------------
 .. automodule:: jwql.instrument_monitors.common_monitors.readnoise_monitor
diff --git a/docs/source/tests.rst b/docs/source/tests.rst
index fd1594a49..1c5c65072 100644
--- a/docs/source/tests.rst
+++ b/docs/source/tests.rst
@@ -56,6 +56,12 @@ test_edb.py
     :members:
     :undoc-members:
 
+test_grating_monitor.py
+--------------------
+.. automodule:: jwql.tests.test_grating_monitor
+    :members:
+    :undoc-members:
+
 test_instrument_properties.py
 -----------------------------
 .. automodule:: jwql.tests.test_instrument_properties
diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py
index 0a9378959..b9e320ec8 100755
--- a/jwql/database/database_interface.py
+++ b/jwql/database/database_interface.py
@@ -426,6 +426,8 @@ def set_read_permissions():
 NIRISSBiasStats = monitor_orm_factory('niriss_bias_stats')
 NIRSpecBiasQueryHistory = monitor_orm_factory('nirspec_bias_query_history')
 NIRSpecBiasStats = monitor_orm_factory('nirspec_bias_stats')
+NIRSpecGratingQueryHistory = monitor_orm_factory('nirspec_grating_query_history')
+NIRSpecGratingStats = monitor_orm_factory('nirspec_grating_stats')
 NIRCamBadPixelQueryHistory = monitor_orm_factory('nircam_bad_pixel_query_history')
 NIRCamBadPixelStats = monitor_orm_factory('nircam_bad_pixel_stats')
 NIRISSBadPixelQueryHistory = monitor_orm_factory('niriss_bad_pixel_query_history')
diff --git a/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_query_history.txt b/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_query_history.txt
new file mode 100644
index 000000000..c6deea152
--- /dev/null
+++ b/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_query_history.txt
@@ -0,0 +1,8 @@
+INSTRUMENT, string
+APERTURE, string
+START_TIME_MJD, float
+END_TIME_MJD, float
+ENTRIES_FOUND, integer
+FILES_FOUND, integer
+RUN_MONITOR, bool
+ENTRY_DATE, datetime
\ No newline at end of file
diff --git a/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_stats.txt b/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_stats.txt
new file mode 100644
index 000000000..89a359b72
--- /dev/null
+++ b/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_stats.txt
@@ -0,0 +1,11 @@
+APERTURE, string
+DETECTOR, string
+READ_PATTERN, string
+NINTS, string
+NGROUPS, string
+EXPSTART, string
+INRSH_GWA_ADCMGAIN, float
+INRSH_GWA_ADCMOFFSET, float
+INRSH_GWA_MOTOR_VREF, float
+INRSI_C_GWA_X_POSITION, float
+INRSI_C_GWA_Y_POSITION, float
diff --git a/jwql/instrument_monitors/common_monitors/grating_monitor.py b/jwql/instrument_monitors/common_monitors/grating_monitor.py
new file mode 100755
index 000000000..6ffab5038
--- /dev/null
+++ b/jwql/instrument_monitors/common_monitors/grating_monitor.py
@@ -0,0 +1,467 @@
+#! /usr/bin/env python
+
+"""This module contains code for the grating wheel monitor, which monitors
+the grating wheel positions over time. This code has been adapted from bias_monitor.py
+
+Author
+------
+    - Teagan King
+
+Use
+---
+    This module can be used from the command line as such:
+
+    ::
+
+        python grating_monitor.py
+"""
+
+from collections import OrderedDict
+import datetime
+import logging
+import os
+
+from astropy.io import fits
+from astropy.stats import sigma_clip, sigma_clipped_stats
+from astropy.time import Time
+from astropy.visualization import ZScaleInterval
+import matplotlib
+matplotlib.use('Agg')
+import matplotlib.pyplot as plt
+from mpl_toolkits.axes_grid1 import make_axes_locatable
+import numpy as np
+from pysiaf import Siaf
+from sqlalchemy.sql.expression import and_
+
+from jwql.database.database_interface import session
+from jwql.database.database_interface import NIRSpecGratingQueryHistory, NIRSpecGratingStats
+from jwql.instrument_monitors import pipeline_tools
+from jwql.instrument_monitors.common_monitors.dark_monitor import mast_query_darks
+from jwql.utils import instrument_properties
+from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE
+from jwql.utils.logging_functions import log_info, log_fail
+from jwql.utils.monitor_utils import update_monitor_table
+from jwql.utils.permissions import set_permissions
+from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config, initialize_instrument_monitor
+
+
+class Grating():
+    """Class for executing the grating wheel monitor.
+
+    This class will search for a particular type of file in the file
+    system for each instrument and will run the monitor on these files.
+    The monitor will output the contents into a new file located in a
+    working directory. It will then perform statistical measurements
+    on these files in order to monitor the grating wheel telemetry
+    over time. Results are all saved to database tables.
+
+    Attributes
+    ----------
+    output_dir : str
+        Path into which outputs will be placed.
+
+    data_dir : str
+        Path into which new dark files will be copied to be worked on.
+
+    query_start : float
+        MJD start date to use for querying MAST.
+
+    query_end : float
+        MJD end date to use for querying MAST.
+
+    instrument : str
+        Name of instrument used to collect the data.
+
+    aperture : str
+        Name of the aperture used (e.g. ``NRS1``).
+    """
+
+    def __init__(self):
+        """Initialize an instance of the ``Grating`` class."""
+
+    def collapse_image(self, image):
+        """Median-collapse the rows and columns of an image.
+
+        Parameters
+        ----------
+        image : numpy.ndarray
+            2D array on which to calculate statistics.
+
+        Returns
+        -------
+        collapsed_rows : numpy.ndarray
+            1D array of the collapsed row values.
+
+        collapsed_columns : numpy.ndarray
+            1D array of the collapsed column values.
+        """
+
+        collapsed_rows = np.nanmedian(image, axis=1)
+        collapsed_columns = np.nanmedian(image, axis=0)
+
+        return collapsed_rows, collapsed_columns
+
+    def determine_pipeline_steps(self):
+        """Determines the necessary JWST pipelines steps to run on a
+        given dark file.
+
+        Returns
+        -------
+        pipeline_steps : collections.OrderedDict
+            The pipeline steps to run.
+        """
+
+        pipeline_steps = OrderedDict({})
+
+        # Determine if the file needs group_scale step run
+        if self.read_pattern not in pipeline_tools.GROUPSCALE_READOUT_PATTERNS:
+            pipeline_steps['group_scale'] = False
+        else:
+            pipeline_steps['group_scale'] = True
+
+        # Run the DQ step on all files
+        pipeline_steps['dq_init'] = True
+
+        # Only run the superbias step for NIR instruments
+        if self.instrument != 'miri':
+            pipeline_steps['superbias'] = True
+        else:
+            pipeline_steps['superbias'] = False
+
+        # Run the refpix step on all files
+        pipeline_steps['refpix'] = True
+
+        return pipeline_steps
+
+    def extract_zeroth_group(self, filename):
+        """Extracts the 0th group of a fits image and outputs it into
+        a new fits file.
+
+        Parameters
+        ----------
+        filename : str
+            The fits file from which the 0th group will be extracted.
+
+        Returns
+        -------
+        output_filename : str
+            The full path to the output file.
+        """
+
+        output_filename = os.path.join(self.data_dir, os.path.basename(filename).replace('.fits', '_0thgroup.fits'))
+
+        # Write a new fits file containing the primary and science
+        # headers from the input file, as well as the 0th group
+        # data of the first integration
+        hdu = fits.open(filename)
+        new_hdu = fits.HDUList([hdu['PRIMARY'], hdu['SCI']])
+        new_hdu['SCI'].data = hdu['SCI'].data[0:1, 0:1, :, :]
+        new_hdu.writeto(output_filename, overwrite=True)
+        hdu.close()
+        new_hdu.close()
+        set_permissions(output_filename)
+        logging.info('\t{} created'.format(output_filename))
+
+        return output_filename
+
+    def file_exists_in_database(self, filename):
+        """Checks if an entry for filename exists in the grating stats
+        database.
+
+        Parameters
+        ----------
+        filename : str
+            The full path to the uncal filename.
+
+        Returns
+        -------
+        file_exists : bool
+            ``True`` if filename exists in the grating stats database.
+        """
+
+        query = session.query(self.stats_table)
+        results = query.filter(self.stats_table.uncal_filename == filename).all()
+
+        if len(results) != 0:
+            file_exists = True
+        else:
+            file_exists = False
+
+        return file_exists
+
+    def identify_tables(self):
+        """Determine which database tables to use for a run of the grating wheel
+        monitor.
+        """
+
+        mixed_case_name = JWST_INSTRUMENT_NAMES_MIXEDCASE[self.instrument]
+        self.query_table = eval('{}GratingQueryHistory'.format(mixed_case_name))
+        self.stats_table = eval('{}GratingStats'.format(mixed_case_name))
+
+    def image_to_png(self, image, outname):
+        """Ouputs an image array into a png file.
+
+        Parameters
+        ----------
+        image : numpy.ndarray
+            2D image array.
+
+        outname : str
+            The name given to the output png file.
+
+        Returns
+        -------
+        output_filename : str
+            The full path to the output png file.
+        """
+
+        output_filename = os.path.join(self.data_dir, '{}.png'.format(outname))
+
+        # Get image scale limits
+        z = ZScaleInterval()
+        vmin, vmax = z.get_limits(image)
+
+        # Plot the image
+        plt.figure(figsize=(12, 12))
+        ax = plt.gca()
+        im = ax.imshow(image, cmap='gray', origin='lower', vmin=vmin, vmax=vmax)
+        ax.set_title(outname.split('_uncal')[0])
+
+        # Make the colorbar
+        divider = make_axes_locatable(ax)
+        cax = divider.append_axes("right", size="5%", pad=0.4)
+        cbar = plt.colorbar(im, cax=cax)
+        cbar.set_label('Signal [DN]')
+
+        # Save the image
+        plt.savefig(output_filename, bbox_inches='tight', dpi=200)
+        set_permissions(output_filename)
+        logging.info('\t{} created'.format(output_filename))
+
+        return output_filename
+
+    def make_histogram(self, data):
+        """Creates a histogram of the input data and returns the bin
+        centers and the counts in each bin.
+
+        Parameters
+        ----------
+        data : numpy.ndarray
+            The input data.
+
+        Returns
+        -------
+        counts : numpy.ndarray
+            The counts in each histogram bin.
+
+        bin_centers : numpy.ndarray
+            The histogram bin centers.
+        """
+
+        # Calculate the histogram range as that within 5 sigma from the mean
+        data = data.flatten()
+        clipped = sigma_clip(data, sigma=3.0, maxiters=5)
+        mean, stddev = np.nanmean(clipped), np.nanstd(clipped)
+        lower_thresh, upper_thresh = mean - 4 * stddev, mean + 4 * stddev
+
+        # Make the histogram
+        counts, bin_edges = np.histogram(data, bins='auto', range=(lower_thresh, upper_thresh))
+        bin_centers = (bin_edges[:-1] + bin_edges[1:]) / 2
+
+        return counts, bin_centers
+
+    def most_recent_search(self):
+        """Query the query history database and return the information
+        on the most recent query for the given ``aperture_name`` where
+        the grating wheel monitor was executed.
+
+        Returns
+        -------
+        query_result : float
+            Date (in MJD) of the ending range of the previous MAST query
+            where the grating wheel monitor was run.
+        """
+
+        query = session.query(self.query_table).filter(and_(self.query_table.aperture == self.aperture,
+            self.query_table.run_monitor == True)).order_by(self.query_table.end_time_mjd).all()
+
+        if len(query) == 0:
+            query_result = 57357.0  # a.k.a. Dec 1, 2015 == CV3
+            logging.info(('\tNo query history for {}. Beginning search date will be set to {}.'.format(self.aperture, query_result)))
+        else:
+            query_result = query[-1].end_time_mjd
+
+        return query_result
+
+    def process(self, file_list):
+        """The main method for processing darks.  See module docstrings
+        for further details.
+
+        Parameters
+        ----------
+        file_list : list
+            List of filenames (including full paths) to the dark current
+            files.
+        """
+
+        for filename in file_list:
+            logging.info('\tWorking on file: {}'.format(filename))
+
+            # Get relevant header info for this file
+            self.read_pattern = fits.getheader(filename, 0)['READPATT']
+            self.expstart = '{}T{}'.format(fits.getheader(filename, 0)['DATE-OBS'], fits.getheader(filename, 0)['TIME-OBS'])
+
+            # Run the file through the necessary pipeline steps
+            pipeline_steps = self.determine_pipeline_steps()
+            logging.info('\tRunning pipeline on {}'.format(filename))
+            try:
+                processed_file = pipeline_tools.run_calwebb_detector1_steps(filename, pipeline_steps)
+                logging.info('\tPipeline complete. Output: {}'.format(processed_file))
+                set_permissions(processed_file)
+            except:
+                logging.info('\tPipeline processing failed for {}'.format(filename))
+                continue
+
+            # Find amplifier boundaries so per-amp statistics can be calculated
+            _, amp_bounds = instrument_properties.amplifier_info(processed_file, omit_reference_pixels=True)
+            logging.info('\tAmplifier boundaries: {}'.format(amp_bounds))
+
+            # # Get the uncalibrated 0th group data for this file
+            # uncal_data = fits.getdata(filename, 'SCI')[0, 0, :, :].astype(float)
+            # DETERMINE TELEMETRY VALUES AND INCLUDE THEM IN THE GRATING_DB_ENTRY
+
+            # Calculate image statistics on the calibrated image
+            cal_data = fits.getdata(processed_file, 'SCI')[0, 0, :, :]
+            mean, median, stddev = sigma_clipped_stats(cal_data, sigma=3.0, maxiters=5)
+            collapsed_rows, collapsed_columns = self.collapse_image(cal_data)
+            counts, bin_centers = self.make_histogram(cal_data)
+            logging.info('\tCalculated calibrated image stats: {:.3f} +/- {:.3f}'.format(mean, stddev))
+
+            # Save a png of the calibrated image for visual inspection
+            logging.info('\tCreating png of calibrated image')
+            output_png = self.image_to_png(cal_data, outname=os.path.basename(processed_file).replace('.fits', ''))
+
+            # Construct new entry for this file for the grating wheel database table.
+            # Can't insert values with numpy.float32 datatypes into database
+            # so need to change the datatypes of these values.
+            grating_db_entry = {'aperture': self.aperture,
+                                'uncal_filename': filename,
+                                'cal_filename': processed_file,
+                                'cal_image': output_png,
+                                'expstart': self.expstart,
+                                'mean': float(mean),
+                                'median': float(median),
+                                'stddev': float(stddev),
+                                'collapsed_rows': collapsed_rows.astype(float),
+                                'collapsed_columns': collapsed_columns.astype(float),
+                                'counts': counts.astype(float),
+                                'bin_centers': bin_centers.astype(float),
+                                'entry_date': datetime.datetime.now()
+                                }
+
+            # Add this new entry to the grating database table
+            self.stats_table.__table__.insert().execute(grating_db_entry)
+            logging.info('\tNew entry added to grating database table: {}'.format(grating_db_entry))
+
+    @log_fail
+    @log_info
+    def run(self):
+        """The main method.  See module docstrings for further details."""
+
+        logging.info('Begin logging for grating_monitor')
+
+        # Get the output directory and setup a directory to store the data
+        self.output_dir = os.path.join(get_config()['outputs'], 'grating_monitor')
+        ensure_dir_exists(os.path.join(self.output_dir, 'data'))
+
+        # Use the current time as the end time for MAST query
+        self.query_end = Time.now().mjd
+
+        # Loop over all instruments
+        for instrument in ['nirspec']:
+            self.instrument = instrument
+
+            # Identify which database tables to use
+            self.identify_tables()
+
+            # Get a list of all possible full-frame apertures for this instrument
+            siaf = Siaf(self.instrument)
+            possible_apertures = [aperture for aperture in siaf.apertures if siaf[aperture].AperType == 'FULLSCA']
+
+            for aperture in possible_apertures:
+
+                logging.info('Working on aperture {} in {}'.format(aperture, instrument))
+                self.aperture = aperture
+
+                # Locate the record of most recent MAST search; use this time
+                # (plus a 30 day buffer to catch any missing files from
+                # previous run) as the start time in the new MAST search.
+                most_recent_search = self.most_recent_search()
+                self.query_start = most_recent_search - 30
+
+                # Query MAST for new dark files for this instrument/aperture
+                logging.info('\tQuery times: {} {}'.format(self.query_start, self.query_end))
+                new_entries = mast_query_darks(instrument, aperture, self.query_start, self.query_end)
+                logging.info('\tAperture: {}, new entries: {}'.format(self.aperture, len(new_entries)))
+
+                # Set up a directory to store the data for this aperture
+                self.data_dir = os.path.join(self.output_dir, 'data/{}_{}'.format(self.instrument.lower(), self.aperture.lower()))
+                if len(new_entries) > 0:
+                    ensure_dir_exists(self.data_dir)
+
+                # Get any new files to process
+                new_files = []
+                for file_entry in new_entries:
+                    output_filename = os.path.join(self.data_dir, file_entry['filename'])
+                    output_filename = output_filename.replace('_uncal.fits', '_uncal_0thgroup.fits').replace('_dark.fits', '_uncal_0thgroup.fits')
+
+                    # Dont process files that already exist in the grating stats database
+                    file_exists = self.file_exists_in_database(output_filename)
+                    if file_exists:
+                        logging.info('\t{} already exists in the grating database table.'.format(output_filename))
+                        continue
+
+                    # Save the 0th group image from each new file in the output directory; some dont exist in JWQL filesystem.
+                    try:
+                        filename = filesystem_path(file_entry['filename'])
+                        uncal_filename = filename.replace('_dark', '_uncal')
+                        if not os.path.isfile(uncal_filename):
+                            logging.info('\t{} does not exist in JWQL filesystem, even though {} does'.format(uncal_filename, filename))
+                        else:
+                            new_file = self.extract_zeroth_group(uncal_filename)
+                            new_files.append(new_file)
+                    except FileNotFoundError:
+                        logging.info('\t{} does not exist in JWQL filesystem'.format(file_entry['filename']))
+
+                # Run the grating monitor on any new files
+                if len(new_files) > 0:
+                    self.process(new_files)
+                    monitor_run = True
+                else:
+                    logging.info('\tGrating monitor skipped. {} new dark files for {}, {}.'.format(len(new_files), instrument, aperture))
+                    monitor_run = False
+
+                # Update the query history
+                new_entry = {'instrument': instrument,
+                             'aperture': aperture,
+                             'start_time_mjd': self.query_start,
+                             'end_time_mjd': self.query_end,
+                             'entries_found': len(new_entries),
+                             'files_found': len(new_files),
+                             'run_monitor': monitor_run,
+                             'entry_date': datetime.datetime.now()}
+                self.query_table.__table__.insert().execute(new_entry)
+                logging.info('\tUpdated the query history table')
+
+        logging.info('Grating Monitor completed successfully.')
+
+
+if __name__ == '__main__':
+
+    module = os.path.basename(__file__).strip('.py')
+    start_time, log_file = initialize_instrument_monitor(module)
+
+    monitor = Grating()
+    monitor.run()
+
+    update_monitor_table(module, start_time, log_file)
diff --git a/jwql/tests/test_grating_monitor.py b/jwql/tests/test_grating_monitor.py
new file mode 100644
index 000000000..8a3a94591
--- /dev/null
+++ b/jwql/tests/test_grating_monitor.py
@@ -0,0 +1,50 @@
+#! /usr/bin/env python
+
+"""Tests for the ``grating_monitor`` module.
+
+Authors
+-------
+
+    - Teagan King
+
+Use
+---
+
+    These tests can be run via the command line (omit the ``-s`` to
+    suppress verbose output to stdout):
+    ::
+
+        pytest -s test_grating_monitor.py
+"""
+
+import os
+import pytest
+import shutil
+
+from astropy.io import fits
+import numpy as np
+
+from jwql.database.database_interface import NIRSpecGratingQueryHistory, NIRSpecGratingStats
+from jwql.instrument_monitors.common_monitors import grating_monitor
+from jwql.utils.utils import get_config
+
+ON_GITHUB_ACTIONS = '/home/runner' in os.path.expanduser('~') or '/Users/runner' in os.path.expanduser('~')
+
+
+# @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.')
+# def test_query_mast():
+#     """Test the ``query_mast`` function
+#     NEED TO UPDATE
+#     """
+
+#     gw = grating_monitor()
+#     _, _, _, aperture = define_test_data(1)
+
+#     gw.aperture = aperture
+#     gw.instrument = 'nirspec'
+#     gw.query_start = 57357.0
+#     gw.query_end = 57405.0
+
+#     result = gw.query_mast()
+
+#     assert len(result) == 5
diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py
index 3523552d6..98721c732 100644
--- a/jwql/utils/constants.py
+++ b/jwql/utils/constants.py
@@ -208,13 +208,16 @@
                                       'G235H', 'G395H', 'PRISM']
                           }
 
+GRATING_TELEMETRY = ['INRSH_GWA_ADCMGAIN', 'INRSH_GWA_ADCMOFFSET', 'INRSH_GWA_MOTOR_VREF', 'INRSI_C_GWA_X_POSITION', 'INRSI_C_GWA_Y_POSITION']
+
 # Possible suffix types for guider exposures
 GUIDER_SUFFIX_TYPES = ['stream', 'stacked_uncal', 'image_uncal', 'stacked_cal', 'image_cal']
 
 # Instrument monitor database tables
 INSTRUMENT_MONITOR_DATABASE_TABLES = {
     'dark_monitor': ['<instrument>_dark_dark_current', '<instrument>_dark_pixel_stats', '<instrument>_dark_query_history'],
-    'bad_pixel_monitor': ['<instrument>_bad_pixel_stats', '<instrument>_bad_pixel_query_history']}
+    'bad_pixel_monitor': ['<instrument>_bad_pixel_stats', '<instrument>_bad_pixel_query_history'],
+    'grating_monitor': ['<instrument>_grating_stats', '<instrument>_grating_query_history']}
 
 INSTRUMENT_SERVICE_MATCH = {
     'FGS': 'Mast.Jwst.Filtered.Fgs',
@@ -280,6 +283,7 @@
     'nirspec': [('Optical Short Monitor', '#'),
                 ('Bad Pixel Monitor', '/nirspec/bad_pixel_monitor'),
                 ('Readnoise Monitor', '/nirspec/readnoise_monitor'),
+                ('Grating Wheel Monitor', '/nirspec/grating_monitor'),
                 ('Target Acquisition Monitor', '#'),
                 ('Data Trending', '#'),
                 ('Detector Health Monitor', '#'),
diff --git a/jwql/website/apps/jwql/bokeh_containers.py b/jwql/website/apps/jwql/bokeh_containers.py
index b375657d9..c5809d304 100644
--- a/jwql/website/apps/jwql/bokeh_containers.py
+++ b/jwql/website/apps/jwql/bokeh_containers.py
@@ -26,7 +26,7 @@
 from bokeh.models.widgets import Tabs, Panel
 
 from . import monitor_pages
-from jwql.utils.constants import BAD_PIXEL_TYPES, FULL_FRAME_APERTURES
+from jwql.utils.constants import BAD_PIXEL_TYPES, FULL_FRAME_APERTURES, GRATING_TELEMETRY
 from jwql.utils.utils import get_config
 
 __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
@@ -292,6 +292,50 @@ def dark_monitor_tabs(instrument):
     return div, script
 
 
+def grating_monitor_tabs(instrument):
+    """Creates the various tabs of the grating wheel monitor results page.
+
+    Parameters
+    ----------
+    instrument : str
+        The JWST instrument of interest (e.g. ``nircam``).
+
+    Returns
+    -------
+    div : str
+        The HTML div to render grating wheel monitor plots
+    script : str
+        The JS script to render grating wheel monitor plots
+    """
+
+    # Make a separate tab for each aperture
+    tabs = []
+    for aperture in FULL_FRAME_APERTURES[instrument.upper()]:
+        monitor_template = monitor_pages.GratingMonitor()
+        monitor_template.input_parameters = (instrument, aperture)
+
+        # Add the telemetry vs time plots for grating wheel sensor telemetry value
+        plots = []
+        for telemetry in GRATING_TELEMETRY:
+            grating_plot = monitor_template.refs['figure_{}'.format(telemetry)]
+            grating_plot.sizing_mode = 'scale_width'  # Make sure the sizing is adjustable
+            plots.append(grating_plot)
+
+        # Put grating plots on the top row.
+        grating_layout = layout(plots[0:4])  # MAKE SURE ALL PLOTS ARE INCLUDED HERE
+        grating_layout.sizing_mode = 'scale_width'
+        grating_tab = Panel(child=grating_layout, title=aperture)
+        tabs.append(grating_tab)
+
+    # Build tabs
+    tabs = Tabs(tabs=tabs)
+
+    # Return tab HTML and JavaScript to web app
+    script, div = components(tabs)
+
+    return div, script
+
+
 def readnoise_monitor_tabs(instrument):
     """Creates the various tabs of the readnoise monitor results page.
 
diff --git a/jwql/website/apps/jwql/monitor_pages/__init__.py b/jwql/website/apps/jwql/monitor_pages/__init__.py
index ed0aaf2fd..32ff53a3c 100644
--- a/jwql/website/apps/jwql/monitor_pages/__init__.py
+++ b/jwql/website/apps/jwql/monitor_pages/__init__.py
@@ -1,6 +1,7 @@
 from .monitor_bad_pixel_bokeh import BadPixelMonitor
 from .monitor_bias_bokeh import BiasMonitor
 from .monitor_dark_bokeh import DarkMonitor
+from .monitor_grating_bokeh import GratingMonitor
 from .monitor_filesystem_bokeh import MonitorFilesystem
 from .monitor_mast_bokeh import MastMonitor
 from .monitor_readnoise_bokeh import ReadnoiseMonitor
diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_grating_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_grating_bokeh.py
new file mode 100644
index 000000000..904d55706
--- /dev/null
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_grating_bokeh.py
@@ -0,0 +1,117 @@
+
+"""This module contains code for the grating wheel monitor Bokeh plots.
+This code was adapted from monitor_bias_bokeh.py.
+
+Author
+------
+
+    - Teagan King
+
+Use
+---
+
+    This module can be used from the command line as such:
+
+    ::
+
+        from jwql.website.apps.jwql import monitor_pages
+        monitor_template = monitor_pages.GratingMonitor()
+"""
+
+from datetime import datetime, timedelta
+import os
+
+from astropy.stats import sigma_clip
+import numpy as np
+
+from jwql.bokeh_templating import BokehTemplate
+from jwql.database.database_interface import session, NIRSpecGratingStats
+from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE
+from jwql.utils.constants import GRATING_TELEMETRY
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+
+class GratingMonitor(BokehTemplate):
+
+    # Combine the input parameters into a single property because we
+    # do not want to invoke the setter unless all are updated
+    @property
+    def input_parameters(self):
+        return (self._instrument, self._aperture)
+
+    @input_parameters.setter
+    def input_parameters(self, info):
+        self._instrument, self._aperture = info
+        self.pre_init()
+        self.post_init()
+
+    def identify_tables(self):
+        """Determine which database tables to use for the given instrument"""
+
+        mixed_case_name = JWST_INSTRUMENT_NAMES_MIXEDCASE[self._instrument.lower()]
+        self.stats_table = eval('{}GratingStats'.format(mixed_case_name))
+
+    def load_data(self):
+        """Query the database tables to get all of the relevant grating wheel data"""
+
+        # Determine which database tables are needed based on instrument
+        self.identify_tables()
+
+        # Query database for all data in grating wheel stats with a matching aperture,
+        # and sort the data by exposure start time.
+        self.query_results = session.query(self.stats_table) \
+            .filter(self.stats_table.aperture == self._aperture) \
+            .order_by(self.stats_table.expstart) \
+            .all()
+
+    def pre_init(self):
+
+        # Start with default values for instrument and aperture because
+        # BokehTemplate's __init__ method does not allow input arguments
+        try:
+            dummy_instrument = self._instrument
+            dummy_aperture = self._aperture
+        except AttributeError:
+            self._instrument = 'NIRSpec'
+            self._aperture = ''
+
+        self._embed = True
+        self.format_string = None
+        self.interface_file = os.path.join(SCRIPT_DIR, 'yaml', 'monitor_grating_interface.yaml')
+
+    def post_init(self):
+
+        # Load the grating wheel data
+        self.load_data()
+
+        # Update the mean grating wheel over time figures
+        self.update_mean_grating_figures()
+
+    def update_mean_grating_figures(self):
+        """Updates the mean bias over time bokeh plots"""
+
+        # POSSIBLY REPLACE DARK EXPOSURES WITH SOMETHING ELSE
+        # Get the dark exposures and their starts times
+        filenames = [os.path.basename(result.uncal_filename).replace('_uncal.fits', '') for result in self.query_results]
+        expstarts_iso = np.array([result.expstart for result in self.query_results])
+        expstarts = np.array([datetime.strptime(date, '%Y-%m-%dT%H:%M:%S.%f') for date in expstarts_iso])
+
+        # Update the figures for all GWA telemetry values
+        for telemetry in GRATING_TELEMETRY:
+            gwa_vals = np.array([getattr(result, '{}'.format(telemetry)) for result in self.query_results])
+            self.refs['source_{}'.format(telemetry)].data = {'time': expstarts,
+                                                             'time_iso': expstarts_iso,
+                                                             'gwa_vals': gwa_vals,
+                                                             'filename': filenames}
+            self.refs['figure_{}'.format(telemetry)].title.text = '{}'.format(telemetry)
+            self.refs['figure_{}'.format(telemetry)].hover.tooltips = [('file', '@filename'),
+                                                                       ('time', '@time_iso'),
+                                                                       ('gwa val', '@gwa_vals')]
+
+            # Update plot limits if data exists
+            if len(gwa_vals) != 0:
+                self.refs['gwa_x_{}'.format(telemetry)].start = expstarts.min() - timedelta(days=3)
+                self.refs['gwa_x_{}'.format(telemetry)].end = expstarts.max() + timedelta(days=3)
+                self.refs['gwa_y_{}'.format(telemetry)].start = gwa_vals.min() - 20
+                self.refs['gwa_y_{}'.format(telemetry)].end = gwa_vals.max() + 20
diff --git a/jwql/website/apps/jwql/monitor_pages/yaml/monitor_grating_interface.yaml b/jwql/website/apps/jwql/monitor_pages/yaml/monitor_grating_interface.yaml
new file mode 100644
index 000000000..508ee0e26
--- /dev/null
+++ b/jwql/website/apps/jwql/monitor_pages/yaml/monitor_grating_interface.yaml
@@ -0,0 +1,170 @@
+# YAML file defining bokeh figures for the grating wheel monitor
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# INRSH_GWA_ADCMGAIN vs Time Figure
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+- !ColumnDataSource: &source_INRSH_GWA_ADCMGAIN
+    ref: "source_INRSH_GWA_ADCMGAIN"
+    data:
+        time: []
+        time_iso: []
+        grating: []
+        filename: []
+- !Range1d: &gwa_x_INRSH_GWA_ADCMGAIN
+    ref: "gwa_x_INRSH_GWA_ADCMGAIN"
+    start: 0
+    end: 1
+    bounds: 'auto'
+- !Range1d: &gwa_y_INRSH_GWA_ADCMGAIN
+    ref: "gwa_y_INRSH_GWA_ADCMGAIN"
+    start: 0
+    end: 10
+    bounds: 'auto'
+- !Figure: &figure_INRSH_GWA_ADCMGAIN
+    ref: "figure_INRSH_GWA_ADCMGAIN"
+    x_axis_label: "Date"
+    x_axis_type: "datetime"
+    y_axis_label: "INRSH_GWA_ADCMGAIN [DN]"
+    x_range: *gwa_x_INRSH_GWA_ADCMGAIN
+    y_range: *gwa_y_INRSH_GWA_ADCMGAIN
+    height: 800
+    width: 800
+    tools: "hover, wheel_zoom, pan, reset"
+    elements:
+        - {'kind': 'circle', 'x': 'time', 'y': 'grating', 'size': 6, 'source': *source_INRSH_GWA_ADCMGAIN}
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# INRSH_GWA_ADCMOFFSET vs Time Figure
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+- !ColumnDataSource: &source_INRSH_GWA_ADCMOFFSET
+    ref: "source_INRSH_GWA_ADCMOFFSET"
+    data:
+        time: []
+        time_iso: []
+        grating: []
+        filename: []
+- !Range1d: &gwa_x_INRSH_GWA_ADCMOFFSET
+    ref: "gwa_x_INRSH_GWA_ADCMOFFSET"
+    start: 0
+    end: 1
+    bounds: 'auto'
+- !Range1d: &gwa_y_INRSH_GWA_ADCMOFFSET
+    ref: "gwa_y_INRSH_GWA_ADCMOFFSET"
+    start: 0
+    end: 10
+    bounds: 'auto'
+- !Figure: &figure_INRSH_GWA_ADCMOFFSET
+    ref: "figure_INRSH_GWA_ADCMOFFSET"
+    x_axis_label: "Date"
+    x_axis_type: "datetime"
+    y_axis_label: "INRSH_GWA_ADCMOFFSET [DN]"
+    x_range: *gwa_x_INRSH_GWA_ADCMOFFSET
+    y_range: *gwa_y_INRSH_GWA_ADCMOFFSET
+    height: 800
+    width: 800
+    tools: "hover, wheel_zoom, pan, reset"
+    elements:
+        - {'kind': 'circle', 'x': 'time', 'y': 'grating', 'size': 6, 'source': *source_INRSH_GWA_ADCMOFFSET}
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# INRSH_GWA_MOTOR_VREF vs Time Figure
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+- !ColumnDataSource: &source_INRSH_GWA_MOTOR_VREF
+    ref: "source_INRSH_GWA_MOTOR_VREF"
+    data:
+        time: []
+        time_iso: []
+        grating: []
+        filename: []
+- !Range1d: &gwa_x_INRSH_GWA_MOTOR_VREF
+    ref: "gwa_x_INRSH_GWA_MOTOR_VREF"
+    start: 0
+    end: 1
+    bounds: 'auto'
+- !Range1d: &gwa_y_INRSH_GWA_MOTOR_VREF
+    ref: "gwa_y_INRSH_GWA_MOTOR_VREF"
+    start: 0
+    end: 10
+    bounds: 'auto'
+- !Figure: &figure_INRSH_GWA_MOTOR_VREF
+    ref: "figure_INRSH_GWA_MOTOR_VREF"
+    x_axis_label: "Date"
+    x_axis_type: "datetime"
+    y_axis_label: "INRSH_GWA_MOTOR_VREF [DN]"
+    x_range: *gwa_x_INRSH_GWA_MOTOR_VREF
+    y_range: *gwa_y_INRSH_GWA_MOTOR_VREF
+    height: 800
+    width: 800
+    tools: "hover, wheel_zoom, pan, reset"
+    elements:
+        - {'kind': 'circle', 'x': 'time', 'y': 'grating', 'size': 6, 'source': *source_INRSH_GWA_MOTOR_VREF}
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# INRSI_C_GWA_X_POSITION vs Time Figure
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+- !ColumnDataSource: &source_INRSI_C_GWA_X_POSITION
+    ref: "source_INRSI_C_GWA_X_POSITION"
+    data:
+        time: []
+        time_iso: []
+        grating: []
+        filename: []
+- !Range1d: &gwa_x_INRSI_C_GWA_X_POSITION
+    ref: "gwa_x_INRSI_C_GWA_X_POSITION"
+    start: 0
+    end: 1
+    bounds: 'auto'
+- !Range1d: &gwa_y_INRSI_C_GWA_X_POSITION
+    ref: "gwa_y_INRSI_C_GWA_X_POSITION"
+    start: 0
+    end: 10
+    bounds: 'auto'
+- !Figure: &figure_INRSI_C_GWA_X_POSITION
+    ref: "figure_INRSI_C_GWA_X_POSITION"
+    x_axis_label: "Date"
+    x_axis_type: "datetime"
+    y_axis_label: "INRSI_C_GWA_X_POSITION [DN]"
+    x_range: *gwa_x_INRSI_C_GWA_X_POSITION
+    y_range: *gwa_y_INRSI_C_GWA_X_POSITION
+    height: 800
+    width: 800
+    tools: "hover, wheel_zoom, pan, reset"
+    elements:
+        - {'kind': 'circle', 'x': 'time', 'y': 'grating', 'size': 6, 'source': *source_INRSI_C_GWA_X_POSITION}
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# INRSI_C_GWA_Y_POSITION vs Time Figure
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+- !ColumnDataSource: &source_INRSI_C_GWA_Y_POSITION
+    ref: "source_INRSI_C_GWA_Y_POSITION"
+    data:
+        time: []
+        time_iso: []
+        grating: []
+        filename: []
+- !Range1d: &gwa_x_INRSI_C_GWA_Y_POSITION
+    ref: "gwa_x_INRSI_C_GWA_Y_POSITION"
+    start: 0
+    end: 1
+    bounds: 'auto'
+- !Range1d: &gwa_y_INRSI_C_GWA_Y_POSITION
+    ref: "gwa_y_INRSI_C_GWA_Y_POSITION"
+    start: 0
+    end: 10
+    bounds: 'auto'
+- !Figure: &figure_INRSI_C_GWA_Y_POSITION
+    ref: "figure_INRSI_C_GWA_Y_POSITION"
+    x_axis_label: "Date"
+    x_axis_type: "datetime"
+    y_axis_label: "INRSI_C_GWA_Y_POSITION [DN]"
+    x_range: *gwa_x_INRSI_C_GWA_Y_POSITION
+    y_range: *gwa_y_INRSI_C_GWA_Y_POSITION
+    height: 800
+    width: 800
+    tools: "hover, wheel_zoom, pan, reset"
+    elements:
+        - {'kind': 'circle', 'x': 'time', 'y': 'grating', 'size': 6, 'source': *source_INRSI_C_GWA_Y_POSITION}
+
+# Document structure
+# - !Document:
+#     - !row:
+#         - *grating_figure
diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py
index 61baa02d9..56cd675ba 100644
--- a/jwql/website/apps/jwql/monitor_views.py
+++ b/jwql/website/apps/jwql/monitor_views.py
@@ -139,6 +139,39 @@ def dark_monitor(request, inst):
     return render(request, template, context)
 
 
+def grating_monitor(request, inst):
+    """Generate the grating wheel monitor page for a given instrument
+
+    Parameters
+    ----------
+    request : HttpRequest object
+        Incoming request from the webpage
+    inst : str
+        Name of JWST instrument
+
+    Returns
+    -------
+    HttpResponse object
+        Outgoing response sent to the webpage
+    """
+
+    # Ensure the instrument is correctly capitalized
+    inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()]
+
+    # Get the html and JS needed to render the grating wheel tab plots
+    tabs_components = bokeh_containers.grating_monitor_tabs(inst)
+
+    template = "grating_monitor.html"
+
+    context = {
+        'inst': inst,
+        'tabs_components': tabs_components,
+    }
+
+    # Return a HTTP response with the template and dictionary of variables
+    return render(request, template, context)
+
+
 def readnoise_monitor(request, inst):
     """Generate the readnoise monitor page for a given instrument
 
diff --git a/jwql/website/apps/jwql/templates/grating_monitor.html b/jwql/website/apps/jwql/templates/grating_monitor.html
new file mode 100644
index 000000000..e197dcb9b
--- /dev/null
+++ b/jwql/website/apps/jwql/templates/grating_monitor.html
@@ -0,0 +1,30 @@
+{% extends "base.html" %}
+
+{% block preamble %}
+
+	<title>{{ inst }} Grating Wheel Monitor- JWQL</title>
+
+{% endblock %}
+
+{% block content %}
+
+    <main role="main" class="container">
+
+		<h1>{{ inst }} Grating Wheel Monitor</h1>
+		<hr>
+
+		<div width=100%>
+			<!--Add Bokeh HTML-->
+			{{ tabs_components[0] | safe }}
+		</div>
+
+		<!--Add Bokeh scripts-->
+		{{ tabs_components[1] | safe }}
+
+		<h1>{{ inst }} Grating Wheel Stats Table</h1>
+		<hr>
+
+		<a href="{{'/jwqldb/%s_grating_stats'%inst.lower()}}" name=test_link class="btn btn-primary my-2" type="submit">Grating Wheel Stats</a>
+	</main>
+
+{% endblock %}
diff --git a/jwql/website/apps/jwql/urls.py b/jwql/website/apps/jwql/urls.py
index 845c3a2f1..78d282a85 100644
--- a/jwql/website/apps/jwql/urls.py
+++ b/jwql/website/apps/jwql/urls.py
@@ -66,6 +66,7 @@
     re_path(r'^(?P<inst>({}))/dark_monitor/$'.format(instruments), monitor_views.dark_monitor, name='dark_monitor'),
     re_path(r'^(?P<inst>({}))/bad_pixel_monitor/$'.format(instruments), monitor_views.bad_pixel_monitor, name='bad_pixel_monitor'),
     re_path(r'^(?P<inst>({}))/bias_monitor/$'.format(instruments), monitor_views.bias_monitor, name='bias_monitor'),
+    re_path(r'^(?P<inst>({}))/grating_monitor/$'.format(instruments), monitor_views.grating_monitor, name='grating_monitor'),
     re_path(r'^(?P<inst>({}))/readnoise_monitor/$'.format(instruments), monitor_views.readnoise_monitor, name='readnoise_monitor'),
 
     # Main site views

From 081921dcd2fbd83d2ffb3957d8cb3e65371ad520 Mon Sep 17 00:00:00 2001
From: Teagan King <55458851+tnking97@users.noreply.github.com>
Date: Mon, 18 Oct 2021 13:08:26 -0400
Subject: [PATCH 02/18] implement logging in data_containers.py

---
 jwql/website/apps/jwql/data_containers.py             | 11 ++++++-----
 .../apps/jwql/monitor_pages/monitor_grating_bokeh.py  |  1 +
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py
index bb327c612..def39cfc5 100644
--- a/jwql/website/apps/jwql/data_containers.py
+++ b/jwql/website/apps/jwql/data_containers.py
@@ -24,6 +24,7 @@
 
 import copy
 import glob
+import logging
 import os
 import re
 import tempfile
@@ -458,7 +459,7 @@ def get_expstart(instrument, rootname):
     result = response[0].json()
     if result['data'] == []:
         expstart = 0
-        print("WARNING: no data")
+        logging.warning("No data retreived from MAST")
     else:
         expstart = min([item['expstart'] for item in result['data']])
 
@@ -943,7 +944,7 @@ def get_thumbnails_all_instruments(parameters):
 
     # Determine whether or not queried anomalies are flagged
     final_subset = []
-    
+
     if anomalies != {'miri': [], 'nirspec': [], 'niriss': [], 'nircam': [], 'fgs': []}:
         for thumbnail in thumbnails_subset:
             components = thumbnail.split('_')
@@ -957,7 +958,7 @@ def get_thumbnails_all_instruments(parameters):
                             # thumbnail contains an anomaly selected in the query
                             final_subset.append(thumbnail)
             except KeyError:
-                print("Error with thumbnail: ", thumbnail)
+                logging.warning("In determining anomalies present, encountered error with thumbnail: {}".format(thumbnail))
     else:
         # if no anomalies are flagged, return all thumbnails from query
         final_subset = thumbnails_subset
@@ -1160,7 +1161,7 @@ def thumbnails_ajax(inst, proposal=None):
             data_dict['file_data'][rootname]['expstart'] = get_expstart(inst, rootname)
             data_dict['file_data'][rootname]['expstart_iso'] = Time(data_dict['file_data'][rootname]['expstart'], format='mjd').iso.split('.')[0]
         except:
-            print("issue with get_expstart for {}".format(rootname))
+            logging.warning("Issue with get_expstart for {}".format(rootname))
 
     # Extract information for sorting with dropdown menus
     # (Don't include the proposal as a sorting parameter if the proposal has already been specified)
@@ -1236,7 +1237,7 @@ def thumbnails_query_ajax(rootnames):
             data_dict['file_data'][rootname]['inst'] = JWST_INSTRUMENT_NAMES_MIXEDCASE[filename_parser(rootname)['instrument']]
         except KeyError:
             data_dict['file_data'][rootname]['inst'] = "MIRI"
-            print("Warning: assuming instrument is MIRI")
+            logging.warning("Assuming instrument is MIRI because filename parser failed.")
         data_dict['file_data'][rootname]['filename_dict'] = filename_dict
         data_dict['file_data'][rootname]['available_files'] = available_files
         data_dict['file_data'][rootname]['expstart'] = get_expstart(data_dict['file_data'][rootname]['inst'], rootname)
diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_grating_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_grating_bokeh.py
index 904d55706..92544e27e 100644
--- a/jwql/website/apps/jwql/monitor_pages/monitor_grating_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_grating_bokeh.py
@@ -60,6 +60,7 @@ def load_data(self):
 
         # Query database for all data in grating wheel stats with a matching aperture,
         # and sort the data by exposure start time.
+        # FAILING HERE: (psycopg2.errors.UndefinedTable) relation "nirspec_grating_stats" does not exist
         self.query_results = session.query(self.stats_table) \
             .filter(self.stats_table.aperture == self._aperture) \
             .order_by(self.stats_table.expstart) \

From 2d36ed9c01c6544be481e0f32c762b1a653b2547 Mon Sep 17 00:00:00 2001
From: Teagan King <55458851+tnking97@users.noreply.github.com>
Date: Mon, 18 Oct 2021 16:27:50 -0400
Subject: [PATCH 03/18] match monitor table definitions with grating_db_entry

---
 .../nirspec/nirspec_grating_query_history.txt |  2 +-
 .../nirspec/nirspec_grating_stats.txt         |  3 --
 .../common_monitors/grating_monitor.py        | 30 ++++++++++++-------
 3 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_query_history.txt b/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_query_history.txt
index c6deea152..3a5ff5add 100644
--- a/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_query_history.txt
+++ b/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_query_history.txt
@@ -5,4 +5,4 @@ END_TIME_MJD, float
 ENTRIES_FOUND, integer
 FILES_FOUND, integer
 RUN_MONITOR, bool
-ENTRY_DATE, datetime
\ No newline at end of file
+ENTRY_DATE, datetime
diff --git a/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_stats.txt b/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_stats.txt
index 89a359b72..d8145783d 100644
--- a/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_stats.txt
+++ b/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_stats.txt
@@ -1,8 +1,5 @@
 APERTURE, string
-DETECTOR, string
 READ_PATTERN, string
-NINTS, string
-NGROUPS, string
 EXPSTART, string
 INRSH_GWA_ADCMGAIN, float
 INRSH_GWA_ADCMOFFSET, float
diff --git a/jwql/instrument_monitors/common_monitors/grating_monitor.py b/jwql/instrument_monitors/common_monitors/grating_monitor.py
index 6ffab5038..2ca65386c 100755
--- a/jwql/instrument_monitors/common_monitors/grating_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/grating_monitor.py
@@ -345,18 +345,26 @@ def process(self, file_list):
             # Can't insert values with numpy.float32 datatypes into database
             # so need to change the datatypes of these values.
             grating_db_entry = {'aperture': self.aperture,
-                                'uncal_filename': filename,
-                                'cal_filename': processed_file,
-                                'cal_image': output_png,
+                                'read_pattern': self.read_pattern,
                                 'expstart': self.expstart,
-                                'mean': float(mean),
-                                'median': float(median),
-                                'stddev': float(stddev),
-                                'collapsed_rows': collapsed_rows.astype(float),
-                                'collapsed_columns': collapsed_columns.astype(float),
-                                'counts': counts.astype(float),
-                                'bin_centers': bin_centers.astype(float),
-                                'entry_date': datetime.datetime.now()
+                                'inrsh_gwa_adcmgain': "ADCMGAIN",
+                                'inrsh_gwa_adcmoffset': "ADCMOFFSET",
+                                'inrsh_gwa_motor_vref': "MOTOR_VREF",
+                                'inrsi_c_gwa_x_position': "X",
+                                'inrsi_c_gwa_y_position': "Y"
+
+                                # 'uncal_filename': filename,
+                                # 'cal_filename': processed_file,
+                                # 'cal_image': output_png,
+                                # 'expstart': self.expstart,
+                                # 'mean': float(mean),
+                                # 'median': float(median),
+                                # 'stddev': float(stddev),
+                                # 'collapsed_rows': collapsed_rows.astype(float),
+                                # 'collapsed_columns': collapsed_columns.astype(float),
+                                # 'counts': counts.astype(float),
+                                # 'bin_centers': bin_centers.astype(float),
+                                # 'entry_date': datetime.datetime.now()
                                 }
 
             # Add this new entry to the grating database table

From 198dabe57bf141d6caea0d4486b47e666a2726b9 Mon Sep 17 00:00:00 2001
From: Teagan King <55458851+tnking97@users.noreply.github.com>
Date: Mon, 18 Oct 2021 16:38:18 -0400
Subject: [PATCH 04/18] use dummy telemetry values to get correct data type

---
 .../nirspec/nirspec_grating_query_history.txt |  6 -----
 .../common_monitors/grating_monitor.py        | 24 +++++--------------
 2 files changed, 6 insertions(+), 24 deletions(-)

diff --git a/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_query_history.txt b/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_query_history.txt
index 3a5ff5add..5fc6b9a49 100644
--- a/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_query_history.txt
+++ b/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_query_history.txt
@@ -1,8 +1,2 @@
-INSTRUMENT, string
 APERTURE, string
-START_TIME_MJD, float
-END_TIME_MJD, float
-ENTRIES_FOUND, integer
-FILES_FOUND, integer
 RUN_MONITOR, bool
-ENTRY_DATE, datetime
diff --git a/jwql/instrument_monitors/common_monitors/grating_monitor.py b/jwql/instrument_monitors/common_monitors/grating_monitor.py
index 2ca65386c..26deee61f 100755
--- a/jwql/instrument_monitors/common_monitors/grating_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/grating_monitor.py
@@ -347,24 +347,12 @@ def process(self, file_list):
             grating_db_entry = {'aperture': self.aperture,
                                 'read_pattern': self.read_pattern,
                                 'expstart': self.expstart,
-                                'inrsh_gwa_adcmgain': "ADCMGAIN",
-                                'inrsh_gwa_adcmoffset': "ADCMOFFSET",
-                                'inrsh_gwa_motor_vref': "MOTOR_VREF",
-                                'inrsi_c_gwa_x_position': "X",
-                                'inrsi_c_gwa_y_position': "Y"
-
-                                # 'uncal_filename': filename,
-                                # 'cal_filename': processed_file,
-                                # 'cal_image': output_png,
-                                # 'expstart': self.expstart,
-                                # 'mean': float(mean),
-                                # 'median': float(median),
-                                # 'stddev': float(stddev),
-                                # 'collapsed_rows': collapsed_rows.astype(float),
-                                # 'collapsed_columns': collapsed_columns.astype(float),
-                                # 'counts': counts.astype(float),
-                                # 'bin_centers': bin_centers.astype(float),
-                                # 'entry_date': datetime.datetime.now()
+                                'inrsh_gwa_adcmgain': 10,
+                                'inrsh_gwa_adcmoffset': 10,
+                                'inrsh_gwa_motor_vref': 10,
+                                'inrsi_c_gwa_x_position': 10,
+                                'inrsi_c_gwa_y_position': 10,
+                                'run_monitor': False
                                 }
 
             # Add this new entry to the grating database table

From b7d498f552353fc2084ffcd1a6faaaea80569539 Mon Sep 17 00:00:00 2001
From: Teagan King <55458851+tnking97@users.noreply.github.com>
Date: Tue, 2 Nov 2021 11:27:19 -0400
Subject: [PATCH 05/18] updates to database entries

---
 .../nirspec/nirspec_grating_query_history.txt |   5 +
 .../nirspec/nirspec_grating_stats.txt         |   1 +
 .../common_monitors/grating_monitor.py        | 301 +++++++-----------
 jwql/utils/constants.py                       |  11 +-
 jwql/website/apps/jwql/bokeh_containers.py    |   5 +-
 jwql/website/apps/jwql/data_containers.py     |   1 -
 .../monitor_pages/monitor_grating_bokeh.py    |  51 +--
 7 files changed, 158 insertions(+), 217 deletions(-)

diff --git a/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_query_history.txt b/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_query_history.txt
index 5fc6b9a49..39e615bfc 100644
--- a/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_query_history.txt
+++ b/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_query_history.txt
@@ -1,2 +1,7 @@
 APERTURE, string
+START_TIME_MJD, float
+END_TIME_MJD, float
+ENTRIES_FOUND, integer
+FILES_FOUND, integer
 RUN_MONITOR, bool
+ENTRY_DATE, datetime
\ No newline at end of file
diff --git a/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_stats.txt b/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_stats.txt
index d8145783d..cdae30ecd 100644
--- a/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_stats.txt
+++ b/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_stats.txt
@@ -1,4 +1,5 @@
 APERTURE, string
+UNCAL_FILENAME, string
 READ_PATTERN, string
 EXPSTART, string
 INRSH_GWA_ADCMGAIN, float
diff --git a/jwql/instrument_monitors/common_monitors/grating_monitor.py b/jwql/instrument_monitors/common_monitors/grating_monitor.py
index 26deee61f..6e2c204ea 100755
--- a/jwql/instrument_monitors/common_monitors/grating_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/grating_monitor.py
@@ -22,9 +22,11 @@
 import os
 
 from astropy.io import fits
-from astropy.stats import sigma_clip, sigma_clipped_stats
+from astropy.stats import sigma_clip
 from astropy.time import Time
 from astropy.visualization import ZScaleInterval
+from jwql.edb.engineering_database import get_mnemonic
+from jwql.edb.engineering_database import get_mnemonics
 import matplotlib
 matplotlib.use('Agg')
 import matplotlib.pyplot as plt
@@ -35,10 +37,8 @@
 
 from jwql.database.database_interface import session
 from jwql.database.database_interface import NIRSpecGratingQueryHistory, NIRSpecGratingStats
-from jwql.instrument_monitors import pipeline_tools
 from jwql.instrument_monitors.common_monitors.dark_monitor import mast_query_darks
-from jwql.utils import instrument_properties
-from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE
+from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE, GRATING_TELEMETRY
 from jwql.utils.logging_functions import log_info, log_fail
 from jwql.utils.monitor_utils import update_monitor_table
 from jwql.utils.permissions import set_permissions
@@ -79,60 +79,6 @@ class Grating():
     def __init__(self):
         """Initialize an instance of the ``Grating`` class."""
 
-    def collapse_image(self, image):
-        """Median-collapse the rows and columns of an image.
-
-        Parameters
-        ----------
-        image : numpy.ndarray
-            2D array on which to calculate statistics.
-
-        Returns
-        -------
-        collapsed_rows : numpy.ndarray
-            1D array of the collapsed row values.
-
-        collapsed_columns : numpy.ndarray
-            1D array of the collapsed column values.
-        """
-
-        collapsed_rows = np.nanmedian(image, axis=1)
-        collapsed_columns = np.nanmedian(image, axis=0)
-
-        return collapsed_rows, collapsed_columns
-
-    def determine_pipeline_steps(self):
-        """Determines the necessary JWST pipelines steps to run on a
-        given dark file.
-
-        Returns
-        -------
-        pipeline_steps : collections.OrderedDict
-            The pipeline steps to run.
-        """
-
-        pipeline_steps = OrderedDict({})
-
-        # Determine if the file needs group_scale step run
-        if self.read_pattern not in pipeline_tools.GROUPSCALE_READOUT_PATTERNS:
-            pipeline_steps['group_scale'] = False
-        else:
-            pipeline_steps['group_scale'] = True
-
-        # Run the DQ step on all files
-        pipeline_steps['dq_init'] = True
-
-        # Only run the superbias step for NIR instruments
-        if self.instrument != 'miri':
-            pipeline_steps['superbias'] = True
-        else:
-            pipeline_steps['superbias'] = False
-
-        # Run the refpix step on all files
-        pipeline_steps['refpix'] = True
-
-        return pipeline_steps
-
     def extract_zeroth_group(self, filename):
         """Extracts the 0th group of a fits image and outputs it into
         a new fits file.
@@ -281,7 +227,6 @@ def most_recent_search(self):
             Date (in MJD) of the ending range of the previous MAST query
             where the grating wheel monitor was run.
         """
-
         query = session.query(self.query_table).filter(and_(self.query_table.aperture == self.aperture,
             self.query_table.run_monitor == True)).order_by(self.query_table.end_time_mjd).all()
 
@@ -303,61 +248,42 @@ def process(self, file_list):
             List of filenames (including full paths) to the dark current
             files.
         """
-
-        for filename in file_list:
-            logging.info('\tWorking on file: {}'.format(filename))
-
-            # Get relevant header info for this file
-            self.read_pattern = fits.getheader(filename, 0)['READPATT']
-            self.expstart = '{}T{}'.format(fits.getheader(filename, 0)['DATE-OBS'], fits.getheader(filename, 0)['TIME-OBS'])
-
-            # Run the file through the necessary pipeline steps
-            pipeline_steps = self.determine_pipeline_steps()
-            logging.info('\tRunning pipeline on {}'.format(filename))
-            try:
-                processed_file = pipeline_tools.run_calwebb_detector1_steps(filename, pipeline_steps)
-                logging.info('\tPipeline complete. Output: {}'.format(processed_file))
-                set_permissions(processed_file)
-            except:
-                logging.info('\tPipeline processing failed for {}'.format(filename))
-                continue
-
-            # Find amplifier boundaries so per-amp statistics can be calculated
-            _, amp_bounds = instrument_properties.amplifier_info(processed_file, omit_reference_pixels=True)
-            logging.info('\tAmplifier boundaries: {}'.format(amp_bounds))
-
-            # # Get the uncalibrated 0th group data for this file
-            # uncal_data = fits.getdata(filename, 'SCI')[0, 0, :, :].astype(float)
-            # DETERMINE TELEMETRY VALUES AND INCLUDE THEM IN THE GRATING_DB_ENTRY
-
-            # Calculate image statistics on the calibrated image
-            cal_data = fits.getdata(processed_file, 'SCI')[0, 0, :, :]
-            mean, median, stddev = sigma_clipped_stats(cal_data, sigma=3.0, maxiters=5)
-            collapsed_rows, collapsed_columns = self.collapse_image(cal_data)
-            counts, bin_centers = self.make_histogram(cal_data)
-            logging.info('\tCalculated calibrated image stats: {:.3f} +/- {:.3f}'.format(mean, stddev))
-
-            # Save a png of the calibrated image for visual inspection
-            logging.info('\tCreating png of calibrated image')
-            output_png = self.image_to_png(cal_data, outname=os.path.basename(processed_file).replace('.fits', ''))
-
-            # Construct new entry for this file for the grating wheel database table.
-            # Can't insert values with numpy.float32 datatypes into database
-            # so need to change the datatypes of these values.
-            grating_db_entry = {'aperture': self.aperture,
-                                'read_pattern': self.read_pattern,
-                                'expstart': self.expstart,
-                                'inrsh_gwa_adcmgain': 10,
-                                'inrsh_gwa_adcmoffset': 10,
-                                'inrsh_gwa_motor_vref': 10,
-                                'inrsi_c_gwa_x_position': 10,
-                                'inrsi_c_gwa_y_position': 10,
-                                'run_monitor': False
-                                }
-
-            # Add this new entry to the grating database table
-            self.stats_table.__table__.insert().execute(grating_db_entry)
-            logging.info('\tNew entry added to grating database table: {}'.format(grating_db_entry))
+        start_telemetry_time = Time(Time.now(), format='decimalyear') - 300  # Use telemetry from previous 30 days
+        end_telemetry_time = Time(Time.now(), format='decimalyear')
+
+        # Construct new entry for this file for the grating wheel database table.
+        for telem in GRATING_TELEMETRY.keys():
+            mnemonic = get_mnemonic(telem, start_telemetry_time, end_telemetry_time)
+            for time in mnemonic.data['MJD']:
+                grating_db_entry = {'aperture': self.aperture,
+                                    'read_pattern': "temp",
+                                    'expstart': time,
+                                    'inrsh_gwa_adcmgain_time': mnemonic.data['euvalue'][np.where(mnemonic.data['MJD'] == time)],
+                                    'run_monitor': False,
+                                    'entry_date': datetime.datetime.now()  # need slightly different times to add to database
+                                    }
+
+                # Add this new entry to the grating database table
+                self.stats_table.__table__.insert().execute(grating_db_entry)
+                logging.info('\tNew entry added to grating database table: {}'.format(grating_db_entry))
+
+        # FAILS BECAUSE LISTS OF FLOATS NOT FLOATS...
+        # mnemonics = get_mnemonics(GRATING_TELEMETRY, start_telemetry_time, end_telemetry_time)
+        # grating_db_entry = {'aperture': self.aperture,
+        #                     'read_pattern': "temp",
+        #                     'expstart': "temp",
+        #                     'inrsh_gwa_adcmgain_time': mnemonics['INRSH_GWA_ADCMGAIN'].data['MJD'],
+        #                     'inrsh_gwa_adcmgain': mnemonics['INRSH_GWA_ADCMGAIN'].data['euvalue'],
+        #                     'inrsh_gwa_adcmoffset_time': mnemonics['INRSH_GWA_ADCMOFFSET'].data['MJD'],
+        #                     'inrsh_gwa_adcmoffset': mnemonics['INRSH_GWA_ADCMOFFSET'].data['euvalue'],
+        #                     'inrsh_gwa_motor_vref_time': mnemonics['INRSH_GWA_MOTOR_VREF'].data['MJD'],
+        #                     'inrsh_gwa_motor_vref': mnemonics['INRSH_GWA_MOTOR_VREF'].data['euvalue'],
+        #                     'inrsi_c_gwa_x_position_time': mnemonics['INRSI_C_GWA_X_POSITION'].data['MJD'],
+        #                     'inrsi_c_gwa_x_position': mnemonics['INRSI_C_GWA_X_POSITION'].data['euvalue'],
+        #                     'inrsi_c_gwa_y_position_time': mnemonics['INRSI_C_GWA_Y_POSITION'].data['MJD'],
+        #                     'inrsi_c_gwa_y_position': mnemonics['INRSI_C_GWA_Y_POSITION'].data['euvalue'],
+        #                     'run_monitor': False
+        #                     }
 
     @log_fail
     @log_info
@@ -373,81 +299,78 @@ def run(self):
         # Use the current time as the end time for MAST query
         self.query_end = Time.now().mjd
 
-        # Loop over all instruments
-        for instrument in ['nirspec']:
-            self.instrument = instrument
-
-            # Identify which database tables to use
-            self.identify_tables()
-
-            # Get a list of all possible full-frame apertures for this instrument
-            siaf = Siaf(self.instrument)
-            possible_apertures = [aperture for aperture in siaf.apertures if siaf[aperture].AperType == 'FULLSCA']
-
-            for aperture in possible_apertures:
-
-                logging.info('Working on aperture {} in {}'.format(aperture, instrument))
-                self.aperture = aperture
-
-                # Locate the record of most recent MAST search; use this time
-                # (plus a 30 day buffer to catch any missing files from
-                # previous run) as the start time in the new MAST search.
-                most_recent_search = self.most_recent_search()
-                self.query_start = most_recent_search - 30
-
-                # Query MAST for new dark files for this instrument/aperture
-                logging.info('\tQuery times: {} {}'.format(self.query_start, self.query_end))
-                new_entries = mast_query_darks(instrument, aperture, self.query_start, self.query_end)
-                logging.info('\tAperture: {}, new entries: {}'.format(self.aperture, len(new_entries)))
-
-                # Set up a directory to store the data for this aperture
-                self.data_dir = os.path.join(self.output_dir, 'data/{}_{}'.format(self.instrument.lower(), self.aperture.lower()))
-                if len(new_entries) > 0:
-                    ensure_dir_exists(self.data_dir)
-
-                # Get any new files to process
-                new_files = []
-                for file_entry in new_entries:
-                    output_filename = os.path.join(self.data_dir, file_entry['filename'])
-                    output_filename = output_filename.replace('_uncal.fits', '_uncal_0thgroup.fits').replace('_dark.fits', '_uncal_0thgroup.fits')
-
-                    # Dont process files that already exist in the grating stats database
-                    file_exists = self.file_exists_in_database(output_filename)
-                    if file_exists:
-                        logging.info('\t{} already exists in the grating database table.'.format(output_filename))
-                        continue
-
-                    # Save the 0th group image from each new file in the output directory; some dont exist in JWQL filesystem.
-                    try:
-                        filename = filesystem_path(file_entry['filename'])
-                        uncal_filename = filename.replace('_dark', '_uncal')
-                        if not os.path.isfile(uncal_filename):
-                            logging.info('\t{} does not exist in JWQL filesystem, even though {} does'.format(uncal_filename, filename))
-                        else:
-                            new_file = self.extract_zeroth_group(uncal_filename)
-                            new_files.append(new_file)
-                    except FileNotFoundError:
-                        logging.info('\t{} does not exist in JWQL filesystem'.format(file_entry['filename']))
-
-                # Run the grating monitor on any new files
-                if len(new_files) > 0:
-                    self.process(new_files)
-                    monitor_run = True
-                else:
-                    logging.info('\tGrating monitor skipped. {} new dark files for {}, {}.'.format(len(new_files), instrument, aperture))
-                    monitor_run = False
-
-                # Update the query history
-                new_entry = {'instrument': instrument,
-                             'aperture': aperture,
-                             'start_time_mjd': self.query_start,
-                             'end_time_mjd': self.query_end,
-                             'entries_found': len(new_entries),
-                             'files_found': len(new_files),
-                             'run_monitor': monitor_run,
-                             'entry_date': datetime.datetime.now()}
-                self.query_table.__table__.insert().execute(new_entry)
-                logging.info('\tUpdated the query history table')
+        self.instrument = 'nirspec'
+
+        # Identify which database tables to use
+        self.identify_tables()
+
+        # Get a list of all possible full-frame apertures for this instrument
+        siaf = Siaf(self.instrument)
+        possible_apertures = [aperture for aperture in siaf.apertures if siaf[aperture].AperType == 'FULLSCA']
+
+        for aperture in possible_apertures:
+            logging.info('Working on aperture {} in {}'.format(aperture, self.instrument))
+            self.aperture = aperture
+
+            # Locate the record of most recent MAST search; use this time
+            # (plus a 30 day buffer to catch any missing files from
+            # previous run) as the start time in the new MAST search.
+            most_recent_search = self.most_recent_search()
+            self.query_start = most_recent_search - 30
+
+            # Query MAST for new dark files for this instrument/aperture
+            logging.info('\tQuery times: {} {}'.format(self.query_start, self.query_end))
+            new_entries = mast_query_darks(self.instrument, aperture, self.query_start, self.query_end)
+            logging.info('\tAperture: {}, new entries: {}'.format(self.aperture, len(new_entries)))
+
+            # Set up a directory to store the data for this aperture
+            self.data_dir = os.path.join(self.output_dir, 'data/{}_{}'.format(self.instrument.lower(), self.aperture.lower()))
+            if len(new_entries) > 0:
+                ensure_dir_exists(self.data_dir)
+
+            # Get any new files to process
+            new_files = []
+            for file_entry in new_entries:
+                output_filename = os.path.join(self.data_dir, file_entry['filename'])
+                output_filename = output_filename.replace('_uncal.fits', '_uncal_0thgroup.fits').replace('_dark.fits', '_uncal_0thgroup.fits')
+
+                # Dont process files that already exist in the grating stats database
+                file_exists = self.file_exists_in_database(output_filename)
+                if file_exists:
+                    logging.info('\t{} already exists in the grating database table.'.format(output_filename))
+                    continue
+
+                # Save the 0th group image from each new file in the output directory; some dont exist in JWQL filesystem.
+                try:
+                    filename = filesystem_path(file_entry['filename'])
+                    uncal_filename = filename.replace('_dark', '_uncal')
+                    if not os.path.isfile(uncal_filename):
+                        logging.info('\t{} does not exist in JWQL filesystem, even though {} does'.format(uncal_filename, filename))
+                    else:
+                        new_file = self.extract_zeroth_group(uncal_filename)
+                        new_files.append(new_file)
+                except FileNotFoundError:
+                    logging.info('\t{} does not exist in JWQL filesystem'.format(file_entry['filename']))
+            # Run the grating monitor on any new files
+            # NEED TO CHANGE SUCH THAT MONITOR IS UPDATED WHEN NEW TELEMETRY IS AVAILABLE?
+            if len(new_files) > 0:
+                self.process(new_files)
+                monitor_run = True
+            else:
+                logging.info('\tGrating monitor skipped. {} new dark files for {}, {}.'.format(len(new_files), self.instrument, aperture))
+                monitor_run = False
+
+            # Update the query history
+            new_entry = {'instrument': self.instrument,
+                         'aperture': aperture,
+                         'start_time_mjd': self.query_start,
+                         'end_time_mjd': self.query_end,
+                         'entries_found': len(new_entries),
+                         'files_found': len(new_files),
+                         'run_monitor': monitor_run,
+                         'entry_date': datetime.datetime.now()}
+            self.query_table.__table__.insert().execute(new_entry)
+            logging.info('\tUpdated the query history table')
 
         logging.info('Grating Monitor completed successfully.')
 
diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py
index 98721c732..7d69bc25a 100644
--- a/jwql/utils/constants.py
+++ b/jwql/utils/constants.py
@@ -27,7 +27,7 @@
 import inflection
 
 # Each amplifier is represented by 2 tuples, the first for x coordinates
-# and the second for y coordinates. Within each tuple are value for
+# and the second for y coordinates. Within each tuple are values for
 # starting, ending, and step size. Step size is needed for MIRI, where
 # the pixels corresponding to the 4 amplifiers are interleaved.
 AMPLIFIER_BOUNDARIES = {'nircam': {'1': [(0, 512, 1), (0, 2048, 1)],
@@ -208,7 +208,14 @@
                                       'G235H', 'G395H', 'PRISM']
                           }
 
-GRATING_TELEMETRY = ['INRSH_GWA_ADCMGAIN', 'INRSH_GWA_ADCMOFFSET', 'INRSH_GWA_MOTOR_VREF', 'INRSI_C_GWA_X_POSITION', 'INRSI_C_GWA_Y_POSITION']
+# Dictionary with keys that are mnemonics used for grating wheel telemetry monitor
+# Corresponding limits are given for Lower Red, Lower Yellow, High Yellow, High Red,
+# Switch Low, and Switch High, respectively
+GRATING_TELEMETRY = {'INRSH_GWA_ADCMGAIN': [2.36, 2.39, 2.62, 2.65, 1.00, 3.00],
+                     'INRSH_GWA_ADCMOFFSET': [-0.0600, -0.0420, 0.0420, 0.0600, 1.00, 3.00],
+                     'INRSH_GWA_MOTOR_VREF': [-0.0600, -0.0420, 2.530, 2.55, 1.00, 3.00],
+                     'INRSI_C_GWA_X_POSITION': [-374, -360., 360., 374, 1.00, 3.00],
+                     'INRSI_C_GWA_Y_POSITION': [-374, -200., 200., 374, 1.00, 3.00]}
 
 # Possible suffix types for guider exposures
 GUIDER_SUFFIX_TYPES = ['stream', 'stacked_uncal', 'image_uncal', 'stacked_cal', 'image_cal']
diff --git a/jwql/website/apps/jwql/bokeh_containers.py b/jwql/website/apps/jwql/bokeh_containers.py
index c5809d304..dc56a381a 100644
--- a/jwql/website/apps/jwql/bokeh_containers.py
+++ b/jwql/website/apps/jwql/bokeh_containers.py
@@ -316,7 +316,7 @@ def grating_monitor_tabs(instrument):
 
         # Add the telemetry vs time plots for grating wheel sensor telemetry value
         plots = []
-        for telemetry in GRATING_TELEMETRY:
+        for telemetry in GRATING_TELEMETRY.keys():
             grating_plot = monitor_template.refs['figure_{}'.format(telemetry)]
             grating_plot.sizing_mode = 'scale_width'  # Make sure the sizing is adjustable
             plots.append(grating_plot)
@@ -332,6 +332,9 @@ def grating_monitor_tabs(instrument):
 
     # Return tab HTML and JavaScript to web app
     script, div = components(tabs)
+    # bad column name here ERROR:bokeh.core.validation.check:E-1001 (BAD_COLUMN_NAME):
+    # Glyph refers to nonexistent column name. This could either be due to a misspelling or typo,
+    # or due to an expected column being missing. : key "y" value "grating" [renderer: GlyphRenderer(id='1278', ...)]
 
     return div, script
 
diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py
index def39cfc5..461e8d903 100644
--- a/jwql/website/apps/jwql/data_containers.py
+++ b/jwql/website/apps/jwql/data_containers.py
@@ -577,7 +577,6 @@ def get_header_info(filename):
 
     # Extract header information from file
     for ext in range(0, len(hdulist)):
-    #for ext in range(0, 1):
 
         # Initialize dictionary to store header information for particular extension
         header_info[ext] = {}
diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_grating_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_grating_bokeh.py
index 92544e27e..1c5909c4b 100644
--- a/jwql/website/apps/jwql/monitor_pages/monitor_grating_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_grating_bokeh.py
@@ -19,6 +19,7 @@
 """
 
 from datetime import datetime, timedelta
+from astropy.time import Time
 import os
 
 from astropy.stats import sigma_clip
@@ -60,11 +61,11 @@ def load_data(self):
 
         # Query database for all data in grating wheel stats with a matching aperture,
         # and sort the data by exposure start time.
-        # FAILING HERE: (psycopg2.errors.UndefinedTable) relation "nirspec_grating_stats" does not exist
-        self.query_results = session.query(self.stats_table) \
-            .filter(self.stats_table.aperture == self._aperture) \
-            .order_by(self.stats_table.expstart) \
-            .all()
+        # If monitor fails here because relation does not exist, need to reset database
+        self.query_results = session.query(self.stats_table).all() \
+            # .filter(self.stats_table.aperture == self._aperture) \
+            # .order_by(self.stats_table.expstart) #\
+            #.all()
 
     def pre_init(self):
 
@@ -87,32 +88,34 @@ def post_init(self):
         self.load_data()
 
         # Update the mean grating wheel over time figures
-        self.update_mean_grating_figures()
+        self.update_grating_figures()
 
-    def update_mean_grating_figures(self):
-        """Updates the mean bias over time bokeh plots"""
-
-        # POSSIBLY REPLACE DARK EXPOSURES WITH SOMETHING ELSE
-        # Get the dark exposures and their starts times
-        filenames = [os.path.basename(result.uncal_filename).replace('_uncal.fits', '') for result in self.query_results]
-        expstarts_iso = np.array([result.expstart for result in self.query_results])
-        expstarts = np.array([datetime.strptime(date, '%Y-%m-%dT%H:%M:%S.%f') for date in expstarts_iso])
+    def update_grating_figures(self):
+        """Updates the grating telemetry over time bokeh plots"""
 
         # Update the figures for all GWA telemetry values
-        for telemetry in GRATING_TELEMETRY:
-            gwa_vals = np.array([getattr(result, '{}'.format(telemetry)) for result in self.query_results])
-            self.refs['source_{}'.format(telemetry)].data = {'time': expstarts,
-                                                             'time_iso': expstarts_iso,
-                                                             'gwa_vals': gwa_vals,
-                                                             'filename': filenames}
+        for telemetry in GRATING_TELEMETRY.keys():
+            # telemetry = telemetry.lower()
+            gwa_vals = np.array([getattr(result, '{}'.format(telemetry.lower())) for result in self.query_results])
+            # self.query_results is empty array...
+
+            # need to figure out why grating_monitor values aren't populating session.query(self.stats_table).filter(self.stats_table.aperture == self._aperture).order_by(self.stats_table.expstart).all()
+            # it seems like this is putting the right column labels in though, eg  self.stats_table.inrsi_c_gwa_y_position exists
+            times = []
+            for result in self.query_results:
+                times.append(float(result.expstart))
+            # import pdb
+            # pdb.set_trace()
+            gwa_vals[np.where(gwa_vals == None)] = 0.0  # temp fix to eliminate None's
+            self.refs['source_{}'.format(telemetry)].data = {'time': [datetime.now(), datetime.now()+timedelta(days=3), datetime.now()+timedelta(days=8)], #, #np.array(times), #[result.expstart] for result in self.query_results), # if self.query_results else [0.0,0.5,5.0],  # retreive from grating query data
+                                                             'gwa_vals': [0.0, 4.0, 0.9]}  # gwa_vals}
             self.refs['figure_{}'.format(telemetry)].title.text = '{}'.format(telemetry)
-            self.refs['figure_{}'.format(telemetry)].hover.tooltips = [('file', '@filename'),
-                                                                       ('time', '@time_iso'),
+            self.refs['figure_{}'.format(telemetry)].hover.tooltips = [('time', '@time'),
                                                                        ('gwa val', '@gwa_vals')]
 
             # Update plot limits if data exists
             if len(gwa_vals) != 0:
-                self.refs['gwa_x_{}'.format(telemetry)].start = expstarts.min() - timedelta(days=3)
-                self.refs['gwa_x_{}'.format(telemetry)].end = expstarts.max() + timedelta(days=3)
+                self.refs['gwa_x_{}'.format(telemetry)].start = datetime.now() - timedelta(days=3)
+                self.refs['gwa_x_{}'.format(telemetry)].end = datetime.now()
                 self.refs['gwa_y_{}'.format(telemetry)].start = gwa_vals.min() - 20
                 self.refs['gwa_y_{}'.format(telemetry)].end = gwa_vals.max() + 20

From e184a1c72b7139d2303a93032144037d5b14c3ef Mon Sep 17 00:00:00 2001
From: Teagan King <55458851+tnking97@users.noreply.github.com>
Date: Wed, 3 Nov 2021 09:12:58 -0400
Subject: [PATCH 06/18] telemetry plots debugging

---
 .../apps/jwql/monitor_pages/monitor_grating_bokeh.py   |  9 ++++++---
 .../monitor_pages/yaml/monitor_grating_interface.yaml  | 10 +++++-----
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_grating_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_grating_bokeh.py
index 1c5909c4b..d8e8f3cf8 100644
--- a/jwql/website/apps/jwql/monitor_pages/monitor_grating_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_grating_bokeh.py
@@ -107,11 +107,14 @@ def update_grating_figures(self):
             # import pdb
             # pdb.set_trace()
             gwa_vals[np.where(gwa_vals == None)] = 0.0  # temp fix to eliminate None's
-            self.refs['source_{}'.format(telemetry)].data = {'time': [datetime.now(), datetime.now()+timedelta(days=3), datetime.now()+timedelta(days=8)], #, #np.array(times), #[result.expstart] for result in self.query_results), # if self.query_results else [0.0,0.5,5.0],  # retreive from grating query data
-                                                             'gwa_vals': [0.0, 4.0, 0.9]}  # gwa_vals}
+            self.refs['source_{}'.format(telemetry)].data = {'time': [datetime.now(), datetime.now()-timedelta(days=1), datetime.now()-timedelta(days=2)],
+                                                             #, #np.array(times), #[result.expstart] for result in self.query_results), # if self.query_results else [0.0,0.5,5.0],  # retreive from grating query data
+                                                             'time_iso': [datetime.now(), datetime.now()-timedelta(days=1), datetime.now()-timedelta(days=8)],  # not actually used
+                                                             'grating': [0.0, 4.0, -9.9],
+                                                             'filename': ["tempfile1", "tempfile2", "tempfile3"]}  # gwa_vals}
             self.refs['figure_{}'.format(telemetry)].title.text = '{}'.format(telemetry)
             self.refs['figure_{}'.format(telemetry)].hover.tooltips = [('time', '@time'),
-                                                                       ('gwa val', '@gwa_vals')]
+                                                                       ('gwa val', '@grating')]
 
             # Update plot limits if data exists
             if len(gwa_vals) != 0:
diff --git a/jwql/website/apps/jwql/monitor_pages/yaml/monitor_grating_interface.yaml b/jwql/website/apps/jwql/monitor_pages/yaml/monitor_grating_interface.yaml
index 508ee0e26..18f65b931 100644
--- a/jwql/website/apps/jwql/monitor_pages/yaml/monitor_grating_interface.yaml
+++ b/jwql/website/apps/jwql/monitor_pages/yaml/monitor_grating_interface.yaml
@@ -23,7 +23,7 @@
     ref: "figure_INRSH_GWA_ADCMGAIN"
     x_axis_label: "Date"
     x_axis_type: "datetime"
-    y_axis_label: "INRSH_GWA_ADCMGAIN [DN]"
+    y_axis_label: "INRSH_GWA_ADCMGAIN [V]"
     x_range: *gwa_x_INRSH_GWA_ADCMGAIN
     y_range: *gwa_y_INRSH_GWA_ADCMGAIN
     height: 800
@@ -56,7 +56,7 @@
     ref: "figure_INRSH_GWA_ADCMOFFSET"
     x_axis_label: "Date"
     x_axis_type: "datetime"
-    y_axis_label: "INRSH_GWA_ADCMOFFSET [DN]"
+    y_axis_label: "INRSH_GWA_ADCMOFFSET [V]"
     x_range: *gwa_x_INRSH_GWA_ADCMOFFSET
     y_range: *gwa_y_INRSH_GWA_ADCMOFFSET
     height: 800
@@ -89,7 +89,7 @@
     ref: "figure_INRSH_GWA_MOTOR_VREF"
     x_axis_label: "Date"
     x_axis_type: "datetime"
-    y_axis_label: "INRSH_GWA_MOTOR_VREF [DN]"
+    y_axis_label: "INRSH_GWA_MOTOR_VREF [V]"
     x_range: *gwa_x_INRSH_GWA_MOTOR_VREF
     y_range: *gwa_y_INRSH_GWA_MOTOR_VREF
     height: 800
@@ -122,7 +122,7 @@
     ref: "figure_INRSI_C_GWA_X_POSITION"
     x_axis_label: "Date"
     x_axis_type: "datetime"
-    y_axis_label: "INRSI_C_GWA_X_POSITION [DN]"
+    y_axis_label: "INRSI_C_GWA_X_POSITION [mV]"
     x_range: *gwa_x_INRSI_C_GWA_X_POSITION
     y_range: *gwa_y_INRSI_C_GWA_X_POSITION
     height: 800
@@ -155,7 +155,7 @@
     ref: "figure_INRSI_C_GWA_Y_POSITION"
     x_axis_label: "Date"
     x_axis_type: "datetime"
-    y_axis_label: "INRSI_C_GWA_Y_POSITION [DN]"
+    y_axis_label: "INRSI_C_GWA_Y_POSITION [mV]"
     x_range: *gwa_x_INRSI_C_GWA_Y_POSITION
     y_range: *gwa_y_INRSI_C_GWA_Y_POSITION
     height: 800

From 4c1179746fe74ff6a60ae15f645b95d2f4f1facf Mon Sep 17 00:00:00 2001
From: Teagan King <55458851+tnking97@users.noreply.github.com>
Date: Mon, 8 Nov 2021 16:04:20 -0500
Subject: [PATCH 07/18] include plot points

---
 .../common_monitors/grating_monitor.py        | 26 +++++++----
 .../monitor_pages/monitor_grating_bokeh.py    | 45 +++++++++----------
 2 files changed, 40 insertions(+), 31 deletions(-)

diff --git a/jwql/instrument_monitors/common_monitors/grating_monitor.py b/jwql/instrument_monitors/common_monitors/grating_monitor.py
index 6e2c204ea..03e367e38 100755
--- a/jwql/instrument_monitors/common_monitors/grating_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/grating_monitor.py
@@ -254,14 +254,24 @@ def process(self, file_list):
         # Construct new entry for this file for the grating wheel database table.
         for telem in GRATING_TELEMETRY.keys():
             mnemonic = get_mnemonic(telem, start_telemetry_time, end_telemetry_time)
+            other_telems = GRATING_TELEMETRY.keys()
+            other_telems_dict = {}
+            for telems in other_telems:
+                other_telems_dict[telems.lower()] = 0.0  # Change to None
+            other_telems_dict.pop(telem.lower())
             for time in mnemonic.data['MJD']:
-                grating_db_entry = {'aperture': self.aperture,
-                                    'read_pattern': "temp",
-                                    'expstart': time,
-                                    'inrsh_gwa_adcmgain_time': mnemonic.data['euvalue'][np.where(mnemonic.data['MJD'] == time)],
-                                    'run_monitor': False,
-                                    'entry_date': datetime.datetime.now()  # need slightly different times to add to database
-                                    }
+                try:
+                    grating_db_entry = {'aperture': self.aperture,
+                                        'read_pattern': "temporary",
+                                        'expstart': time,
+                                        telem.lower(): float(mnemonic.data['euvalue'][np.where(mnemonic.data['MJD'] == time)]),
+                                        'run_monitor': False,
+                                        'entry_date': datetime.datetime.now()  # need slightly different times to add to database
+                                        }
+                    grating_db_entry.update(other_telems_dict)
+                except TypeError:
+                    logging.warning("may be skipping a value with same entry_date. grating_db_entry: {}".format(grating_db_entry))
+                    continue   # if repeat entries?
 
                 # Add this new entry to the grating database table
                 self.stats_table.__table__.insert().execute(grating_db_entry)
@@ -353,7 +363,7 @@ def run(self):
                     logging.info('\t{} does not exist in JWQL filesystem'.format(file_entry['filename']))
             # Run the grating monitor on any new files
             # NEED TO CHANGE SUCH THAT MONITOR IS UPDATED WHEN NEW TELEMETRY IS AVAILABLE?
-            if len(new_files) > 0:
+            if len(new_files) == 0:  # WAS >  BUT NEED TO CHANGE!!
                 self.process(new_files)
                 monitor_run = True
             else:
diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_grating_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_grating_bokeh.py
index d8e8f3cf8..0b8e3781a 100644
--- a/jwql/website/apps/jwql/monitor_pages/monitor_grating_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_grating_bokeh.py
@@ -16,6 +16,7 @@
 
         from jwql.website.apps.jwql import monitor_pages
         monitor_template = monitor_pages.GratingMonitor()
+        monitor_template.input_parameters = ('NIRSpec', 'NRS1_FULL')
 """
 
 from datetime import datetime, timedelta
@@ -62,10 +63,10 @@ def load_data(self):
         # Query database for all data in grating wheel stats with a matching aperture,
         # and sort the data by exposure start time.
         # If monitor fails here because relation does not exist, need to reset database
-        self.query_results = session.query(self.stats_table).all() \
-            # .filter(self.stats_table.aperture == self._aperture) \
-            # .order_by(self.stats_table.expstart) #\
-            #.all()
+        self.query_results = session.query(self.stats_table) \
+            .filter(self.stats_table.aperture == self._aperture) \
+            .order_by(self.stats_table.expstart) \
+            .all()
 
     def pre_init(self):
 
@@ -95,30 +96,28 @@ def update_grating_figures(self):
 
         # Update the figures for all GWA telemetry values
         for telemetry in GRATING_TELEMETRY.keys():
-            # telemetry = telemetry.lower()
             gwa_vals = np.array([getattr(result, '{}'.format(telemetry.lower())) for result in self.query_results])
-            # self.query_results is empty array...
-
-            # need to figure out why grating_monitor values aren't populating session.query(self.stats_table).filter(self.stats_table.aperture == self._aperture).order_by(self.stats_table.expstart).all()
-            # it seems like this is putting the right column labels in though, eg  self.stats_table.inrsi_c_gwa_y_position exists
-            times = []
-            for result in self.query_results:
-                times.append(float(result.expstart))
-            # import pdb
-            # pdb.set_trace()
             gwa_vals[np.where(gwa_vals == None)] = 0.0  # temp fix to eliminate None's
-            self.refs['source_{}'.format(telemetry)].data = {'time': [datetime.now(), datetime.now()-timedelta(days=1), datetime.now()-timedelta(days=2)],
-                                                             #, #np.array(times), #[result.expstart] for result in self.query_results), # if self.query_results else [0.0,0.5,5.0],  # retreive from grating query data
-                                                             'time_iso': [datetime.now(), datetime.now()-timedelta(days=1), datetime.now()-timedelta(days=8)],  # not actually used
-                                                             'grating': [0.0, 4.0, -9.9],
-                                                             'filename': ["tempfile1", "tempfile2", "tempfile3"]}  # gwa_vals}
-            self.refs['figure_{}'.format(telemetry)].title.text = '{}'.format(telemetry)
-            self.refs['figure_{}'.format(telemetry)].hover.tooltips = [('time', '@time'),
-                                                                       ('gwa val', '@grating')]
+
+            if len(gwa_vals) != 0:  # SHOULD UPDATE WITHOUT IF STATEMENT
+                expstarts_iso = np.array([str(Time(Time(result.expstart, format='mjd'), format='iso')) for result in self.query_results])
+                expstarts = np.array([datetime.strptime(date, '%Y-%m-%d %H:%M:%S.%f') for date in expstarts_iso])
+
+                fake_filenames = []
+                for length in range(len(gwa_vals)):
+                    fake_filenames.append("file{}".format(length))
+                self.refs['source_{}'.format(telemetry)].data = {'time': expstarts,  # [datetime.now(), datetime.now()-timedelta(days=1), datetime.now()-timedelta(days=2)],
+                                                                 'time_iso': expstarts,   # SHOULD BE EXPSTARTS_ISO  # [datetime.now(), datetime.now()-timedelta(days=1), datetime.now()-timedelta(days=8)],  # not actually used
+                                                                 'grating': gwa_vals,
+                                                                 'filename': fake_filenames}
+
+                self.refs['figure_{}'.format(telemetry)].title.text = '{}'.format(telemetry)
+                self.refs['figure_{}'.format(telemetry)].hover.tooltips = [('time', '@time'),
+                                                                           ('gwa val', '@grating')]
 
             # Update plot limits if data exists
             if len(gwa_vals) != 0:
-                self.refs['gwa_x_{}'.format(telemetry)].start = datetime.now() - timedelta(days=3)
+                self.refs['gwa_x_{}'.format(telemetry)].start = datetime.now() - timedelta(days=300)
                 self.refs['gwa_x_{}'.format(telemetry)].end = datetime.now()
                 self.refs['gwa_y_{}'.format(telemetry)].start = gwa_vals.min() - 20
                 self.refs['gwa_y_{}'.format(telemetry)].end = gwa_vals.max() + 20

From 11bb268bd35a7d8f13db87e90303bfa662b161cb Mon Sep 17 00:00:00 2001
From: Teagan King <55458851+tnking97@users.noreply.github.com>
Date: Tue, 9 Nov 2021 16:22:45 -0500
Subject: [PATCH 08/18] grating bokeh plot update and color bands

---
 .../common_monitors/grating_monitor.py        |  2 +-
 jwql/website/apps/jwql/bokeh_containers.py    | 35 +++++++++++++++++++
 .../monitor_pages/monitor_grating_bokeh.py    | 10 +++---
 3 files changed, 41 insertions(+), 6 deletions(-)

diff --git a/jwql/instrument_monitors/common_monitors/grating_monitor.py b/jwql/instrument_monitors/common_monitors/grating_monitor.py
index 03e367e38..9238368c9 100755
--- a/jwql/instrument_monitors/common_monitors/grating_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/grating_monitor.py
@@ -257,7 +257,7 @@ def process(self, file_list):
             other_telems = GRATING_TELEMETRY.keys()
             other_telems_dict = {}
             for telems in other_telems:
-                other_telems_dict[telems.lower()] = 0.0  # Change to None
+                other_telems_dict[telems.lower()] = None
             other_telems_dict.pop(telem.lower())
             for time in mnemonic.data['MJD']:
                 try:
diff --git a/jwql/website/apps/jwql/bokeh_containers.py b/jwql/website/apps/jwql/bokeh_containers.py
index dc56a381a..24791a4ef 100644
--- a/jwql/website/apps/jwql/bokeh_containers.py
+++ b/jwql/website/apps/jwql/bokeh_containers.py
@@ -23,6 +23,7 @@
 
 from bokeh.embed import components
 from bokeh.layouts import layout
+from bokeh.models import BoxAnnotation
 from bokeh.models.widgets import Tabs, Panel
 
 from . import monitor_pages
@@ -319,6 +320,40 @@ def grating_monitor_tabs(instrument):
         for telemetry in GRATING_TELEMETRY.keys():
             grating_plot = monitor_template.refs['figure_{}'.format(telemetry)]
             grating_plot.sizing_mode = 'scale_width'  # Make sure the sizing is adjustable
+
+            # TEST OUT ADDING COLORED BOXES TO PLOTS
+            # Lower Red, Lower Yellow, High Yellow, High Red, Switch Low, and Switch High
+            
+            red_low_lower_threshold = GRATING_TELEMETRY[telemetry][4]
+            yellow_low_lower_threshold = GRATING_TELEMETRY[telemetry][0]
+            green_lower_threshold = GRATING_TELEMETRY[telemetry][1]
+
+            green_higher_threshold = GRATING_TELEMETRY[telemetry][2]
+            yellow_high_higher_threshold = GRATING_TELEMETRY[telemetry][3]
+            red_high_higher_threshold = GRATING_TELEMETRY[telemetry][5]
+
+            # {'INRSH_GWA_ADCMGAIN': [2.36, 2.39, 2.62, 2.65, 1.00, 3.00],
+            # 'INRSH_GWA_ADCMOFFSET': [-0.0600, -0.0420, 0.0420, 0.0600, 1.00, 3.00],  # ****
+            # 'INRSH_GWA_MOTOR_VREF': [-0.0600, -0.0420, 2.530, 2.55, 1.00, 3.00],  # ****
+            # 'INRSI_C_GWA_X_POSITION': [-374, -360., 360., 374, 1.00, 3.00],  # ****
+            # 'INRSI_C_GWA_Y_POSITION': [-374, -200., 200., 374, 1.00, 3.00]}  # ****
+
+            yellow_low_higher_threshold = green_lower_threshold
+            yellow_high_lower_threshold = green_higher_threshold
+            red_low_higher_threshold = yellow_low_lower_threshold
+            red_high_lower_threshold = yellow_high_higher_threshold
+
+            green = BoxAnnotation(bottom=green_lower_threshold, top=green_higher_threshold, fill_color='chartreuse', fill_alpha=0.2)
+            grating_plot.add_layout(green)
+            yellow_high = BoxAnnotation(bottom=yellow_high_lower_threshold, top=yellow_high_higher_threshold, fill_color='gold', fill_alpha=0.2)
+            grating_plot.add_layout(yellow_high)
+            yellow_low = BoxAnnotation(bottom=yellow_low_lower_threshold, top=yellow_low_higher_threshold, fill_color='gold', fill_alpha=0.2)
+            grating_plot.add_layout(yellow_low)          
+            red_high = BoxAnnotation(bottom=red_high_lower_threshold, top=red_high_higher_threshold, fill_color='red', fill_alpha=0.1)
+            grating_plot.add_layout(red_high)
+            # red_low = BoxAnnotation(bottom=red_low_lower_threshold, top=red_low_higher_threshold, fill_color='red', fill_alpha=0.1)
+            # grating_plot.add_layout(red_low)
+
             plots.append(grating_plot)
 
         # Put grating plots on the top row.
diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_grating_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_grating_bokeh.py
index 0b8e3781a..cadd4d61b 100644
--- a/jwql/website/apps/jwql/monitor_pages/monitor_grating_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_grating_bokeh.py
@@ -97,7 +97,6 @@ def update_grating_figures(self):
         # Update the figures for all GWA telemetry values
         for telemetry in GRATING_TELEMETRY.keys():
             gwa_vals = np.array([getattr(result, '{}'.format(telemetry.lower())) for result in self.query_results])
-            gwa_vals[np.where(gwa_vals == None)] = 0.0  # temp fix to eliminate None's
 
             if len(gwa_vals) != 0:  # SHOULD UPDATE WITHOUT IF STATEMENT
                 expstarts_iso = np.array([str(Time(Time(result.expstart, format='mjd'), format='iso')) for result in self.query_results])
@@ -116,8 +115,9 @@ def update_grating_figures(self):
                                                                            ('gwa val', '@grating')]
 
             # Update plot limits if data exists
-            if len(gwa_vals) != 0:
-                self.refs['gwa_x_{}'.format(telemetry)].start = datetime.now() - timedelta(days=300)
+            gwa_vals_cut = gwa_vals[np.where(gwa_vals!=None)]  # USE ABOVE AS WELL?
+            if len(gwa_vals_cut) != 0:
+                self.refs['gwa_x_{}'.format(telemetry)].start = datetime.now() - timedelta(days=100)
                 self.refs['gwa_x_{}'.format(telemetry)].end = datetime.now()
-                self.refs['gwa_y_{}'.format(telemetry)].start = gwa_vals.min() - 20
-                self.refs['gwa_y_{}'.format(telemetry)].end = gwa_vals.max() + 20
+                self.refs['gwa_y_{}'.format(telemetry)].start = gwa_vals_cut.min() - 0.5
+                self.refs['gwa_y_{}'.format(telemetry)].end = gwa_vals_cut.max() + 0.5

From 07f2849d4d8e4e206e3ce9ced893d3dd055de88c Mon Sep 17 00:00:00 2001
From: Teagan King <55458851+tnking97@users.noreply.github.com>
Date: Thu, 11 Nov 2021 10:52:42 -0500
Subject: [PATCH 09/18] finalize colored boxes for telemetry plots

---
 jwql/website/apps/jwql/bokeh_containers.py | 25 ++++++----------------
 1 file changed, 7 insertions(+), 18 deletions(-)

diff --git a/jwql/website/apps/jwql/bokeh_containers.py b/jwql/website/apps/jwql/bokeh_containers.py
index 24791a4ef..7bad30f72 100644
--- a/jwql/website/apps/jwql/bokeh_containers.py
+++ b/jwql/website/apps/jwql/bokeh_containers.py
@@ -321,23 +321,15 @@ def grating_monitor_tabs(instrument):
             grating_plot = monitor_template.refs['figure_{}'.format(telemetry)]
             grating_plot.sizing_mode = 'scale_width'  # Make sure the sizing is adjustable
 
-            # TEST OUT ADDING COLORED BOXES TO PLOTS
+            # Add colored boxes to plots indicating green, yellow, and red regions
+            # Grating Telemetry dictionary indices are in the following order:
             # Lower Red, Lower Yellow, High Yellow, High Red, Switch Low, and Switch High
-            
-            red_low_lower_threshold = GRATING_TELEMETRY[telemetry][4]
+            red_low_lower_threshold = GRATING_TELEMETRY[telemetry][0] - 100  # arbitrary box height to show red on rest of plot
             yellow_low_lower_threshold = GRATING_TELEMETRY[telemetry][0]
             green_lower_threshold = GRATING_TELEMETRY[telemetry][1]
-
             green_higher_threshold = GRATING_TELEMETRY[telemetry][2]
             yellow_high_higher_threshold = GRATING_TELEMETRY[telemetry][3]
-            red_high_higher_threshold = GRATING_TELEMETRY[telemetry][5]
-
-            # {'INRSH_GWA_ADCMGAIN': [2.36, 2.39, 2.62, 2.65, 1.00, 3.00],
-            # 'INRSH_GWA_ADCMOFFSET': [-0.0600, -0.0420, 0.0420, 0.0600, 1.00, 3.00],  # ****
-            # 'INRSH_GWA_MOTOR_VREF': [-0.0600, -0.0420, 2.530, 2.55, 1.00, 3.00],  # ****
-            # 'INRSI_C_GWA_X_POSITION': [-374, -360., 360., 374, 1.00, 3.00],  # ****
-            # 'INRSI_C_GWA_Y_POSITION': [-374, -200., 200., 374, 1.00, 3.00]}  # ****
-
+            red_high_higher_threshold = GRATING_TELEMETRY[telemetry][3] + 100  # arbitrary box height to show red on rest of plot
             yellow_low_higher_threshold = green_lower_threshold
             yellow_high_lower_threshold = green_higher_threshold
             red_low_higher_threshold = yellow_low_lower_threshold
@@ -348,11 +340,11 @@ def grating_monitor_tabs(instrument):
             yellow_high = BoxAnnotation(bottom=yellow_high_lower_threshold, top=yellow_high_higher_threshold, fill_color='gold', fill_alpha=0.2)
             grating_plot.add_layout(yellow_high)
             yellow_low = BoxAnnotation(bottom=yellow_low_lower_threshold, top=yellow_low_higher_threshold, fill_color='gold', fill_alpha=0.2)
-            grating_plot.add_layout(yellow_low)          
+            grating_plot.add_layout(yellow_low)
             red_high = BoxAnnotation(bottom=red_high_lower_threshold, top=red_high_higher_threshold, fill_color='red', fill_alpha=0.1)
             grating_plot.add_layout(red_high)
-            # red_low = BoxAnnotation(bottom=red_low_lower_threshold, top=red_low_higher_threshold, fill_color='red', fill_alpha=0.1)
-            # grating_plot.add_layout(red_low)
+            red_low = BoxAnnotation(bottom=red_low_lower_threshold, top=red_low_higher_threshold, fill_color='red', fill_alpha=0.1)
+            grating_plot.add_layout(red_low)
 
             plots.append(grating_plot)
 
@@ -367,9 +359,6 @@ def grating_monitor_tabs(instrument):
 
     # Return tab HTML and JavaScript to web app
     script, div = components(tabs)
-    # bad column name here ERROR:bokeh.core.validation.check:E-1001 (BAD_COLUMN_NAME):
-    # Glyph refers to nonexistent column name. This could either be due to a misspelling or typo,
-    # or due to an expected column being missing. : key "y" value "grating" [renderer: GlyphRenderer(id='1278', ...)]
 
     return div, script
 

From bf29cd19daccb70f161a84eb39df645d941890c0 Mon Sep 17 00:00:00 2001
From: Teagan King <55458851+tnking97@users.noreply.github.com>
Date: Fri, 12 Nov 2021 15:18:46 -0500
Subject: [PATCH 10/18] removed unnecessary table columns, updated one plot per
 grating, changed file processing to telemetry database existence/always run

---
 .../nirspec/nirspec_grating_query_history.txt |   3 -
 .../nirspec/nirspec_grating_stats.txt         |  23 +-
 .../common_monitors/grating_monitor.py        | 342 +++--------
 jwql/utils/constants.py                       |  27 +-
 jwql/website/apps/jwql/bokeh_containers.py    |  89 ++-
 .../monitor_pages/monitor_grating_bokeh.py    |  25 +-
 .../yaml/monitor_grating_interface.yaml       | 530 ++++++++++++++++--
 7 files changed, 673 insertions(+), 366 deletions(-)

diff --git a/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_query_history.txt b/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_query_history.txt
index 39e615bfc..d4490d3ff 100644
--- a/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_query_history.txt
+++ b/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_query_history.txt
@@ -1,7 +1,4 @@
-APERTURE, string
 START_TIME_MJD, float
 END_TIME_MJD, float
-ENTRIES_FOUND, integer
-FILES_FOUND, integer
 RUN_MONITOR, bool
 ENTRY_DATE, datetime
\ No newline at end of file
diff --git a/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_stats.txt b/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_stats.txt
index cdae30ecd..ee5273436 100644
--- a/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_stats.txt
+++ b/jwql/database/monitor_table_definitions/nirspec/nirspec_grating_stats.txt
@@ -1,9 +1,20 @@
-APERTURE, string
-UNCAL_FILENAME, string
-READ_PATTERN, string
-EXPSTART, string
+TIME, string
 INRSH_GWA_ADCMGAIN, float
 INRSH_GWA_ADCMOFFSET, float
 INRSH_GWA_MOTOR_VREF, float
-INRSI_C_GWA_X_POSITION, float
-INRSI_C_GWA_Y_POSITION, float
+PRISM_INRSI_C_GWA_X_POSITION, float
+PRISM_INRSI_C_GWA_Y_POSITION, float
+MIRROR_INRSI_C_GWA_X_POSITION, float
+MIRROR_INRSI_C_GWA_Y_POSITION, float
+G140H_INRSI_C_GWA_X_POSITION, float
+G140H_INRSI_C_GWA_Y_POSITION, float
+G235H_INRSI_C_GWA_X_POSITION, float
+G235H_INRSI_C_GWA_Y_POSITION, float
+G395H_INRSI_C_GWA_X_POSITION, float
+G395H_INRSI_C_GWA_Y_POSITION, float
+G140M_INRSI_C_GWA_X_POSITION, float
+G140M_INRSI_C_GWA_Y_POSITION, float
+G235M_INRSI_C_GWA_X_POSITION, float
+G235M_INRSI_C_GWA_Y_POSITION, float
+G395M_INRSI_C_GWA_X_POSITION, float
+G395M_INRSI_C_GWA_Y_POSITION, float
diff --git a/jwql/instrument_monitors/common_monitors/grating_monitor.py b/jwql/instrument_monitors/common_monitors/grating_monitor.py
index 9238368c9..0bfebd7ba 100755
--- a/jwql/instrument_monitors/common_monitors/grating_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/grating_monitor.py
@@ -16,33 +16,23 @@
         python grating_monitor.py
 """
 
-from collections import OrderedDict
 import datetime
 import logging
 import os
 
-from astropy.io import fits
-from astropy.stats import sigma_clip
 from astropy.time import Time
-from astropy.visualization import ZScaleInterval
 from jwql.edb.engineering_database import get_mnemonic
 from jwql.edb.engineering_database import get_mnemonics
 import matplotlib
 matplotlib.use('Agg')
-import matplotlib.pyplot as plt
-from mpl_toolkits.axes_grid1 import make_axes_locatable
 import numpy as np
-from pysiaf import Siaf
-from sqlalchemy.sql.expression import and_
 
 from jwql.database.database_interface import session
 from jwql.database.database_interface import NIRSpecGratingQueryHistory, NIRSpecGratingStats
-from jwql.instrument_monitors.common_monitors.dark_monitor import mast_query_darks
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE, GRATING_TELEMETRY
 from jwql.utils.logging_functions import log_info, log_fail
 from jwql.utils.monitor_utils import update_monitor_table
-from jwql.utils.permissions import set_permissions
-from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config, initialize_instrument_monitor
+from jwql.utils.utils import ensure_dir_exists, get_config, initialize_instrument_monitor
 
 
 class Grating():
@@ -71,70 +61,11 @@ class Grating():
 
     instrument : str
         Name of instrument used to collect the data.
-
-    aperture : str
-        Name of the aperture used (e.g. ``NRS1``).
     """
 
     def __init__(self):
         """Initialize an instance of the ``Grating`` class."""
 
-    def extract_zeroth_group(self, filename):
-        """Extracts the 0th group of a fits image and outputs it into
-        a new fits file.
-
-        Parameters
-        ----------
-        filename : str
-            The fits file from which the 0th group will be extracted.
-
-        Returns
-        -------
-        output_filename : str
-            The full path to the output file.
-        """
-
-        output_filename = os.path.join(self.data_dir, os.path.basename(filename).replace('.fits', '_0thgroup.fits'))
-
-        # Write a new fits file containing the primary and science
-        # headers from the input file, as well as the 0th group
-        # data of the first integration
-        hdu = fits.open(filename)
-        new_hdu = fits.HDUList([hdu['PRIMARY'], hdu['SCI']])
-        new_hdu['SCI'].data = hdu['SCI'].data[0:1, 0:1, :, :]
-        new_hdu.writeto(output_filename, overwrite=True)
-        hdu.close()
-        new_hdu.close()
-        set_permissions(output_filename)
-        logging.info('\t{} created'.format(output_filename))
-
-        return output_filename
-
-    def file_exists_in_database(self, filename):
-        """Checks if an entry for filename exists in the grating stats
-        database.
-
-        Parameters
-        ----------
-        filename : str
-            The full path to the uncal filename.
-
-        Returns
-        -------
-        file_exists : bool
-            ``True`` if filename exists in the grating stats database.
-        """
-
-        query = session.query(self.stats_table)
-        results = query.filter(self.stats_table.uncal_filename == filename).all()
-
-        if len(results) != 0:
-            file_exists = True
-        else:
-            file_exists = False
-
-        return file_exists
-
     def identify_tables(self):
         """Determine which database tables to use for a run of the grating wheel
         monitor.
@@ -144,82 +75,9 @@ def identify_tables(self):
         self.query_table = eval('{}GratingQueryHistory'.format(mixed_case_name))
         self.stats_table = eval('{}GratingStats'.format(mixed_case_name))
 
-    def image_to_png(self, image, outname):
-        """Ouputs an image array into a png file.
-
-        Parameters
-        ----------
-        image : numpy.ndarray
-            2D image array.
-
-        outname : str
-            The name given to the output png file.
-
-        Returns
-        -------
-        output_filename : str
-            The full path to the output png file.
-        """
-
-        output_filename = os.path.join(self.data_dir, '{}.png'.format(outname))
-
-        # Get image scale limits
-        z = ZScaleInterval()
-        vmin, vmax = z.get_limits(image)
-
-        # Plot the image
-        plt.figure(figsize=(12, 12))
-        ax = plt.gca()
-        im = ax.imshow(image, cmap='gray', origin='lower', vmin=vmin, vmax=vmax)
-        ax.set_title(outname.split('_uncal')[0])
-
-        # Make the colorbar
-        divider = make_axes_locatable(ax)
-        cax = divider.append_axes("right", size="5%", pad=0.4)
-        cbar = plt.colorbar(im, cax=cax)
-        cbar.set_label('Signal [DN]')
-
-        # Save the image
-        plt.savefig(output_filename, bbox_inches='tight', dpi=200)
-        set_permissions(output_filename)
-        logging.info('\t{} created'.format(output_filename))
-
-        return output_filename
-
-    def make_histogram(self, data):
-        """Creates a histogram of the input data and returns the bin
-        centers and the counts in each bin.
-
-        Parameters
-        ----------
-        data : numpy.ndarray
-            The input data.
-
-        Returns
-        -------
-        counts : numpy.ndarray
-            The counts in each histogram bin.
-
-        bin_centers : numpy.ndarray
-            The histogram bin centers.
-        """
-
-        # Calculate the histogram range as that within 5 sigma from the mean
-        data = data.flatten()
-        clipped = sigma_clip(data, sigma=3.0, maxiters=5)
-        mean, stddev = np.nanmean(clipped), np.nanstd(clipped)
-        lower_thresh, upper_thresh = mean - 4 * stddev, mean + 4 * stddev
-
-        # Make the histogram
-        counts, bin_edges = np.histogram(data, bins='auto', range=(lower_thresh, upper_thresh))
-        bin_centers = (bin_edges[:-1] + bin_edges[1:]) / 2
-
-        return counts, bin_centers
-
     def most_recent_search(self):
         """Query the query history database and return the information
-        on the most recent query for the given ``aperture_name`` where
-        the grating wheel monitor was executed.
+        on the most recent query where the grating wheel monitor was executed.
 
         Returns
         -------
@@ -227,73 +85,89 @@ def most_recent_search(self):
             Date (in MJD) of the ending range of the previous MAST query
             where the grating wheel monitor was run.
         """
-        query = session.query(self.query_table).filter(and_(self.query_table.aperture == self.aperture,
-            self.query_table.run_monitor == True)).order_by(self.query_table.end_time_mjd).all()
+        query = session.query(self.query_table).filter(self.query_table.run_monitor == True).order_by(self.query_table.end_time_mjd).all()
 
         if len(query) == 0:
             query_result = 57357.0  # a.k.a. Dec 1, 2015 == CV3
-            logging.info(('\tNo query history for {}. Beginning search date will be set to {}.'.format(self.aperture, query_result)))
+            logging.info(('\tNo query history. Beginning search date will be set to {}.'.format(query_result)))
         else:
             query_result = query[-1].end_time_mjd
 
         return query_result
 
-    def process(self, file_list):
+    def process(self):
         """The main method for processing darks.  See module docstrings
         for further details.
 
-        Parameters
-        ----------
-        file_list : list
-            List of filenames (including full paths) to the dark current
-            files.
         """
-        start_telemetry_time = Time(Time.now(), format='decimalyear') - 300  # Use telemetry from previous 30 days
-        end_telemetry_time = Time(Time.now(), format='decimalyear')
+
+        start_telemetry_time = Time(Time(self.query_start, format='mjd'), format='decimalyear')
+        end_telemetry_time = Time(Time(self.query_end, format='mjd'), format='decimalyear')
+        # grating_val gets bogged down if more than about a day
 
         # Construct new entry for this file for the grating wheel database table.
+        retrieved_grat = False
         for telem in GRATING_TELEMETRY.keys():
-            mnemonic = get_mnemonic(telem, start_telemetry_time, end_telemetry_time)
+            if 'POSITION' in telem:
+                grating = telem.split('_')[0]
+                telemetry = telem.replace(grating+"_", "")
+                if retrieved_grat is False:
+                    try:
+                        mnemonics = get_mnemonics([telemetry, 'INRSI_GWA_MECH_POS'], start_telemetry_time, end_telemetry_time)
+                        mnemonic = mnemonics[telemetry]
+                        grating_val = mnemonics['INRSI_GWA_MECH_POS']
+                        retrieved_grat = True
+                    except TypeError:
+                        print("type error because data was empty")
+            else:
+                try:
+                    mnemonic = get_mnemonic(telem, start_telemetry_time, end_telemetry_time)
+                except TypeError:
+                    print("type error because data was empty")
+                    continue
             other_telems = GRATING_TELEMETRY.keys()
             other_telems_dict = {}
             for telems in other_telems:
                 other_telems_dict[telems.lower()] = None
             other_telems_dict.pop(telem.lower())
             for time in mnemonic.data['MJD']:
-                try:
-                    grating_db_entry = {'aperture': self.aperture,
-                                        'read_pattern': "temporary",
-                                        'expstart': time,
-                                        telem.lower(): float(mnemonic.data['euvalue'][np.where(mnemonic.data['MJD'] == time)]),
-                                        'run_monitor': False,
-                                        'entry_date': datetime.datetime.now()  # need slightly different times to add to database
-                                        }
-                    grating_db_entry.update(other_telems_dict)
-                except TypeError:
-                    logging.warning("may be skipping a value with same entry_date. grating_db_entry: {}".format(grating_db_entry))
-                    continue   # if repeat entries?
-
-                # Add this new entry to the grating database table
-                self.stats_table.__table__.insert().execute(grating_db_entry)
-                logging.info('\tNew entry added to grating database table: {}'.format(grating_db_entry))
-
-        # FAILS BECAUSE LISTS OF FLOATS NOT FLOATS...
-        # mnemonics = get_mnemonics(GRATING_TELEMETRY, start_telemetry_time, end_telemetry_time)
-        # grating_db_entry = {'aperture': self.aperture,
-        #                     'read_pattern': "temp",
-        #                     'expstart': "temp",
-        #                     'inrsh_gwa_adcmgain_time': mnemonics['INRSH_GWA_ADCMGAIN'].data['MJD'],
-        #                     'inrsh_gwa_adcmgain': mnemonics['INRSH_GWA_ADCMGAIN'].data['euvalue'],
-        #                     'inrsh_gwa_adcmoffset_time': mnemonics['INRSH_GWA_ADCMOFFSET'].data['MJD'],
-        #                     'inrsh_gwa_adcmoffset': mnemonics['INRSH_GWA_ADCMOFFSET'].data['euvalue'],
-        #                     'inrsh_gwa_motor_vref_time': mnemonics['INRSH_GWA_MOTOR_VREF'].data['MJD'],
-        #                     'inrsh_gwa_motor_vref': mnemonics['INRSH_GWA_MOTOR_VREF'].data['euvalue'],
-        #                     'inrsi_c_gwa_x_position_time': mnemonics['INRSI_C_GWA_X_POSITION'].data['MJD'],
-        #                     'inrsi_c_gwa_x_position': mnemonics['INRSI_C_GWA_X_POSITION'].data['euvalue'],
-        #                     'inrsi_c_gwa_y_position_time': mnemonics['INRSI_C_GWA_Y_POSITION'].data['MJD'],
-        #                     'inrsi_c_gwa_y_position': mnemonics['INRSI_C_GWA_Y_POSITION'].data['euvalue'],
-        #                     'run_monitor': False
-        #                     }
+                if 'POSITION' in telem:
+                    min_distance = np.min(abs(grating_val.data['MJD']-time))
+                    if min_distance > 0.01:  # determine whether this is realistic time difference between data for voltage and grating value
+                        logging.warning("Grating retrieved may not match grating for which voltage is being determined")
+                    closest_location = np.where(abs(grating_val.data['MJD']-time) == min_distance)[0][0]
+                    grating_used = grating_val.data['euvalue'][closest_location]
+                    if grating_used == grating:
+                        try:
+                            grating_db_entry = {'time': time,
+                                                telem.lower(): float(mnemonic.data['euvalue'][np.where(mnemonic.data['MJD'] == time)]),
+                                                'run_monitor': True,  # UPDATE
+                                                'entry_date': datetime.datetime.now()  # need slightly different times to add to database
+                                                }
+                            grating_db_entry.update(other_telems_dict)
+                            # Add this new entry to the grating database table
+                            self.stats_table.__table__.insert().execute(grating_db_entry)
+                            logging.info('\tNew entry added to grating database table: {}'.format(grating_db_entry))
+                        except TypeError:
+                            logging.warning("May be skipping a value with same entry_date")
+                            continue
+                    else:
+                        logging.warning("Not adding entry because data is for {} rather than {}".format(grating_used, grating))
+                        continue
+                else:
+                    try:
+                        grating_db_entry = {'time': time,
+                                            telem.lower(): float(mnemonic.data['euvalue'][np.where(mnemonic.data['MJD'] == time)]),
+                                            'run_monitor': True,  # UPDATE
+                                            'entry_date': datetime.datetime.now()  # need slightly different times to add to database
+                                            }
+                        grating_db_entry.update(other_telems_dict)
+                        # Add this new entry to the grating database table
+                        self.stats_table.__table__.insert().execute(grating_db_entry)
+                        logging.info('\tNew entry added to grating database table: {}'.format(grating_db_entry))
+                    except TypeError:
+                        logging.warning("may be skipping a value with same entry_date.")
+                        continue
 
     @log_fail
     @log_info
@@ -314,73 +188,27 @@ def run(self):
         # Identify which database tables to use
         self.identify_tables()
 
-        # Get a list of all possible full-frame apertures for this instrument
-        siaf = Siaf(self.instrument)
-        possible_apertures = [aperture for aperture in siaf.apertures if siaf[aperture].AperType == 'FULLSCA']
-
-        for aperture in possible_apertures:
-            logging.info('Working on aperture {} in {}'.format(aperture, self.instrument))
-            self.aperture = aperture
-
-            # Locate the record of most recent MAST search; use this time
-            # (plus a 30 day buffer to catch any missing files from
-            # previous run) as the start time in the new MAST search.
-            most_recent_search = self.most_recent_search()
-            self.query_start = most_recent_search - 30
-
-            # Query MAST for new dark files for this instrument/aperture
-            logging.info('\tQuery times: {} {}'.format(self.query_start, self.query_end))
-            new_entries = mast_query_darks(self.instrument, aperture, self.query_start, self.query_end)
-            logging.info('\tAperture: {}, new entries: {}'.format(self.aperture, len(new_entries)))
-
-            # Set up a directory to store the data for this aperture
-            self.data_dir = os.path.join(self.output_dir, 'data/{}_{}'.format(self.instrument.lower(), self.aperture.lower()))
-            if len(new_entries) > 0:
-                ensure_dir_exists(self.data_dir)
-
-            # Get any new files to process
-            new_files = []
-            for file_entry in new_entries:
-                output_filename = os.path.join(self.data_dir, file_entry['filename'])
-                output_filename = output_filename.replace('_uncal.fits', '_uncal_0thgroup.fits').replace('_dark.fits', '_uncal_0thgroup.fits')
-
-                # Dont process files that already exist in the grating stats database
-                file_exists = self.file_exists_in_database(output_filename)
-                if file_exists:
-                    logging.info('\t{} already exists in the grating database table.'.format(output_filename))
-                    continue
-
-                # Save the 0th group image from each new file in the output directory; some dont exist in JWQL filesystem.
-                try:
-                    filename = filesystem_path(file_entry['filename'])
-                    uncal_filename = filename.replace('_dark', '_uncal')
-                    if not os.path.isfile(uncal_filename):
-                        logging.info('\t{} does not exist in JWQL filesystem, even though {} does'.format(uncal_filename, filename))
-                    else:
-                        new_file = self.extract_zeroth_group(uncal_filename)
-                        new_files.append(new_file)
-                except FileNotFoundError:
-                    logging.info('\t{} does not exist in JWQL filesystem'.format(file_entry['filename']))
-            # Run the grating monitor on any new files
-            # NEED TO CHANGE SUCH THAT MONITOR IS UPDATED WHEN NEW TELEMETRY IS AVAILABLE?
-            if len(new_files) == 0:  # WAS >  BUT NEED TO CHANGE!!
-                self.process(new_files)
-                monitor_run = True
-            else:
-                logging.info('\tGrating monitor skipped. {} new dark files for {}, {}.'.format(len(new_files), self.instrument, aperture))
-                monitor_run = False
-
-            # Update the query history
-            new_entry = {'instrument': self.instrument,
-                         'aperture': aperture,
-                         'start_time_mjd': self.query_start,
-                         'end_time_mjd': self.query_end,
-                         'entries_found': len(new_entries),
-                         'files_found': len(new_files),
-                         'run_monitor': monitor_run,
-                         'entry_date': datetime.datetime.now()}
-            self.query_table.__table__.insert().execute(new_entry)
-            logging.info('\tUpdated the query history table')
+        # Locate the record of most recent MAST search; use this time
+        # (plus a 30 day buffer to catch any missing files from
+        # previous run) as the start time in the new MAST search.
+        most_recent_search = self.most_recent_search()
+        self.query_start = most_recent_search  # MAYBE USE 30 DAYS BUFFER BEFORE THAT?
+        logging.info('\tQuery times: {} {}'.format(self.query_start, self.query_end))
+
+        self.process()
+        monitor_run = True
+        # else:  # DETERMINE CASE WHEN DON"T RUN MONITOR-- THERE SHOULD FREQUENTLY BE UPDATED TELEMETRY THOUGH
+        #     logging.info('\tGrating monitor skipped. {} new dark files for {}.'.format(len(new_files), self.instrument))
+        #     monitor_run = False
+
+        # Update the query history
+        new_entry = {'instrument': self.instrument,
+                     'start_time_mjd': self.query_start,
+                     'end_time_mjd': self.query_end,
+                     'run_monitor': monitor_run,
+                     'entry_date': datetime.datetime.now()}
+        self.query_table.__table__.insert().execute(new_entry)
+        logging.info('\tUpdated the query history table')
 
         logging.info('Grating Monitor completed successfully.')
 
diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py
index 7d69bc25a..8af517ac2 100644
--- a/jwql/utils/constants.py
+++ b/jwql/utils/constants.py
@@ -209,13 +209,26 @@
                           }
 
 # Dictionary with keys that are mnemonics used for grating wheel telemetry monitor
-# Corresponding limits are given for Lower Red, Lower Yellow, High Yellow, High Red,
-# Switch Low, and Switch High, respectively
-GRATING_TELEMETRY = {'INRSH_GWA_ADCMGAIN': [2.36, 2.39, 2.62, 2.65, 1.00, 3.00],
-                     'INRSH_GWA_ADCMOFFSET': [-0.0600, -0.0420, 0.0420, 0.0600, 1.00, 3.00],
-                     'INRSH_GWA_MOTOR_VREF': [-0.0600, -0.0420, 2.530, 2.55, 1.00, 3.00],
-                     'INRSI_C_GWA_X_POSITION': [-374, -360., 360., 374, 1.00, 3.00],
-                     'INRSI_C_GWA_Y_POSITION': [-374, -200., 200., 374, 1.00, 3.00]}
+# Corresponding limits are given for Lower Red, Lower Yellow, High Yellow, High Red
+GRATING_TELEMETRY = {'INRSH_GWA_ADCMGAIN': [2.36, 2.39, 2.62, 2.65],
+                     'INRSH_GWA_ADCMOFFSET': [-0.0600, -0.0420, 0.0420, 0.0600],
+                     'INRSH_GWA_MOTOR_VREF': [-0.0600, -0.0420, 2.530, 2.55],
+                     'PRISM_INRSI_C_GWA_X_POSITION': [-374, -360., 360., 374],
+                     'PRISM_INRSI_C_GWA_Y_POSITION': [-374, -200., 200., 374],
+                     'MIRROR_INRSI_C_GWA_X_POSITION': [-374, -360., 360., 374],
+                     'MIRROR_INRSI_C_GWA_Y_POSITION': [-374, -200., 200., 374],
+                     'G140H_INRSI_C_GWA_X_POSITION': [-374, -360., 360., 374],
+                     'G140H_INRSI_C_GWA_Y_POSITION': [-374, -200., 200., 374],
+                     'G235H_INRSI_C_GWA_X_POSITION': [-374, -360., 360., 374],
+                     'G235H_INRSI_C_GWA_Y_POSITION': [-374, -200., 200., 374],
+                     'G395H_INRSI_C_GWA_X_POSITION': [-374, -360., 360., 374],
+                     'G395H_INRSI_C_GWA_Y_POSITION': [-374, -200., 200., 374],
+                     'G140M_INRSI_C_GWA_X_POSITION': [-374, -360., 360., 374],
+                     'G140M_INRSI_C_GWA_Y_POSITION': [-374, -200., 200., 374],
+                     'G235M_INRSI_C_GWA_X_POSITION': [-374, -360., 360., 374],
+                     'G235M_INRSI_C_GWA_Y_POSITION': [-374, -200., 200., 374],
+                     'G395M_INRSI_C_GWA_X_POSITION': [-374, -360., 360., 374],
+                     'G395M_INRSI_C_GWA_Y_POSITION': [-374, -200., 200., 374]}
 
 # Possible suffix types for guider exposures
 GUIDER_SUFFIX_TYPES = ['stream', 'stacked_uncal', 'image_uncal', 'stacked_cal', 'image_cal']
diff --git a/jwql/website/apps/jwql/bokeh_containers.py b/jwql/website/apps/jwql/bokeh_containers.py
index 7bad30f72..cdef7434f 100644
--- a/jwql/website/apps/jwql/bokeh_containers.py
+++ b/jwql/website/apps/jwql/bokeh_containers.py
@@ -309,52 +309,51 @@ def grating_monitor_tabs(instrument):
         The JS script to render grating wheel monitor plots
     """
 
-    # Make a separate tab for each aperture
+    # Make a tab
     tabs = []
-    for aperture in FULL_FRAME_APERTURES[instrument.upper()]:
-        monitor_template = monitor_pages.GratingMonitor()
-        monitor_template.input_parameters = (instrument, aperture)
-
-        # Add the telemetry vs time plots for grating wheel sensor telemetry value
-        plots = []
-        for telemetry in GRATING_TELEMETRY.keys():
-            grating_plot = monitor_template.refs['figure_{}'.format(telemetry)]
-            grating_plot.sizing_mode = 'scale_width'  # Make sure the sizing is adjustable
-
-            # Add colored boxes to plots indicating green, yellow, and red regions
-            # Grating Telemetry dictionary indices are in the following order:
-            # Lower Red, Lower Yellow, High Yellow, High Red, Switch Low, and Switch High
-            red_low_lower_threshold = GRATING_TELEMETRY[telemetry][0] - 100  # arbitrary box height to show red on rest of plot
-            yellow_low_lower_threshold = GRATING_TELEMETRY[telemetry][0]
-            green_lower_threshold = GRATING_TELEMETRY[telemetry][1]
-            green_higher_threshold = GRATING_TELEMETRY[telemetry][2]
-            yellow_high_higher_threshold = GRATING_TELEMETRY[telemetry][3]
-            red_high_higher_threshold = GRATING_TELEMETRY[telemetry][3] + 100  # arbitrary box height to show red on rest of plot
-            yellow_low_higher_threshold = green_lower_threshold
-            yellow_high_lower_threshold = green_higher_threshold
-            red_low_higher_threshold = yellow_low_lower_threshold
-            red_high_lower_threshold = yellow_high_higher_threshold
-
-            green = BoxAnnotation(bottom=green_lower_threshold, top=green_higher_threshold, fill_color='chartreuse', fill_alpha=0.2)
-            grating_plot.add_layout(green)
-            yellow_high = BoxAnnotation(bottom=yellow_high_lower_threshold, top=yellow_high_higher_threshold, fill_color='gold', fill_alpha=0.2)
-            grating_plot.add_layout(yellow_high)
-            yellow_low = BoxAnnotation(bottom=yellow_low_lower_threshold, top=yellow_low_higher_threshold, fill_color='gold', fill_alpha=0.2)
-            grating_plot.add_layout(yellow_low)
-            red_high = BoxAnnotation(bottom=red_high_lower_threshold, top=red_high_higher_threshold, fill_color='red', fill_alpha=0.1)
-            grating_plot.add_layout(red_high)
-            red_low = BoxAnnotation(bottom=red_low_lower_threshold, top=red_low_higher_threshold, fill_color='red', fill_alpha=0.1)
-            grating_plot.add_layout(red_low)
-
-            plots.append(grating_plot)
-
-        # Put grating plots on the top row.
-        grating_layout = layout(plots[0:4])  # MAKE SURE ALL PLOTS ARE INCLUDED HERE
-        grating_layout.sizing_mode = 'scale_width'
-        grating_tab = Panel(child=grating_layout, title=aperture)
-        tabs.append(grating_tab)
-
-    # Build tabs
+    monitor_template = monitor_pages.GratingMonitor()
+    monitor_template.input_parameters = (instrument)
+
+    # Add the telemetry vs time plots for grating wheel sensor telemetry value
+    plots = []
+    for telemetry in GRATING_TELEMETRY.keys():
+        grating_plot = monitor_template.refs['figure_{}'.format(telemetry)]
+        grating_plot.sizing_mode = 'scale_width'  # Make sure the sizing is adjustable
+
+        # Add colored boxes to plots indicating green, yellow, and red regions
+        # Grating Telemetry dictionary indices are in the following order:
+        # Lower Red, Lower Yellow, High Yellow, High Red
+        red_low_lower_threshold = GRATING_TELEMETRY[telemetry][0] - 100  # arbitrary box height to show red on rest of plot
+        yellow_low_lower_threshold = GRATING_TELEMETRY[telemetry][0]
+        green_lower_threshold = GRATING_TELEMETRY[telemetry][1]
+        green_higher_threshold = GRATING_TELEMETRY[telemetry][2]
+        yellow_high_higher_threshold = GRATING_TELEMETRY[telemetry][3]
+        red_high_higher_threshold = GRATING_TELEMETRY[telemetry][3] + 100  # arbitrary box height to show red on rest of plot
+        yellow_low_higher_threshold = green_lower_threshold
+        yellow_high_lower_threshold = green_higher_threshold
+        red_low_higher_threshold = yellow_low_lower_threshold
+        red_high_lower_threshold = yellow_high_higher_threshold
+
+        green = BoxAnnotation(bottom=green_lower_threshold, top=green_higher_threshold, fill_color='chartreuse', fill_alpha=0.2)
+        grating_plot.add_layout(green)
+        yellow_high = BoxAnnotation(bottom=yellow_high_lower_threshold, top=yellow_high_higher_threshold, fill_color='gold', fill_alpha=0.2)
+        grating_plot.add_layout(yellow_high)
+        yellow_low = BoxAnnotation(bottom=yellow_low_lower_threshold, top=yellow_low_higher_threshold, fill_color='gold', fill_alpha=0.2)
+        grating_plot.add_layout(yellow_low)
+        red_high = BoxAnnotation(bottom=red_high_lower_threshold, top=red_high_higher_threshold, fill_color='red', fill_alpha=0.1)
+        grating_plot.add_layout(red_high)
+        red_low = BoxAnnotation(bottom=red_low_lower_threshold, top=red_low_higher_threshold, fill_color='red', fill_alpha=0.1)
+        grating_plot.add_layout(red_low)
+
+        plots.append(grating_plot)
+
+    # Put grating plots on the top row.
+    grating_layout = layout(plots[0:18])
+    grating_layout.sizing_mode = 'scale_width'
+    grating_tab = Panel(child=grating_layout, title="Grating Monitor Plots")
+    tabs.append(grating_tab)
+
+    # # Build tabs
     tabs = Tabs(tabs=tabs)
 
     # Return tab HTML and JavaScript to web app
diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_grating_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_grating_bokeh.py
index cadd4d61b..22328f21f 100644
--- a/jwql/website/apps/jwql/monitor_pages/monitor_grating_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_grating_bokeh.py
@@ -40,11 +40,11 @@ class GratingMonitor(BokehTemplate):
     # do not want to invoke the setter unless all are updated
     @property
     def input_parameters(self):
-        return (self._instrument, self._aperture)
+        return (self._instrument)
 
     @input_parameters.setter
     def input_parameters(self, info):
-        self._instrument, self._aperture = info
+        self._instrument = info
         self.pre_init()
         self.post_init()
 
@@ -60,24 +60,21 @@ def load_data(self):
         # Determine which database tables are needed based on instrument
         self.identify_tables()
 
-        # Query database for all data in grating wheel stats with a matching aperture,
+        # Query database for all data in grating wheel stats,
         # and sort the data by exposure start time.
         # If monitor fails here because relation does not exist, need to reset database
         self.query_results = session.query(self.stats_table) \
-            .filter(self.stats_table.aperture == self._aperture) \
-            .order_by(self.stats_table.expstart) \
+            .order_by(self.stats_table.time) \
             .all()
 
     def pre_init(self):
 
-        # Start with default values for instrument and aperture because
+        # Start with default values for instrument because
         # BokehTemplate's __init__ method does not allow input arguments
         try:
             dummy_instrument = self._instrument
-            dummy_aperture = self._aperture
         except AttributeError:
             self._instrument = 'NIRSpec'
-            self._aperture = ''
 
         self._embed = True
         self.format_string = None
@@ -99,23 +96,23 @@ def update_grating_figures(self):
             gwa_vals = np.array([getattr(result, '{}'.format(telemetry.lower())) for result in self.query_results])
 
             if len(gwa_vals) != 0:  # SHOULD UPDATE WITHOUT IF STATEMENT
-                expstarts_iso = np.array([str(Time(Time(result.expstart, format='mjd'), format='iso')) for result in self.query_results])
-                expstarts = np.array([datetime.strptime(date, '%Y-%m-%d %H:%M:%S.%f') for date in expstarts_iso])
+                dates_iso = np.array([str(Time(Time(result.time, format='mjd'), format='iso')) for result in self.query_results])
+                dates = np.array([datetime.strptime(date, '%Y-%m-%d %H:%M:%S.%f') for date in dates_iso])
 
                 fake_filenames = []
                 for length in range(len(gwa_vals)):
                     fake_filenames.append("file{}".format(length))
-                self.refs['source_{}'.format(telemetry)].data = {'time': expstarts,  # [datetime.now(), datetime.now()-timedelta(days=1), datetime.now()-timedelta(days=2)],
-                                                                 'time_iso': expstarts,   # SHOULD BE EXPSTARTS_ISO  # [datetime.now(), datetime.now()-timedelta(days=1), datetime.now()-timedelta(days=8)],  # not actually used
+                self.refs['source_{}'.format(telemetry)].data = {'time': dates,
+                                                                 'time_iso': dates_iso,
                                                                  'grating': gwa_vals,
                                                                  'filename': fake_filenames}
 
                 self.refs['figure_{}'.format(telemetry)].title.text = '{}'.format(telemetry)
-                self.refs['figure_{}'.format(telemetry)].hover.tooltips = [('time', '@time'),
+                self.refs['figure_{}'.format(telemetry)].hover.tooltips = [('time', '@time_iso'),
                                                                            ('gwa val', '@grating')]
 
             # Update plot limits if data exists
-            gwa_vals_cut = gwa_vals[np.where(gwa_vals!=None)]  # USE ABOVE AS WELL?
+            gwa_vals_cut = gwa_vals[np.where(gwa_vals != None)]  # USE ABOVE AS WELL?
             if len(gwa_vals_cut) != 0:
                 self.refs['gwa_x_{}'.format(telemetry)].start = datetime.now() - timedelta(days=100)
                 self.refs['gwa_x_{}'.format(telemetry)].end = datetime.now()
diff --git a/jwql/website/apps/jwql/monitor_pages/yaml/monitor_grating_interface.yaml b/jwql/website/apps/jwql/monitor_pages/yaml/monitor_grating_interface.yaml
index 18f65b931..1c330211e 100644
--- a/jwql/website/apps/jwql/monitor_pages/yaml/monitor_grating_interface.yaml
+++ b/jwql/website/apps/jwql/monitor_pages/yaml/monitor_grating_interface.yaml
@@ -26,8 +26,8 @@
     y_axis_label: "INRSH_GWA_ADCMGAIN [V]"
     x_range: *gwa_x_INRSH_GWA_ADCMGAIN
     y_range: *gwa_y_INRSH_GWA_ADCMGAIN
-    height: 800
-    width: 800
+    height: 400
+    width: 400
     tools: "hover, wheel_zoom, pan, reset"
     elements:
         - {'kind': 'circle', 'x': 'time', 'y': 'grating', 'size': 6, 'source': *source_INRSH_GWA_ADCMGAIN}
@@ -59,8 +59,8 @@
     y_axis_label: "INRSH_GWA_ADCMOFFSET [V]"
     x_range: *gwa_x_INRSH_GWA_ADCMOFFSET
     y_range: *gwa_y_INRSH_GWA_ADCMOFFSET
-    height: 800
-    width: 800
+    height: 400
+    width: 400
     tools: "hover, wheel_zoom, pan, reset"
     elements:
         - {'kind': 'circle', 'x': 'time', 'y': 'grating', 'size': 6, 'source': *source_INRSH_GWA_ADCMOFFSET}
@@ -92,77 +92,539 @@
     y_axis_label: "INRSH_GWA_MOTOR_VREF [V]"
     x_range: *gwa_x_INRSH_GWA_MOTOR_VREF
     y_range: *gwa_y_INRSH_GWA_MOTOR_VREF
-    height: 800
-    width: 800
+    height: 400
+    width: 400
     tools: "hover, wheel_zoom, pan, reset"
     elements:
         - {'kind': 'circle', 'x': 'time', 'y': 'grating', 'size': 6, 'source': *source_INRSH_GWA_MOTOR_VREF}
 
 # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-# INRSI_C_GWA_X_POSITION vs Time Figure
+# PRISM_INRSI_C_GWA_X_POSITION vs Time Figure
 # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-- !ColumnDataSource: &source_INRSI_C_GWA_X_POSITION
-    ref: "source_INRSI_C_GWA_X_POSITION"
+- !ColumnDataSource: &source_PRISM_INRSI_C_GWA_X_POSITION
+    ref: "source_PRISM_INRSI_C_GWA_X_POSITION"
     data:
         time: []
         time_iso: []
         grating: []
         filename: []
-- !Range1d: &gwa_x_INRSI_C_GWA_X_POSITION
-    ref: "gwa_x_INRSI_C_GWA_X_POSITION"
+- !Range1d: &gwa_x_PRISM_INRSI_C_GWA_X_POSITION
+    ref: "gwa_x_PRISM_INRSI_C_GWA_X_POSITION"
     start: 0
     end: 1
     bounds: 'auto'
-- !Range1d: &gwa_y_INRSI_C_GWA_X_POSITION
-    ref: "gwa_y_INRSI_C_GWA_X_POSITION"
+- !Range1d: &gwa_y_PRISM_INRSI_C_GWA_X_POSITION
+    ref: "gwa_y_PRISM_INRSI_C_GWA_X_POSITION"
     start: 0
     end: 10
     bounds: 'auto'
-- !Figure: &figure_INRSI_C_GWA_X_POSITION
-    ref: "figure_INRSI_C_GWA_X_POSITION"
+- !Figure: &figure_PRISM_INRSI_C_GWA_X_POSITION
+    ref: "figure_PRISM_INRSI_C_GWA_X_POSITION"
+    x_axis_label: "Date"
+    x_axis_type: "datetime"
+    y_axis_label: "PRISM_INRSI_C_GWA_X_POSITION [mV]"
+    x_range: *gwa_x_PRISM_INRSI_C_GWA_X_POSITION
+    y_range: *gwa_y_PRISM_INRSI_C_GWA_X_POSITION
+    height: 400
+    width: 400
+    tools: "hover, wheel_zoom, pan, reset"
+    elements:
+        - {'kind': 'circle', 'x': 'time', 'y': 'grating', 'size': 6, 'source': *source_PRISM_INRSI_C_GWA_X_POSITION}
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# PRISM_INRSI_C_GWA_Y_POSITION vs Time Figure
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+- !ColumnDataSource: &source_PRISM_INRSI_C_GWA_Y_POSITION
+    ref: "source_PRISM_INRSI_C_GWA_Y_POSITION"
+    data:
+        time: []
+        time_iso: []
+        grating: []
+        filename: []
+- !Range1d: &gwa_x_PRISM_INRSI_C_GWA_Y_POSITION
+    ref: "gwa_x_PRISM_INRSI_C_GWA_Y_POSITION"
+    start: 0
+    end: 1
+    bounds: 'auto'
+- !Range1d: &gwa_y_PRISM_INRSI_C_GWA_Y_POSITION
+    ref: "gwa_y_PRISM_INRSI_C_GWA_Y_POSITION"
+    start: 0
+    end: 10
+    bounds: 'auto'
+- !Figure: &figure_PRISM_INRSI_C_GWA_Y_POSITION
+    ref: "figure_PRISM_INRSI_C_GWA_Y_POSITION"
+    x_axis_label: "Date"
+    x_axis_type: "datetime"
+    y_axis_label: "PRISM_INRSI_C_GWA_Y_POSITION [mV]"
+    x_range: *gwa_x_PRISM_INRSI_C_GWA_Y_POSITION
+    y_range: *gwa_y_PRISM_INRSI_C_GWA_Y_POSITION
+    height: 400
+    width: 400
+    tools: "hover, wheel_zoom, pan, reset"
+    elements:
+        - {'kind': 'circle', 'x': 'time', 'y': 'grating', 'size': 6, 'source': *source_PRISM_INRSI_C_GWA_Y_POSITION}
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# MIRROR_INRSI_C_GWA_X_POSITION vs Time Figure
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+- !ColumnDataSource: &source_MIRROR_INRSI_C_GWA_X_POSITION
+    ref: "source_MIRROR_INRSI_C_GWA_X_POSITION"
+    data:
+        time: []
+        time_iso: []
+        grating: []
+        filename: []
+- !Range1d: &gwa_x_MIRROR_INRSI_C_GWA_X_POSITION
+    ref: "gwa_x_MIRROR_INRSI_C_GWA_X_POSITION"
+    start: 0
+    end: 1
+    bounds: 'auto'
+- !Range1d: &gwa_y_MIRROR_INRSI_C_GWA_X_POSITION
+    ref: "gwa_y_MIRROR_INRSI_C_GWA_X_POSITION"
+    start: 0
+    end: 10
+    bounds: 'auto'
+- !Figure: &figure_MIRROR_INRSI_C_GWA_X_POSITION
+    ref: "figure_MIRROR_INRSI_C_GWA_X_POSITION"
     x_axis_label: "Date"
     x_axis_type: "datetime"
     y_axis_label: "INRSI_C_GWA_X_POSITION [mV]"
-    x_range: *gwa_x_INRSI_C_GWA_X_POSITION
-    y_range: *gwa_y_INRSI_C_GWA_X_POSITION
-    height: 800
-    width: 800
+    x_range: *gwa_x_MIRROR_INRSI_C_GWA_X_POSITION
+    y_range: *gwa_y_MIRROR_INRSI_C_GWA_X_POSITION
+    height: 400
+    width: 400
     tools: "hover, wheel_zoom, pan, reset"
     elements:
-        - {'kind': 'circle', 'x': 'time', 'y': 'grating', 'size': 6, 'source': *source_INRSI_C_GWA_X_POSITION}
+        - {'kind': 'circle', 'x': 'time', 'y': 'grating', 'size': 6, 'source': *source_MIRROR_INRSI_C_GWA_X_POSITION}
 
 # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-# INRSI_C_GWA_Y_POSITION vs Time Figure
+# MIRROR_INRSI_C_GWA_Y_POSITION vs Time Figure
 # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-- !ColumnDataSource: &source_INRSI_C_GWA_Y_POSITION
-    ref: "source_INRSI_C_GWA_Y_POSITION"
+- !ColumnDataSource: &source_MIRROR_INRSI_C_GWA_Y_POSITION
+    ref: "source_MIRROR_INRSI_C_GWA_Y_POSITION"
     data:
         time: []
         time_iso: []
         grating: []
         filename: []
-- !Range1d: &gwa_x_INRSI_C_GWA_Y_POSITION
-    ref: "gwa_x_INRSI_C_GWA_Y_POSITION"
+- !Range1d: &gwa_x_MIRROR_INRSI_C_GWA_Y_POSITION
+    ref: "gwa_x_MIRROR_INRSI_C_GWA_Y_POSITION"
     start: 0
     end: 1
     bounds: 'auto'
-- !Range1d: &gwa_y_INRSI_C_GWA_Y_POSITION
-    ref: "gwa_y_INRSI_C_GWA_Y_POSITION"
+- !Range1d: &gwa_y_MIRROR_INRSI_C_GWA_Y_POSITION
+    ref: "gwa_y_MIRROR_INRSI_C_GWA_Y_POSITION"
     start: 0
     end: 10
     bounds: 'auto'
-- !Figure: &figure_INRSI_C_GWA_Y_POSITION
-    ref: "figure_INRSI_C_GWA_Y_POSITION"
+- !Figure: &figure_MIRROR_INRSI_C_GWA_Y_POSITION
+    ref: "figure_MIRROR_INRSI_C_GWA_Y_POSITION"
     x_axis_label: "Date"
     x_axis_type: "datetime"
     y_axis_label: "INRSI_C_GWA_Y_POSITION [mV]"
-    x_range: *gwa_x_INRSI_C_GWA_Y_POSITION
-    y_range: *gwa_y_INRSI_C_GWA_Y_POSITION
-    height: 800
-    width: 800
+    x_range: *gwa_x_MIRROR_INRSI_C_GWA_Y_POSITION
+    y_range: *gwa_y_MIRROR_INRSI_C_GWA_Y_POSITION
+    height: 400
+    width: 400
+    tools: "hover, wheel_zoom, pan, reset"
+    elements:
+        - {'kind': 'circle', 'x': 'time', 'y': 'grating', 'size': 6, 'source': *source_MIRROR_INRSI_C_GWA_Y_POSITION}
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# G140H_INRSI_C_GWA_X_POSITION vs Time Figure
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+- !ColumnDataSource: &source_G140H_INRSI_C_GWA_X_POSITION
+    ref: "source_G140H_INRSI_C_GWA_X_POSITION"
+    data:
+        time: []
+        time_iso: []
+        grating: []
+        filename: []
+- !Range1d: &gwa_x_G140H_INRSI_C_GWA_X_POSITION
+    ref: "gwa_x_G140H_INRSI_C_GWA_X_POSITION"
+    start: 0
+    end: 1
+    bounds: 'auto'
+- !Range1d: &gwa_y_G140H_INRSI_C_GWA_X_POSITION
+    ref: "gwa_y_G140H_INRSI_C_GWA_X_POSITION"
+    start: 0
+    end: 10
+    bounds: 'auto'
+- !Figure: &figure_G140H_INRSI_C_GWA_X_POSITION
+    ref: "figure_G140H_INRSI_C_GWA_X_POSITION"
+    x_axis_label: "Date"
+    x_axis_type: "datetime"
+    y_axis_label: "G140H_INRSI_C_GWA_X_POSITION [mV]"
+    x_range: *gwa_x_G140H_INRSI_C_GWA_X_POSITION
+    y_range: *gwa_y_G140H_INRSI_C_GWA_X_POSITION
+    height: 400
+    width: 400
+    tools: "hover, wheel_zoom, pan, reset"
+    elements:
+        - {'kind': 'circle', 'x': 'time', 'y': 'grating', 'size': 6, 'source': *source_G140H_INRSI_C_GWA_X_POSITION}
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# G140H_INRSI_C_GWA_Y_POSITION vs Time Figure
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+- !ColumnDataSource: &source_G140H_INRSI_C_GWA_Y_POSITION
+    ref: "source_G140H_INRSI_C_GWA_Y_POSITION"
+    data:
+        time: []
+        time_iso: []
+        grating: []
+        filename: []
+- !Range1d: &gwa_x_G140H_INRSI_C_GWA_Y_POSITION
+    ref: "gwa_x_G140H_INRSI_C_GWA_Y_POSITION"
+    start: 0
+    end: 1
+    bounds: 'auto'
+- !Range1d: &gwa_y_G140H_INRSI_C_GWA_Y_POSITION
+    ref: "gwa_y_G140H_INRSI_C_GWA_Y_POSITION"
+    start: 0
+    end: 10
+    bounds: 'auto'
+- !Figure: &figure_G140H_INRSI_C_GWA_Y_POSITION
+    ref: "figure_G140H_INRSI_C_GWA_Y_POSITION"
+    x_axis_label: "Date"
+    x_axis_type: "datetime"
+    y_axis_label: "G140H_INRSI_C_GWA_Y_POSITION [mV]"
+    x_range: *gwa_x_G140H_INRSI_C_GWA_Y_POSITION
+    y_range: *gwa_y_G140H_INRSI_C_GWA_Y_POSITION
+    height: 400
+    width: 400
+    tools: "hover, wheel_zoom, pan, reset"
+    elements:
+        - {'kind': 'circle', 'x': 'time', 'y': 'grating', 'size': 6, 'source': *source_G140H_INRSI_C_GWA_Y_POSITION}
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# G235H_INRSI_C_GWA_X_POSITION vs Time Figure
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+- !ColumnDataSource: &source_G235H_INRSI_C_GWA_X_POSITION
+    ref: "source_G235H_INRSI_C_GWA_X_POSITION"
+    data:
+        time: []
+        time_iso: []
+        grating: []
+        filename: []
+- !Range1d: &gwa_x_G235H_INRSI_C_GWA_X_POSITION
+    ref: "gwa_x_G235H_INRSI_C_GWA_X_POSITION"
+    start: 0
+    end: 1
+    bounds: 'auto'
+- !Range1d: &gwa_y_G235H_INRSI_C_GWA_X_POSITION
+    ref: "gwa_y_G235H_INRSI_C_GWA_X_POSITION"
+    start: 0
+    end: 10
+    bounds: 'auto'
+- !Figure: &figure_G235H_INRSI_C_GWA_X_POSITION
+    ref: "figure_G235H_INRSI_C_GWA_X_POSITION"
+    x_axis_label: "Date"
+    x_axis_type: "datetime"
+    y_axis_label: "G235H_INRSI_C_GWA_X_POSITION [mV]"
+    x_range: *gwa_x_G235H_INRSI_C_GWA_X_POSITION
+    y_range: *gwa_y_G235H_INRSI_C_GWA_X_POSITION
+    height: 400
+    width: 400
+    tools: "hover, wheel_zoom, pan, reset"
+    elements:
+        - {'kind': 'circle', 'x': 'time', 'y': 'grating', 'size': 6, 'source': *source_G235H_INRSI_C_GWA_X_POSITION}
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# G235H_INRSI_C_GWA_Y_POSITION vs Time Figure
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+- !ColumnDataSource: &source_G235H_INRSI_C_GWA_Y_POSITION
+    ref: "source_G235H_INRSI_C_GWA_Y_POSITION"
+    data:
+        time: []
+        time_iso: []
+        grating: []
+        filename: []
+- !Range1d: &gwa_x_G235H_INRSI_C_GWA_Y_POSITION
+    ref: "gwa_x_G235H_INRSI_C_GWA_Y_POSITION"
+    start: 0
+    end: 1
+    bounds: 'auto'
+- !Range1d: &gwa_y_G235H_INRSI_C_GWA_Y_POSITION
+    ref: "gwa_y_G235H_INRSI_C_GWA_Y_POSITION"
+    start: 0
+    end: 10
+    bounds: 'auto'
+- !Figure: &figure_G235H_INRSI_C_GWA_Y_POSITION
+    ref: "figure_G235H_INRSI_C_GWA_Y_POSITION"
+    x_axis_label: "Date"
+    x_axis_type: "datetime"
+    y_axis_label: "G235H_INRSI_C_GWA_Y_POSITION [mV]"
+    x_range: *gwa_x_G235H_INRSI_C_GWA_Y_POSITION
+    y_range: *gwa_y_G235H_INRSI_C_GWA_Y_POSITION
+    height: 400
+    width: 400
+    tools: "hover, wheel_zoom, pan, reset"
+    elements:
+        - {'kind': 'circle', 'x': 'time', 'y': 'grating', 'size': 6, 'source': *source_G235H_INRSI_C_GWA_Y_POSITION}
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# G395H_INRSI_C_GWA_X_POSITION vs Time Figure
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+- !ColumnDataSource: &source_G395H_INRSI_C_GWA_X_POSITION
+    ref: "source_G395H_INRSI_C_GWA_X_POSITION"
+    data:
+        time: []
+        time_iso: []
+        grating: []
+        filename: []
+- !Range1d: &gwa_x_G395H_INRSI_C_GWA_X_POSITION
+    ref: "gwa_x_G395H_INRSI_C_GWA_X_POSITION"
+    start: 0
+    end: 1
+    bounds: 'auto'
+- !Range1d: &gwa_y_G395H_INRSI_C_GWA_X_POSITION
+    ref: "gwa_y_G395H_INRSI_C_GWA_X_POSITION"
+    start: 0
+    end: 10
+    bounds: 'auto'
+- !Figure: &figure_G395H_INRSI_C_GWA_X_POSITION
+    ref: "figure_G395H_INRSI_C_GWA_X_POSITION"
+    x_axis_label: "Date"
+    x_axis_type: "datetime"
+    y_axis_label: "G395H_INRSI_C_GWA_X_POSITION [mV]"
+    x_range: *gwa_x_G395H_INRSI_C_GWA_X_POSITION
+    y_range: *gwa_y_G395H_INRSI_C_GWA_X_POSITION
+    height: 400
+    width: 400
+    tools: "hover, wheel_zoom, pan, reset"
+    elements:
+        - {'kind': 'circle', 'x': 'time', 'y': 'grating', 'size': 6, 'source': *source_G395H_INRSI_C_GWA_X_POSITION}
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# G395H_INRSI_C_GWA_Y_POSITION vs Time Figure
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+- !ColumnDataSource: &source_G395H_INRSI_C_GWA_Y_POSITION
+    ref: "source_G395H_INRSI_C_GWA_Y_POSITION"
+    data:
+        time: []
+        time_iso: []
+        grating: []
+        filename: []
+- !Range1d: &gwa_x_G395H_INRSI_C_GWA_Y_POSITION
+    ref: "gwa_x_G395H_INRSI_C_GWA_Y_POSITION"
+    start: 0
+    end: 1
+    bounds: 'auto'
+- !Range1d: &gwa_y_G395H_INRSI_C_GWA_Y_POSITION
+    ref: "gwa_y_G395H_INRSI_C_GWA_Y_POSITION"
+    start: 0
+    end: 10
+    bounds: 'auto'
+- !Figure: &figure_INRSI_C_GWA_Y_POSITION
+    ref: "figure_G395H_INRSI_C_GWA_Y_POSITION"
+    x_axis_label: "Date"
+    x_axis_type: "datetime"
+    y_axis_label: "G395H_INRSI_C_GWA_Y_POSITION [mV]"
+    x_range: *gwa_x_G395H_INRSI_C_GWA_Y_POSITION
+    y_range: *gwa_y_G395H_INRSI_C_GWA_Y_POSITION
+    height: 400
+    width: 400
+    tools: "hover, wheel_zoom, pan, reset"
+    elements:
+        - {'kind': 'circle', 'x': 'time', 'y': 'grating', 'size': 6, 'source': *source_G395H_INRSI_C_GWA_Y_POSITION}
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# G140M_INRSI_C_GWA_X_POSITION vs Time Figure
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+- !ColumnDataSource: &source_G140M_INRSI_C_GWA_X_POSITION
+    ref: "source_G140M_INRSI_C_GWA_X_POSITION"
+    data:
+        time: []
+        time_iso: []
+        grating: []
+        filename: []
+- !Range1d: &gwa_x_G140M_INRSI_C_GWA_X_POSITION
+    ref: "gwa_x_G140M_INRSI_C_GWA_X_POSITION"
+    start: 0
+    end: 1
+    bounds: 'auto'
+- !Range1d: &gwa_y_G140M_INRSI_C_GWA_X_POSITION
+    ref: "gwa_y_G140M_INRSI_C_GWA_X_POSITION"
+    start: 0
+    end: 10
+    bounds: 'auto'
+- !Figure: &figure_G140M_INRSI_C_GWA_X_POSITION
+    ref: "figure_G140M_INRSI_C_GWA_X_POSITION"
+    x_axis_label: "Date"
+    x_axis_type: "datetime"
+    y_axis_label: "G140M_INRSI_C_GWA_X_POSITION [mV]"
+    x_range: *gwa_x_G140M_INRSI_C_GWA_X_POSITION
+    y_range: *gwa_y_G140M_INRSI_C_GWA_X_POSITION
+    height: 400
+    width: 400
+    tools: "hover, wheel_zoom, pan, reset"
+    elements:
+        - {'kind': 'circle', 'x': 'time', 'y': 'grating', 'size': 6, 'source': *source_G140M_INRSI_C_GWA_X_POSITION}
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# G140M_INRSI_C_GWA_Y_POSITION vs Time Figure
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+- !ColumnDataSource: &source_G140M_INRSI_C_GWA_Y_POSITION
+    ref: "source_G140M_INRSI_C_GWA_Y_POSITION"
+    data:
+        time: []
+        time_iso: []
+        grating: []
+        filename: []
+- !Range1d: &gwa_x_G140M_INRSI_C_GWA_Y_POSITION
+    ref: "gwa_x_G140M_INRSI_C_GWA_Y_POSITION"
+    start: 0
+    end: 1
+    bounds: 'auto'
+- !Range1d: &gwa_y_G140M_INRSI_C_GWA_Y_POSITION
+    ref: "gwa_y_G140M_INRSI_C_GWA_Y_POSITION"
+    start: 0
+    end: 10
+    bounds: 'auto'
+- !Figure: &figure_G140M_INRSI_C_GWA_Y_POSITION
+    ref: "figure_G140M_INRSI_C_GWA_Y_POSITION"
+    x_axis_label: "Date"
+    x_axis_type: "datetime"
+    y_axis_label: "G140M_INRSI_C_GWA_Y_POSITION [mV]"
+    x_range: *gwa_x_G140M_INRSI_C_GWA_Y_POSITION
+    y_range: *gwa_y_G140M_INRSI_C_GWA_Y_POSITION
+    height: 400
+    width: 400
+    tools: "hover, wheel_zoom, pan, reset"
+    elements:
+        - {'kind': 'circle', 'x': 'time', 'y': 'grating', 'size': 6, 'source': *source_G140M_INRSI_C_GWA_Y_POSITION}
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# G235M_INRSI_C_GWA_X_POSITION vs Time Figure
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+- !ColumnDataSource: &source_G235M_INRSI_C_GWA_X_POSITION
+    ref: "source_G235M_INRSI_C_GWA_X_POSITION"
+    data:
+        time: []
+        time_iso: []
+        grating: []
+        filename: []
+- !Range1d: &gwa_x_G235M_INRSI_C_GWA_X_POSITION
+    ref: "gwa_x_G235M_INRSI_C_GWA_X_POSITION"
+    start: 0
+    end: 1
+    bounds: 'auto'
+- !Range1d: &gwa_y_G235M_INRSI_C_GWA_X_POSITION
+    ref: "gwa_y_G235M_INRSI_C_GWA_X_POSITION"
+    start: 0
+    end: 10
+    bounds: 'auto'
+- !Figure: &figure_G235M_INRSI_C_GWA_X_POSITION
+    ref: "figure_G235M_INRSI_C_GWA_X_POSITION"
+    x_axis_label: "Date"
+    x_axis_type: "datetime"
+    y_axis_label: "G235M_INRSI_C_GWA_X_POSITION [mV]"
+    x_range: *gwa_x_G235M_INRSI_C_GWA_X_POSITION
+    y_range: *gwa_y_G235M_INRSI_C_GWA_X_POSITION
+    height: 400
+    width: 400
+    tools: "hover, wheel_zoom, pan, reset"
+    elements:
+        - {'kind': 'circle', 'x': 'time', 'y': 'grating', 'size': 6, 'source': *source_G235M_INRSI_C_GWA_X_POSITION}
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# G235M_INRSI_C_GWA_Y_POSITION vs Time Figure
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+- !ColumnDataSource: &source_G235M_INRSI_C_GWA_Y_POSITION
+    ref: "source_G235M_INRSI_C_GWA_Y_POSITION"
+    data:
+        time: []
+        time_iso: []
+        grating: []
+        filename: []
+- !Range1d: &gwa_x_G235M_INRSI_C_GWA_Y_POSITION
+    ref: "gwa_x_G235M_INRSI_C_GWA_Y_POSITION"
+    start: 0
+    end: 1
+    bounds: 'auto'
+- !Range1d: &gwa_y_G235M_INRSI_C_GWA_Y_POSITION
+    ref: "gwa_y_G235M_INRSI_C_GWA_Y_POSITION"
+    start: 0
+    end: 10
+    bounds: 'auto'
+- !Figure: &figure_G235M_INRSI_C_GWA_Y_POSITION
+    ref: "figure_G235M_INRSI_C_GWA_Y_POSITION"
+    x_axis_label: "Date"
+    x_axis_type: "datetime"
+    y_axis_label: "G235M_INRSI_C_GWA_Y_POSITION [mV]"
+    x_range: *gwa_x_G235M_INRSI_C_GWA_Y_POSITION
+    y_range: *gwa_y_G235M_INRSI_C_GWA_Y_POSITION
+    height: 400
+    width: 400
+    tools: "hover, wheel_zoom, pan, reset"
+    elements:
+        - {'kind': 'circle', 'x': 'time', 'y': 'grating', 'size': 6, 'source': *source_G235M_INRSI_C_GWA_Y_POSITION}
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# G395M_INRSI_C_GWA_X_POSITION vs Time Figure
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+- !ColumnDataSource: &source_G395M_INRSI_C_GWA_X_POSITION
+    ref: "source_G395M_INRSI_C_GWA_X_POSITION"
+    data:
+        time: []
+        time_iso: []
+        grating: []
+        filename: []
+- !Range1d: &gwa_x_G395M_INRSI_C_GWA_X_POSITION
+    ref: "gwa_x_G395M_INRSI_C_GWA_X_POSITION"
+    start: 0
+    end: 1
+    bounds: 'auto'
+- !Range1d: &gwa_y_G395M_INRSI_C_GWA_X_POSITION
+    ref: "gwa_y_G395M_INRSI_C_GWA_X_POSITION"
+    start: 0
+    end: 10
+    bounds: 'auto'
+- !Figure: &figure_G395M_INRSI_C_GWA_X_POSITION
+    ref: "figure_G395M_INRSI_C_GWA_X_POSITION"
+    x_axis_label: "Date"
+    x_axis_type: "datetime"
+    y_axis_label: "G395M_INRSI_C_GWA_X_POSITION [mV]"
+    x_range: *gwa_x_G395M_INRSI_C_GWA_X_POSITION
+    y_range: *gwa_y_G395M_INRSI_C_GWA_X_POSITION
+    height: 400
+    width: 400
+    tools: "hover, wheel_zoom, pan, reset"
+    elements:
+        - {'kind': 'circle', 'x': 'time', 'y': 'grating', 'size': 6, 'source': *source_G395M_INRSI_C_GWA_X_POSITION}
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# G395M_INRSI_C_GWA_Y_POSITION vs Time Figure
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+- !ColumnDataSource: &source_G395M_INRSI_C_GWA_Y_POSITION
+    ref: "source_G395M_INRSI_C_GWA_Y_POSITION"
+    data:
+        time: []
+        time_iso: []
+        grating: []
+        filename: []
+- !Range1d: &gwa_x_G395M_INRSI_C_GWA_Y_POSITION
+    ref: "gwa_x_G395M_INRSI_C_GWA_Y_POSITION"
+    start: 0
+    end: 1
+    bounds: 'auto'
+- !Range1d: &gwa_y_G395M_INRSI_C_GWA_Y_POSITION
+    ref: "gwa_y_G395M_INRSI_C_GWA_Y_POSITION"
+    start: 0
+    end: 10
+    bounds: 'auto'
+- !Figure: &figure_G395M_INRSI_C_GWA_Y_POSITION
+    ref: "figure_G395M_INRSI_C_GWA_Y_POSITION"
+    x_axis_label: "Date"
+    x_axis_type: "datetime"
+    y_axis_label: "G395M_INRSI_C_GWA_Y_POSITION [mV]"
+    x_range: *gwa_x_G395M_INRSI_C_GWA_Y_POSITION
+    y_range: *gwa_y_G395M_INRSI_C_GWA_Y_POSITION
+    height: 400
+    width: 400
     tools: "hover, wheel_zoom, pan, reset"
     elements:
-        - {'kind': 'circle', 'x': 'time', 'y': 'grating', 'size': 6, 'source': *source_INRSI_C_GWA_Y_POSITION}
+        - {'kind': 'circle', 'x': 'time', 'y': 'grating', 'size': 6, 'source': *source_G395M_INRSI_C_GWA_Y_POSITION}
 
 # Document structure
 # - !Document:

From 6a7d61c4b176d0dfbf97a18ff7c2995185a052ac Mon Sep 17 00:00:00 2001
From: Teagan King <55458851+tnking97@users.noreply.github.com>
Date: Mon, 15 Nov 2021 13:36:02 -0500
Subject: [PATCH 11/18] included preliminary tests for grating monitor

---
 .../common_monitors/grating_monitor.py        | 92 +++++++++----------
 jwql/tests/test_grating_monitor.py            | 69 +++++++++-----
 2 files changed, 91 insertions(+), 70 deletions(-)

diff --git a/jwql/instrument_monitors/common_monitors/grating_monitor.py b/jwql/instrument_monitors/common_monitors/grating_monitor.py
index 0bfebd7ba..38865c83d 100755
--- a/jwql/instrument_monitors/common_monitors/grating_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/grating_monitor.py
@@ -38,26 +38,19 @@
 class Grating():
     """Class for executing the grating wheel monitor.
 
-    This class will search for a particular type of file in the file
-    system for each instrument and will run the monitor on these files.
-    The monitor will output the contents into a new file located in a
-    working directory. It will then perform statistical measurements
-    on these files in order to monitor the grating wheel telemetry
-    over time. Results are all saved to database tables.
+    This class will search for a particular mnemonic for each
+    instrument and will run the monitor on these files. The monitor 
+    will perform statistical measurements and plot the mnemonic data
+    in order to monitor the grating wheel telemetry over time.
+    Results are all saved to database tables.
 
     Attributes
     ----------
-    output_dir : str
-        Path into which outputs will be placed.
-
-    data_dir : str
-        Path into which new dark files will be copied to be worked on.
-
     query_start : float
-        MJD start date to use for querying MAST.
+        MJD start date to use for querying EDB.
 
     query_end : float
-        MJD end date to use for querying MAST.
+        MJD end date to use for querying EDB.
 
     instrument : str
         Name of instrument used to collect the data.
@@ -96,18 +89,21 @@ def most_recent_search(self):
         return query_result
 
     def process(self):
-        """The main method for processing darks.  See module docstrings
+        """The main method for processing telemetry.  See module docstrings
         for further details.
 
         """
 
-        start_telemetry_time = Time(Time(self.query_start, format='mjd'), format='decimalyear')
+        start_telemetry_time = Time(Time(self.query_start, format='mjd'), format='decimalyear')-10  # REMOVE 100!
         end_telemetry_time = Time(Time(self.query_end, format='mjd'), format='decimalyear')
+        # end_telemetry_time = Time(Time(Time.now(), format='mjd'), format='decimalyear')
         # grating_val gets bogged down if more than about a day
 
         # Construct new entry for this file for the grating wheel database table.
         retrieved_grat = False
         for telem in GRATING_TELEMETRY.keys():
+            # if telem in ['INRSH_GWA_ADCMGAIN', 'INRSH_GWA_ADCMOFFSET', 'INRSH_GWA_MOTOR_VREF']:
+            #     continue
             if 'POSITION' in telem:
                 grating = telem.split('_')[0]
                 telemetry = telem.replace(grating+"_", "")
@@ -118,26 +114,47 @@ def process(self):
                         grating_val = mnemonics['INRSI_GWA_MECH_POS']
                         retrieved_grat = True
                     except TypeError:
-                        print("type error because data was empty")
+                        type_error = True
+                        print("type error because data was empty; try using earlier start date if you want this to run. Data should be present already.")
+                        # FIGURE OUT HOW TO DEAL WITH THIS ERROR
             else:
                 try:
                     mnemonic = get_mnemonic(telem, start_telemetry_time, end_telemetry_time)
                 except TypeError:
-                    print("type error because data was empty")
+                    type_error = True
+                    print("type error because data was empty 2")
                     continue
             other_telems = GRATING_TELEMETRY.keys()
             other_telems_dict = {}
             for telems in other_telems:
                 other_telems_dict[telems.lower()] = None
             other_telems_dict.pop(telem.lower())
-            for time in mnemonic.data['MJD']:
-                if 'POSITION' in telem:
-                    min_distance = np.min(abs(grating_val.data['MJD']-time))
-                    if min_distance > 0.01:  # determine whether this is realistic time difference between data for voltage and grating value
-                        logging.warning("Grating retrieved may not match grating for which voltage is being determined")
-                    closest_location = np.where(abs(grating_val.data['MJD']-time) == min_distance)[0][0]
-                    grating_used = grating_val.data['euvalue'][closest_location]
-                    if grating_used == grating:
+            if type_error is False:
+                for time in mnemonic.data['MJD']:
+                    if 'POSITION' in telem:
+                        min_distance = np.min(abs(grating_val.data['MJD']-time))
+                        if min_distance > 0.01:  # determine whether this is realistic time difference between data for voltage and grating value
+                            logging.warning("Grating retrieved may not match grating for which voltage is being determined")
+                        closest_location = np.where(abs(grating_val.data['MJD']-time) == min_distance)[0][0]
+                        grating_used = grating_val.data['euvalue'][closest_location]
+                        if grating_used == grating:
+                            try:
+                                grating_db_entry = {'time': time,
+                                                    telem.lower(): float(mnemonic.data['euvalue'][np.where(mnemonic.data['MJD'] == time)]),
+                                                    'run_monitor': True,  # UPDATE
+                                                    'entry_date': datetime.datetime.now()  # need slightly different times to add to database
+                                                    }
+                                grating_db_entry.update(other_telems_dict)
+                                # Add this new entry to the grating database table
+                                self.stats_table.__table__.insert().execute(grating_db_entry)
+                                logging.info('\tNew entry added to grating database table: {}'.format(grating_db_entry))
+                            except TypeError:
+                                logging.warning("May be skipping a value with same entry_date")
+                                continue
+                        else:
+                            logging.warning("Not adding entry because data is for {} rather than {}".format(grating_used, grating))
+                            continue
+                    else:
                         try:
                             grating_db_entry = {'time': time,
                                                 telem.lower(): float(mnemonic.data['euvalue'][np.where(mnemonic.data['MJD'] == time)]),
@@ -149,25 +166,8 @@ def process(self):
                             self.stats_table.__table__.insert().execute(grating_db_entry)
                             logging.info('\tNew entry added to grating database table: {}'.format(grating_db_entry))
                         except TypeError:
-                            logging.warning("May be skipping a value with same entry_date")
+                            logging.warning("may be skipping a value with same entry_date.")
                             continue
-                    else:
-                        logging.warning("Not adding entry because data is for {} rather than {}".format(grating_used, grating))
-                        continue
-                else:
-                    try:
-                        grating_db_entry = {'time': time,
-                                            telem.lower(): float(mnemonic.data['euvalue'][np.where(mnemonic.data['MJD'] == time)]),
-                                            'run_monitor': True,  # UPDATE
-                                            'entry_date': datetime.datetime.now()  # need slightly different times to add to database
-                                            }
-                        grating_db_entry.update(other_telems_dict)
-                        # Add this new entry to the grating database table
-                        self.stats_table.__table__.insert().execute(grating_db_entry)
-                        logging.info('\tNew entry added to grating database table: {}'.format(grating_db_entry))
-                    except TypeError:
-                        logging.warning("may be skipping a value with same entry_date.")
-                        continue
 
     @log_fail
     @log_info
@@ -176,10 +176,6 @@ def run(self):
 
         logging.info('Begin logging for grating_monitor')
 
-        # Get the output directory and setup a directory to store the data
-        self.output_dir = os.path.join(get_config()['outputs'], 'grating_monitor')
-        ensure_dir_exists(os.path.join(self.output_dir, 'data'))
-
         # Use the current time as the end time for MAST query
         self.query_end = Time.now().mjd
 
diff --git a/jwql/tests/test_grating_monitor.py b/jwql/tests/test_grating_monitor.py
index 8a3a94591..0ed35e4b5 100644
--- a/jwql/tests/test_grating_monitor.py
+++ b/jwql/tests/test_grating_monitor.py
@@ -17,34 +17,59 @@
         pytest -s test_grating_monitor.py
 """
 
+from astropy.time import Time
 import os
 import pytest
-import shutil
-
-from astropy.io import fits
-import numpy as np
 
+from jwql.edb.engineering_database import get_mnemonics
 from jwql.database.database_interface import NIRSpecGratingQueryHistory, NIRSpecGratingStats
 from jwql.instrument_monitors.common_monitors import grating_monitor
-from jwql.utils.utils import get_config
+from jwql.utils.constants import GRATING_TELEMETRY
+from sqlalchemy.sql.sqltypes import TIME
 
 ON_GITHUB_ACTIONS = '/home/runner' in os.path.expanduser('~') or '/Users/runner' in os.path.expanduser('~')
 
 
-# @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.')
-# def test_query_mast():
-#     """Test the ``query_mast`` function
-#     NEED TO UPDATE
-#     """
-
-#     gw = grating_monitor()
-#     _, _, _, aperture = define_test_data(1)
-
-#     gw.aperture = aperture
-#     gw.instrument = 'nirspec'
-#     gw.query_start = 57357.0
-#     gw.query_end = 57405.0
-
-#     result = gw.query_mast()
-
-#     assert len(result) == 5
+def test_identify_tables():
+    """Be sure the correct database tables are identified
+    """
+    grating = grating_monitor.Grating()
+    grating.instrument = 'nirspec'
+    grating.identify_tables()
+    assert grating.query_table == eval('NIRSpecGratingQueryHistory')
+    assert grating.pixel_table == eval('NIRSpecGratingStats')
+
+
+def test_mnemonics_retrieval():
+    start_telemetry_time = Time(Time(59013, format='mjd'), format='decimalyear')
+    end_telemetry_time = Time(Time(55010, format='mjd'), format='decimalyear')
+    # THIS IS A LONG TIME FRAME...
+    mnemonics_list = ['INRSI_GWA_MECH_POS', 'INRSH_GWA_ADCMGAIN', 'INRSH_GWA_ADCMOFFSET', 'INRSH_GWA_MOTOR_VREF', 'INRSI_C_GWA_X_POSITION', 'INRSI_C_GWA_Y_POSITION']
+    mnemonics = get_mnemonics(mnemonics_list, start_telemetry_time, end_telemetry_time)
+
+    grating_val = mnemonics['INRSI_GWA_MECH_POS'].data[0]['euvalue']
+    adcmgain = mnemonics['INRSH_GWA_ADCMGAIN'].data[0]['euvalue']
+    adcmoffset = mnemonics['INRSH_GWA_ADCMOFFSET'].data[0]['euvalue']
+    motor_vref = mnemonics['INRSH_GWA_MOTOR_VREF'].data[0]['euvalue']
+    x_pos = mnemonics['INRSI_C_GWA_X_POSITION'].data[0]['euvalue']
+    y_pos = mnemonics['INRSI_C_GWA_Y_POSITION'].data[0]['euvalue']
+
+    assert grating_val == 'MIRROR'
+    assert adcmgain == 2.5001527415
+    assert adcmoffset == -0.0010681153
+    assert motor_vref == -0.0009155273999999999
+    assert x_pos == 173.1699688
+    assert y_pos == 98.6046408
+
+
+# def test_process():
+#     grating = grating_monitor.Grating()
+#     grating.query_start = SOME_TIME
+#     grating.query_end = SOME_TIME
+#     # dont' want to actually insert data into database table.. stats_table define as a test table? or
+#     # or set a test_value == True if in testing?
+#     # or separate part of this into another function and just retrieve the db_entry?
+
+# Test that a flag is raised if any plots have points in yellow/red zone (within a certain time frame)
+
+# Check input thresholds make sense, eg. time frame and limits? these are set in constants though

From 60b8f135bc018d9d24f50ab5888cd78d0eb8269e Mon Sep 17 00:00:00 2001
From: Teagan King <55458851+tnking97@users.noreply.github.com>
Date: Mon, 15 Nov 2021 14:17:00 -0500
Subject: [PATCH 12/18] minor fix

---
 jwql/instrument_monitors/common_monitors/grating_monitor.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jwql/instrument_monitors/common_monitors/grating_monitor.py b/jwql/instrument_monitors/common_monitors/grating_monitor.py
index 38865c83d..184a5c0c7 100755
--- a/jwql/instrument_monitors/common_monitors/grating_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/grating_monitor.py
@@ -94,7 +94,7 @@ def process(self):
 
         """
 
-        start_telemetry_time = Time(Time(self.query_start, format='mjd'), format='decimalyear')-10  # REMOVE 100!
+        start_telemetry_time = Time(Time(self.query_start, format='mjd'), format='decimalyear')
         end_telemetry_time = Time(Time(self.query_end, format='mjd'), format='decimalyear')
         # end_telemetry_time = Time(Time(Time.now(), format='mjd'), format='decimalyear')
         # grating_val gets bogged down if more than about a day

From cbd21f654079d348d34353b465e909c7117d3857 Mon Sep 17 00:00:00 2001
From: Teagan King <55458851+tnking97@users.noreply.github.com>
Date: Mon, 22 Nov 2021 17:15:46 -0500
Subject: [PATCH 13/18] adjusted grating wheel telemetry date retrieval

---
 .../common_monitors/grating_monitor.py        | 61 +++++++++++--------
 1 file changed, 37 insertions(+), 24 deletions(-)

diff --git a/jwql/instrument_monitors/common_monitors/grating_monitor.py b/jwql/instrument_monitors/common_monitors/grating_monitor.py
index 184a5c0c7..c77b4b5e0 100755
--- a/jwql/instrument_monitors/common_monitors/grating_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/grating_monitor.py
@@ -96,47 +96,58 @@ def process(self):
 
         start_telemetry_time = Time(Time(self.query_start, format='mjd'), format='decimalyear')
         end_telemetry_time = Time(Time(self.query_end, format='mjd'), format='decimalyear')
+        start_time = Time(Time('2019-01-16 00:00:00.000', format='iso'), format='decimalyear')
+        end_time = Time(Time('2019-01-17 00:01:00.000', format='iso'), format='decimalyear')
+        start_telemetry_time = start_time
+        end_telemetry_time = end_time
+
         # end_telemetry_time = Time(Time(Time.now(), format='mjd'), format='decimalyear')
         # grating_val gets bogged down if more than about a day
 
         # Construct new entry for this file for the grating wheel database table.
-        retrieved_grat = False
+        grating_val = mnemonic = get_mnemonic('INRSI_GWA_MECH_POS', start_telemetry_time, end_telemetry_time)
+        type_error = False
+        logging.info('starting telem loop')
         for telem in GRATING_TELEMETRY.keys():
-            # if telem in ['INRSH_GWA_ADCMGAIN', 'INRSH_GWA_ADCMOFFSET', 'INRSH_GWA_MOTOR_VREF']:
-            #     continue
             if 'POSITION' in telem:
                 grating = telem.split('_')[0]
                 telemetry = telem.replace(grating+"_", "")
-                if retrieved_grat is False:
-                    try:
-                        mnemonics = get_mnemonics([telemetry, 'INRSI_GWA_MECH_POS'], start_telemetry_time, end_telemetry_time)
-                        mnemonic = mnemonics[telemetry]
-                        grating_val = mnemonics['INRSI_GWA_MECH_POS']
-                        retrieved_grat = True
-                    except TypeError:
-                        type_error = True
-                        print("type error because data was empty; try using earlier start date if you want this to run. Data should be present already.")
-                        # FIGURE OUT HOW TO DEAL WITH THIS ERROR
             else:
-                try:
-                    mnemonic = get_mnemonic(telem, start_telemetry_time, end_telemetry_time)
-                except TypeError:
-                    type_error = True
-                    print("type error because data was empty 2")
-                    continue
+                telemetry = telem
+            try:
+                logging.info("querying mnemonic")
+                mnemonic = get_mnemonic(telemetry, start_telemetry_time, end_telemetry_time)
+            except TypeError:
+                type_error = True
+                logging.info("TypeError because data was empty; try using earlier start date to run grating_monitor. Data should be present already.")
+            # include placeholder values for other telemetry in order to insert into database
+            logging.info('including other telems')
             other_telems = GRATING_TELEMETRY.keys()
             other_telems_dict = {}
             for telems in other_telems:
                 other_telems_dict[telems.lower()] = None
             other_telems_dict.pop(telem.lower())
             if type_error is False:
+                logging.info("looping through times")
                 for time in mnemonic.data['MJD']:
                     if 'POSITION' in telem:
-                        min_distance = np.min(abs(grating_val.data['MJD']-time))
-                        if min_distance > 0.01:  # determine whether this is realistic time difference between data for voltage and grating value
-                            logging.warning("Grating retrieved may not match grating for which voltage is being determined")
-                        closest_location = np.where(abs(grating_val.data['MJD']-time) == min_distance)[0][0]
-                        grating_used = grating_val.data['euvalue'][closest_location]
+                        # Grating wheel x and y positions are recorded sporadically when the GWA is moved, whereas most telemetry is recorded on a regular time interval
+                        # Find the previous grating wheel value that was recorded closest to the time the mnemonic value was recorded
+                        # We expect the telemetry to be recorded within the past 14 days
+                        # see https://jira.stsci.edu/browse/JSDP-1810
+                        # Telemetry values that exhibit this behavior include INRSI_GWA_X_TILT_AVGED, INRSI_GWA_Y_TILT_AVGED,
+                        # INRSI_C_GWA_X_POSITION, INRSI_C_GWA_Y_POSITION, INRSI_C_GWA_X_POS_REC, INRSI_C_GWA_Y_POS_REC, INRSI_GWA_TILT_TEMP
+                        try:
+                            min_distance = np.min(grating_val.data['MJD'][np.where(grating_val.data['MJD'] > time)]-time)
+                            if min_distance > 14:
+                                logging.warning("Grating retrieved may not match grating for which voltage is being determined")
+                            closest_location = np.where((grating_val.data['MJD'] - time) == min_distance)[0][0]
+                            grating_used = grating_val.data['euvalue'][closest_location]
+                        except ValueError:
+                            logging.warning("Using next rather than previous gwa val")
+                            min_distance = np.min(abs(grating_val.data['MJD']-time))
+                            closest_location = np.where(abs(grating_val.data['MJD'] - time) == min_distance)[0][0]
+                            grating_used = grating_val.data['euvalue'][closest_location]
                         if grating_used == grating:
                             try:
                                 grating_db_entry = {'time': time,
@@ -168,6 +179,8 @@ def process(self):
                         except TypeError:
                             logging.warning("may be skipping a value with same entry_date.")
                             continue
+            else:
+                logging.warning("There was a TypeError causing this data entry to fail.")
 
     @log_fail
     @log_info

From 5ccdc2fa6532823462be85500f430502cff3ac92 Mon Sep 17 00:00:00 2001
From: Teagan King <55458851+tnking97@users.noreply.github.com>
Date: Mon, 29 Nov 2021 13:46:35 -0500
Subject: [PATCH 14/18] minor updates to gwa monitor

---
 .../common_monitors/grating_monitor.py               | 12 ++++--------
 jwql/website/apps/jwql/bokeh_containers.py           |  2 +-
 .../apps/jwql/monitor_pages/monitor_grating_bokeh.py | 12 ++++++++----
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/jwql/instrument_monitors/common_monitors/grating_monitor.py b/jwql/instrument_monitors/common_monitors/grating_monitor.py
index c77b4b5e0..c6a3b0520 100755
--- a/jwql/instrument_monitors/common_monitors/grating_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/grating_monitor.py
@@ -32,7 +32,7 @@
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE, GRATING_TELEMETRY
 from jwql.utils.logging_functions import log_info, log_fail
 from jwql.utils.monitor_utils import update_monitor_table
-from jwql.utils.utils import ensure_dir_exists, get_config, initialize_instrument_monitor
+from jwql.utils.utils import initialize_instrument_monitor
 
 
 class Grating():
@@ -96,13 +96,9 @@ def process(self):
 
         start_telemetry_time = Time(Time(self.query_start, format='mjd'), format='decimalyear')
         end_telemetry_time = Time(Time(self.query_end, format='mjd'), format='decimalyear')
-        start_time = Time(Time('2019-01-16 00:00:00.000', format='iso'), format='decimalyear')
-        end_time = Time(Time('2019-01-17 00:01:00.000', format='iso'), format='decimalyear')
-        start_telemetry_time = start_time
-        end_telemetry_time = end_time
-
-        # end_telemetry_time = Time(Time(Time.now(), format='mjd'), format='decimalyear')
-        # grating_val gets bogged down if more than about a day
+        # note that grating_val gets bogged down if running over a long time period. Can run in smaller chunks of time, eg:
+        # start_telemetry_time = Time(Time('2019-01-28 00:00:00.000', format='iso'), format='decimalyear')
+        # end_telemetry_time = Time(Time('2019-02-15 00:01:00.000', format='iso'), format='decimalyear')
 
         # Construct new entry for this file for the grating wheel database table.
         grating_val = mnemonic = get_mnemonic('INRSI_GWA_MECH_POS', start_telemetry_time, end_telemetry_time)
diff --git a/jwql/website/apps/jwql/bokeh_containers.py b/jwql/website/apps/jwql/bokeh_containers.py
index cdef7434f..1310f69e3 100644
--- a/jwql/website/apps/jwql/bokeh_containers.py
+++ b/jwql/website/apps/jwql/bokeh_containers.py
@@ -348,7 +348,7 @@ def grating_monitor_tabs(instrument):
         plots.append(grating_plot)
 
     # Put grating plots on the top row.
-    grating_layout = layout(plots[0:18])
+    grating_layout = layout(plots[0:19])
     grating_layout.sizing_mode = 'scale_width'
     grating_tab = Panel(child=grating_layout, title="Grating Monitor Plots")
     tabs.append(grating_tab)
diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_grating_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_grating_bokeh.py
index 22328f21f..b70851eca 100644
--- a/jwql/website/apps/jwql/monitor_pages/monitor_grating_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_grating_bokeh.py
@@ -23,7 +23,6 @@
 from astropy.time import Time
 import os
 
-from astropy.stats import sigma_clip
 import numpy as np
 
 from jwql.bokeh_templating import BokehTemplate
@@ -114,7 +113,12 @@ def update_grating_figures(self):
             # Update plot limits if data exists
             gwa_vals_cut = gwa_vals[np.where(gwa_vals != None)]  # USE ABOVE AS WELL?
             if len(gwa_vals_cut) != 0:
-                self.refs['gwa_x_{}'.format(telemetry)].start = datetime.now() - timedelta(days=100)
+                self.refs['gwa_x_{}'.format(telemetry)].start = datetime.now() - timedelta(days=3)
                 self.refs['gwa_x_{}'.format(telemetry)].end = datetime.now()
-                self.refs['gwa_y_{}'.format(telemetry)].start = gwa_vals_cut.min() - 0.5
-                self.refs['gwa_y_{}'.format(telemetry)].end = gwa_vals_cut.max() + 0.5
+                if telemetry in ['INRSH_GWA_ADCMGAIN', 'INRSH_GWA_ADCMOFFSET', 'INRSH_GWA_MOTOR_VREF']:
+                    self.refs['gwa_y_{}'.format(telemetry)].start = gwa_vals_cut.min() - 0.5
+                    self.refs['gwa_y_{}'.format(telemetry)].end = gwa_vals_cut.max() + 0.5
+                else:
+                    # Use wider y-range to see full green/yellow/red areas for x/y position telemetry
+                    self.refs['gwa_y_{}'.format(telemetry)].start = -400
+                    self.refs['gwa_y_{}'.format(telemetry)].end = 400

From 88060fa4c72e0c76d744d0ffb84ff1103442c157 Mon Sep 17 00:00:00 2001
From: Teagan King <55458851+tnking97@users.noreply.github.com>
Date: Mon, 29 Nov 2021 14:03:15 -0500
Subject: [PATCH 15/18] bug fix

---
 jwql/instrument_monitors/common_monitors/grating_monitor.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jwql/instrument_monitors/common_monitors/grating_monitor.py b/jwql/instrument_monitors/common_monitors/grating_monitor.py
index c6a3b0520..777369a2f 100755
--- a/jwql/instrument_monitors/common_monitors/grating_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/grating_monitor.py
@@ -101,7 +101,7 @@ def process(self):
         # end_telemetry_time = Time(Time('2019-02-15 00:01:00.000', format='iso'), format='decimalyear')
 
         # Construct new entry for this file for the grating wheel database table.
-        grating_val = mnemonic = get_mnemonic('INRSI_GWA_MECH_POS', start_telemetry_time, end_telemetry_time)
+        grating_val = get_mnemonic('INRSI_GWA_MECH_POS', start_telemetry_time, end_telemetry_time)
         type_error = False
         logging.info('starting telem loop')
         for telem in GRATING_TELEMETRY.keys():

From 477e472ca915c9a206a928923e9a30ee081fe061 Mon Sep 17 00:00:00 2001
From: Teagan King <55458851+tnking97@users.noreply.github.com>
Date: Mon, 29 Nov 2021 14:24:44 -0500
Subject: [PATCH 16/18] minimized excess gwa monitor logging and improved
 comments

---
 .../common_monitors/grating_monitor.py        | 24 +++++++++----------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/jwql/instrument_monitors/common_monitors/grating_monitor.py b/jwql/instrument_monitors/common_monitors/grating_monitor.py
index 777369a2f..347059168 100755
--- a/jwql/instrument_monitors/common_monitors/grating_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/grating_monitor.py
@@ -103,7 +103,6 @@ def process(self):
         # Construct new entry for this file for the grating wheel database table.
         grating_val = get_mnemonic('INRSI_GWA_MECH_POS', start_telemetry_time, end_telemetry_time)
         type_error = False
-        logging.info('starting telem loop')
         for telem in GRATING_TELEMETRY.keys():
             if 'POSITION' in telem:
                 grating = telem.split('_')[0]
@@ -111,20 +110,19 @@ def process(self):
             else:
                 telemetry = telem
             try:
-                logging.info("querying mnemonic")
                 mnemonic = get_mnemonic(telemetry, start_telemetry_time, end_telemetry_time)
             except TypeError:
                 type_error = True
                 logging.info("TypeError because data was empty; try using earlier start date to run grating_monitor. Data should be present already.")
+
             # include placeholder values for other telemetry in order to insert into database
-            logging.info('including other telems')
             other_telems = GRATING_TELEMETRY.keys()
             other_telems_dict = {}
             for telems in other_telems:
                 other_telems_dict[telems.lower()] = None
             other_telems_dict.pop(telem.lower())
+
             if type_error is False:
-                logging.info("looping through times")
                 for time in mnemonic.data['MJD']:
                     if 'POSITION' in telem:
                         # Grating wheel x and y positions are recorded sporadically when the GWA is moved, whereas most telemetry is recorded on a regular time interval
@@ -148,7 +146,7 @@ def process(self):
                             try:
                                 grating_db_entry = {'time': time,
                                                     telem.lower(): float(mnemonic.data['euvalue'][np.where(mnemonic.data['MJD'] == time)]),
-                                                    'run_monitor': True,  # UPDATE
+                                                    'run_monitor': True,  # Update if monitor_run is set to False
                                                     'entry_date': datetime.datetime.now()  # need slightly different times to add to database
                                                     }
                                 grating_db_entry.update(other_telems_dict)
@@ -159,13 +157,13 @@ def process(self):
                                 logging.warning("May be skipping a value with same entry_date")
                                 continue
                         else:
-                            logging.warning("Not adding entry because data is for {} rather than {}".format(grating_used, grating))
+                            # Not adding entry because data is for a "grating_used" other than the "grating" specified
                             continue
                     else:
                         try:
                             grating_db_entry = {'time': time,
                                                 telem.lower(): float(mnemonic.data['euvalue'][np.where(mnemonic.data['MJD'] == time)]),
-                                                'run_monitor': True,  # UPDATE
+                                                'run_monitor': True,  # Update if monitor_run is set to False
                                                 'entry_date': datetime.datetime.now()  # need slightly different times to add to database
                                                 }
                             grating_db_entry.update(other_telems_dict)
@@ -194,17 +192,17 @@ def run(self):
         self.identify_tables()
 
         # Locate the record of most recent MAST search; use this time
-        # (plus a 30 day buffer to catch any missing files from
-        # previous run) as the start time in the new MAST search.
+        # as the start time in the new MAST search. Could include a 30 day
+        # buffer to catch any missing telemetry, but this would include
+        # quite a lot of extra data and increase run time.
         most_recent_search = self.most_recent_search()
-        self.query_start = most_recent_search  # MAYBE USE 30 DAYS BUFFER BEFORE THAT?
+        self.query_start = most_recent_search
         logging.info('\tQuery times: {} {}'.format(self.query_start, self.query_end))
 
         self.process()
         monitor_run = True
-        # else:  # DETERMINE CASE WHEN DON"T RUN MONITOR-- THERE SHOULD FREQUENTLY BE UPDATED TELEMETRY THOUGH
-        #     logging.info('\tGrating monitor skipped. {} new dark files for {}.'.format(len(new_files), self.instrument))
-        #     monitor_run = False
+        # If there is a case when we wouldn't want to run the monitor, it could be included here, with monitor_run set to False.
+        # However, we should be updating these plots on the daily schedule becuase the telemetry will be updated frequently.
 
         # Update the query history
         new_entry = {'instrument': self.instrument,

From b6dfbc422e8c4c2950f81a6f9f9c3245d09020f8 Mon Sep 17 00:00:00 2001
From: Teagan King <55458851+tnking97@users.noreply.github.com>
Date: Thu, 9 Dec 2021 15:51:29 -0500
Subject: [PATCH 17/18] Update test_grating_monitor.py

---
 jwql/tests/test_grating_monitor.py | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/jwql/tests/test_grating_monitor.py b/jwql/tests/test_grating_monitor.py
index 0ed35e4b5..73460ea7b 100644
--- a/jwql/tests/test_grating_monitor.py
+++ b/jwql/tests/test_grating_monitor.py
@@ -60,16 +60,3 @@ def test_mnemonics_retrieval():
     assert motor_vref == -0.0009155273999999999
     assert x_pos == 173.1699688
     assert y_pos == 98.6046408
-
-
-# def test_process():
-#     grating = grating_monitor.Grating()
-#     grating.query_start = SOME_TIME
-#     grating.query_end = SOME_TIME
-#     # dont' want to actually insert data into database table.. stats_table define as a test table? or
-#     # or set a test_value == True if in testing?
-#     # or separate part of this into another function and just retrieve the db_entry?
-
-# Test that a flag is raised if any plots have points in yellow/red zone (within a certain time frame)
-
-# Check input thresholds make sense, eg. time frame and limits? these are set in constants though

From 3ee3c0273702525091f3df7a33441faefa7943f6 Mon Sep 17 00:00:00 2001
From: Teagan King <55458851+tnking97@users.noreply.github.com>
Date: Thu, 9 Dec 2021 15:53:42 -0500
Subject: [PATCH 18/18] A few pep-8 updates

---
 .../common_monitors/grating_monitor.py                   | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/jwql/instrument_monitors/common_monitors/grating_monitor.py b/jwql/instrument_monitors/common_monitors/grating_monitor.py
index 347059168..27cc08b46 100755
--- a/jwql/instrument_monitors/common_monitors/grating_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/grating_monitor.py
@@ -22,7 +22,6 @@
 
 from astropy.time import Time
 from jwql.edb.engineering_database import get_mnemonic
-from jwql.edb.engineering_database import get_mnemonics
 import matplotlib
 matplotlib.use('Agg')
 import numpy as np
@@ -39,7 +38,7 @@ class Grating():
     """Class for executing the grating wheel monitor.
 
     This class will search for a particular mnemonic for each
-    instrument and will run the monitor on these files. The monitor 
+    instrument and will run the monitor on these files. The monitor
     will perform statistical measurements and plot the mnemonic data
     in order to monitor the grating wheel telemetry over time.
     Results are all saved to database tables.
@@ -106,7 +105,7 @@ def process(self):
         for telem in GRATING_TELEMETRY.keys():
             if 'POSITION' in telem:
                 grating = telem.split('_')[0]
-                telemetry = telem.replace(grating+"_", "")
+                telemetry = telem.replace(grating + "_", "")
             else:
                 telemetry = telem
             try:
@@ -132,14 +131,14 @@ def process(self):
                         # Telemetry values that exhibit this behavior include INRSI_GWA_X_TILT_AVGED, INRSI_GWA_Y_TILT_AVGED,
                         # INRSI_C_GWA_X_POSITION, INRSI_C_GWA_Y_POSITION, INRSI_C_GWA_X_POS_REC, INRSI_C_GWA_Y_POS_REC, INRSI_GWA_TILT_TEMP
                         try:
-                            min_distance = np.min(grating_val.data['MJD'][np.where(grating_val.data['MJD'] > time)]-time)
+                            min_distance = np.min(grating_val.data['MJD'][np.where(grating_val.data['MJD'] > time)] - time)
                             if min_distance > 14:
                                 logging.warning("Grating retrieved may not match grating for which voltage is being determined")
                             closest_location = np.where((grating_val.data['MJD'] - time) == min_distance)[0][0]
                             grating_used = grating_val.data['euvalue'][closest_location]
                         except ValueError:
                             logging.warning("Using next rather than previous gwa val")
-                            min_distance = np.min(abs(grating_val.data['MJD']-time))
+                            min_distance = np.min(abs(grating_val.data['MJD'] - time))
                             closest_location = np.where(abs(grating_val.data['MJD'] - time) == min_distance)[0][0]
                             grating_used = grating_val.data['euvalue'][closest_location]
                         if grating_used == grating: