[WIP] Build Grating Wheel Sensor Telemetry Monitor

docs/source/common_monitors.rst

@@ -20,6 +20,12 @@ dark_monitor.py
     :members:
     :undoc-members:
 
+grating_monitor.py
+--------------------
+.. automodule:: jwql.instrument_monitors.common_monitors.grating_monitor
+    :members:
+    :undoc-members:
+
 readnoise_monitor.py
 --------------------
 .. automodule:: jwql.instrument_monitors.common_monitors.readnoise_monitor

docs/source/tests.rst

@@ -56,6 +56,12 @@ test_edb.py
     :members:
     :undoc-members:
 
+test_grating_monitor.py
+--------------------
+.. automodule:: jwql.tests.test_grating_monitor
+    :members:
+    :undoc-members:
+
 test_instrument_properties.py
 -----------------------------
 .. automodule:: jwql.tests.test_instrument_properties

jwql/database/database_interface.py

@@ -426,6 +426,8 @@ def set_read_permissions():
 NIRISSBiasStats = monitor_orm_factory('niriss_bias_stats')
 NIRSpecBiasQueryHistory = monitor_orm_factory('nirspec_bias_query_history')
 NIRSpecBiasStats = monitor_orm_factory('nirspec_bias_stats')
+NIRSpecGratingQueryHistory = monitor_orm_factory('nirspec_grating_query_history')
+NIRSpecGratingStats = monitor_orm_factory('nirspec_grating_stats')
 NIRCamBadPixelQueryHistory = monitor_orm_factory('nircam_bad_pixel_query_history')
 NIRCamBadPixelStats = monitor_orm_factory('nircam_bad_pixel_stats')
 NIRISSBadPixelQueryHistory = monitor_orm_factory('niriss_bad_pixel_query_history')

jwql/database/monitor_table_definitions/nirspec/nirspec_grating_query_history.txt

@@ -0,0 +1,4 @@
+START_TIME_MJD, float
+END_TIME_MJD, float
+RUN_MONITOR, bool
+ENTRY_DATE, datetime

jwql/database/monitor_table_definitions/nirspec/nirspec_grating_stats.txt

@@ -0,0 +1,20 @@
+TIME, string
+INRSH_GWA_ADCMGAIN, float
+INRSH_GWA_ADCMOFFSET, float
+INRSH_GWA_MOTOR_VREF, float
+PRISM_INRSI_C_GWA_X_POSITION, float
+PRISM_INRSI_C_GWA_Y_POSITION, float
+MIRROR_INRSI_C_GWA_X_POSITION, float
+MIRROR_INRSI_C_GWA_Y_POSITION, float
+G140H_INRSI_C_GWA_X_POSITION, float
+G140H_INRSI_C_GWA_Y_POSITION, float
+G235H_INRSI_C_GWA_X_POSITION, float
+G235H_INRSI_C_GWA_Y_POSITION, float
+G395H_INRSI_C_GWA_X_POSITION, float
+G395H_INRSI_C_GWA_Y_POSITION, float
+G140M_INRSI_C_GWA_X_POSITION, float
+G140M_INRSI_C_GWA_Y_POSITION, float
+G235M_INRSI_C_GWA_X_POSITION, float
+G235M_INRSI_C_GWA_Y_POSITION, float
+G395M_INRSI_C_GWA_X_POSITION, float
+G395M_INRSI_C_GWA_Y_POSITION, float

jwql/instrument_monitors/common_monitors/grating_monitor.py

@@ -0,0 +1,226 @@
+#! /usr/bin/env python
+
+"""This module contains code for the grating wheel monitor, which monitors
+the grating wheel positions over time. This code has been adapted from bias_monitor.py
+
+Author
+------
+    - Teagan King
+
+Use
+---
+    This module can be used from the command line as such:
+
+    ::
+
+        python grating_monitor.py
+"""
+
+import datetime
+import logging
+import os
+
+from astropy.time import Time
+from jwql.edb.engineering_database import get_mnemonic
+import matplotlib
+matplotlib.use('Agg')
+import numpy as np
+
+from jwql.database.database_interface import session
+from jwql.database.database_interface import NIRSpecGratingQueryHistory, NIRSpecGratingStats
+from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE, GRATING_TELEMETRY
+from jwql.utils.logging_functions import log_info, log_fail
+from jwql.utils.monitor_utils import update_monitor_table
+from jwql.utils.utils import initialize_instrument_monitor
+
+
+class Grating():
+    """Class for executing the grating wheel monitor.
+
+    This class will search for a particular mnemonic for each
+    instrument and will run the monitor on these files. The monitor
+    will perform statistical measurements and plot the mnemonic data
+    in order to monitor the grating wheel telemetry over time.
+    Results are all saved to database tables.
+
+    Attributes
+    ----------
+    query_start : float
+        MJD start date to use for querying EDB.
+
+    query_end : float
+        MJD end date to use for querying EDB.
+
+    instrument : str
+        Name of instrument used to collect the data.
+    """
+
+    def __init__(self):
+        """Initialize an instance of the ``Grating`` class."""
+
+    def identify_tables(self):
+        """Determine which database tables to use for a run of the grating wheel
+        monitor.
+        """
+
+        mixed_case_name = JWST_INSTRUMENT_NAMES_MIXEDCASE[self.instrument]
+        self.query_table = eval('{}GratingQueryHistory'.format(mixed_case_name))
+        self.stats_table = eval('{}GratingStats'.format(mixed_case_name))
+
+    def most_recent_search(self):
+        """Query the query history database and return the information
+        on the most recent query where the grating wheel monitor was executed.
+
+        Returns
+        -------
+        query_result : float
+            Date (in MJD) of the ending range of the previous MAST query
+            where the grating wheel monitor was run.
+        """
+        query = session.query(self.query_table).filter(self.query_table.run_monitor == True).order_by(self.query_table.end_time_mjd).all()
+
+        if len(query) == 0:
+            query_result = 57357.0  # a.k.a. Dec 1, 2015 == CV3
+            logging.info(('\tNo query history. Beginning search date will be set to {}.'.format(query_result)))
+        else:
+            query_result = query[-1].end_time_mjd
+
+        return query_result
+
+    def process(self):
+        """The main method for processing telemetry.  See module docstrings
+        for further details.
+
+        """
+
+        start_telemetry_time = Time(Time(self.query_start, format='mjd'), format='decimalyear')
+        end_telemetry_time = Time(Time(self.query_end, format='mjd'), format='decimalyear')
+        # note that grating_val gets bogged down if running over a long time period. Can run in smaller chunks of time, eg:
+        # start_telemetry_time = Time(Time('2019-01-28 00:00:00.000', format='iso'), format='decimalyear')
+        # end_telemetry_time = Time(Time('2019-02-15 00:01:00.000', format='iso'), format='decimalyear')
+
+        # Construct new entry for this file for the grating wheel database table.
+        grating_val = get_mnemonic('INRSI_GWA_MECH_POS', start_telemetry_time, end_telemetry_time)
+        type_error = False
+        for telem in GRATING_TELEMETRY.keys():
+            if 'POSITION' in telem:
+                grating = telem.split('_')[0]
+                telemetry = telem.replace(grating + "_", "")
+            else:
+                telemetry = telem
+            try:
+                mnemonic = get_mnemonic(telemetry, start_telemetry_time, end_telemetry_time)
+            except TypeError:
+                type_error = True
+                logging.info("TypeError because data was empty; try using earlier start date to run grating_monitor. Data should be present already.")
+
+            # include placeholder values for other telemetry in order to insert into database
+            other_telems = GRATING_TELEMETRY.keys()
+            other_telems_dict = {}
+            for telems in other_telems:
+                other_telems_dict[telems.lower()] = None
+            other_telems_dict.pop(telem.lower())
+
+            if type_error is False:
+                for time in mnemonic.data['MJD']:
+                    if 'POSITION' in telem:
+                        # Grating wheel x and y positions are recorded sporadically when the GWA is moved, whereas most telemetry is recorded on a regular time interval
+                        # Find the previous grating wheel value that was recorded closest to the time the mnemonic value was recorded
+                        # We expect the telemetry to be recorded within the past 14 days
+                        # see https://jira.stsci.edu/browse/JSDP-1810
+                        # Telemetry values that exhibit this behavior include INRSI_GWA_X_TILT_AVGED, INRSI_GWA_Y_TILT_AVGED,
+                        # INRSI_C_GWA_X_POSITION, INRSI_C_GWA_Y_POSITION, INRSI_C_GWA_X_POS_REC, INRSI_C_GWA_Y_POS_REC, INRSI_GWA_TILT_TEMP
+                        try:
+                            min_distance = np.min(grating_val.data['MJD'][np.where(grating_val.data['MJD'] > time)] - time)
+                            if min_distance > 14:
+                                logging.warning("Grating retrieved may not match grating for which voltage is being determined")
+                            closest_location = np.where((grating_val.data['MJD'] - time) == min_distance)[0][0]
+                            grating_used = grating_val.data['euvalue'][closest_location]
+                        except ValueError:
+                            logging.warning("Using next rather than previous gwa val")
+                            min_distance = np.min(abs(grating_val.data['MJD'] - time))
+                            closest_location = np.where(abs(grating_val.data['MJD'] - time) == min_distance)[0][0]
+                            grating_used = grating_val.data['euvalue'][closest_location]
+                        if grating_used == grating:
+                            try:
+                                grating_db_entry = {'time': time,
+                                                    telem.lower(): float(mnemonic.data['euvalue'][np.where(mnemonic.data['MJD'] == time)]),
+                                                    'run_monitor': True,  # Update if monitor_run is set to False
+                                                    'entry_date': datetime.datetime.now()  # need slightly different times to add to database
+                                                    }
+                                grating_db_entry.update(other_telems_dict)
+                                # Add this new entry to the grating database table
+                                self.stats_table.__table__.insert().execute(grating_db_entry)
+                                logging.info('\tNew entry added to grating database table: {}'.format(grating_db_entry))
+                            except TypeError:
+                                logging.warning("May be skipping a value with same entry_date")
+                                continue
+                        else:
+                            # Not adding entry because data is for a "grating_used" other than the "grating" specified
+                            continue
+                    else:
+                        try:
+                            grating_db_entry = {'time': time,
+                                                telem.lower(): float(mnemonic.data['euvalue'][np.where(mnemonic.data['MJD'] == time)]),
+                                                'run_monitor': True,  # Update if monitor_run is set to False
+                                                'entry_date': datetime.datetime.now()  # need slightly different times to add to database
+                                                }
+                            grating_db_entry.update(other_telems_dict)
+                            # Add this new entry to the grating database table
+                            self.stats_table.__table__.insert().execute(grating_db_entry)
+                            logging.info('\tNew entry added to grating database table: {}'.format(grating_db_entry))
+                        except TypeError:
+                            logging.warning("may be skipping a value with same entry_date.")
+                            continue
+            else:
+                logging.warning("There was a TypeError causing this data entry to fail.")
+
+    @log_fail
+    @log_info
+    def run(self):
+        """The main method.  See module docstrings for further details."""
+
+        logging.info('Begin logging for grating_monitor')
+
+        # Use the current time as the end time for MAST query
+        self.query_end = Time.now().mjd
+
+        self.instrument = 'nirspec'
+
+        # Identify which database tables to use
+        self.identify_tables()
+
+        # Locate the record of most recent MAST search; use this time
+        # as the start time in the new MAST search. Could include a 30 day
+        # buffer to catch any missing telemetry, but this would include
+        # quite a lot of extra data and increase run time.
+        most_recent_search = self.most_recent_search()
+        self.query_start = most_recent_search
+        logging.info('\tQuery times: {} {}'.format(self.query_start, self.query_end))
+
+        self.process()
+        monitor_run = True
+        # If there is a case when we wouldn't want to run the monitor, it could be included here, with monitor_run set to False.
+        # However, we should be updating these plots on the daily schedule becuase the telemetry will be updated frequently.
+
+        # Update the query history
+        new_entry = {'instrument': self.instrument,
+                     'start_time_mjd': self.query_start,
+                     'end_time_mjd': self.query_end,
+                     'run_monitor': monitor_run,
+                     'entry_date': datetime.datetime.now()}
+        self.query_table.__table__.insert().execute(new_entry)
+        logging.info('\tUpdated the query history table')
+
+        logging.info('Grating Monitor completed successfully.')
+
+
+if __name__ == '__main__':
+
+    module = os.path.basename(__file__).strip('.py')
+    start_time, log_file = initialize_instrument_monitor(module)
+
+    monitor = Grating()
+    monitor.run()
+
+    update_monitor_table(module, start_time, log_file)

jwql/tests/test_grating_monitor.py

@@ -0,0 +1,62 @@
+#! /usr/bin/env python
+
+"""Tests for the ``grating_monitor`` module.
+
+Authors
+-------
+
+    - Teagan King
+
+Use
+---
+
+    These tests can be run via the command line (omit the ``-s`` to
+    suppress verbose output to stdout):
+    ::
+
+        pytest -s test_grating_monitor.py
+"""
+
+from astropy.time import Time
+import os
+import pytest
+
+from jwql.edb.engineering_database import get_mnemonics
+from jwql.database.database_interface import NIRSpecGratingQueryHistory, NIRSpecGratingStats
+from jwql.instrument_monitors.common_monitors import grating_monitor
+from jwql.utils.constants import GRATING_TELEMETRY
+from sqlalchemy.sql.sqltypes import TIME
+
+ON_GITHUB_ACTIONS = '/home/runner' in os.path.expanduser('~') or '/Users/runner' in os.path.expanduser('~')
+
+
+def test_identify_tables():
+    """Be sure the correct database tables are identified
+    """
+    grating = grating_monitor.Grating()
+    grating.instrument = 'nirspec'
+    grating.identify_tables()
+    assert grating.query_table == eval('NIRSpecGratingQueryHistory')
+    assert grating.pixel_table == eval('NIRSpecGratingStats')
+
+
+def test_mnemonics_retrieval():
+    start_telemetry_time = Time(Time(59013, format='mjd'), format='decimalyear')
+    end_telemetry_time = Time(Time(55010, format='mjd'), format='decimalyear')
+    # THIS IS A LONG TIME FRAME...
+    mnemonics_list = ['INRSI_GWA_MECH_POS', 'INRSH_GWA_ADCMGAIN', 'INRSH_GWA_ADCMOFFSET', 'INRSH_GWA_MOTOR_VREF', 'INRSI_C_GWA_X_POSITION', 'INRSI_C_GWA_Y_POSITION']
+    mnemonics = get_mnemonics(mnemonics_list, start_telemetry_time, end_telemetry_time)
+
+    grating_val = mnemonics['INRSI_GWA_MECH_POS'].data[0]['euvalue']
+    adcmgain = mnemonics['INRSH_GWA_ADCMGAIN'].data[0]['euvalue']
+    adcmoffset = mnemonics['INRSH_GWA_ADCMOFFSET'].data[0]['euvalue']
+    motor_vref = mnemonics['INRSH_GWA_MOTOR_VREF'].data[0]['euvalue']
+    x_pos = mnemonics['INRSI_C_GWA_X_POSITION'].data[0]['euvalue']
+    y_pos = mnemonics['INRSI_C_GWA_Y_POSITION'].data[0]['euvalue']
+
+    assert grating_val == 'MIRROR'
+    assert adcmgain == 2.5001527415
+    assert adcmoffset == -0.0010681153
+    assert motor_vref == -0.0009155273999999999
+    assert x_pos == 173.1699688
+    assert y_pos == 98.6046408