diff --git a/jwql/edb/engineering_database.py b/jwql/edb/engineering_database.py index 4f2c2d61c..d58b32123 100644 --- a/jwql/edb/engineering_database.py +++ b/jwql/edb/engineering_database.py @@ -45,7 +45,7 @@ """ import calendar from collections import OrderedDict -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from numbers import Number import os import warnings @@ -1417,9 +1417,15 @@ def change_only_bounding_points(date_list, value_list, starttime, endtime): if isinstance(starttime, Time): starttime = starttime.datetime + if starttime.tzinfo == None or starttime.tzinfo.utcoffset(starttime) == None: + starttime = starttime.replace(tzinfo=timezone.utc) + if isinstance(endtime, Time): endtime = endtime.datetime + if endtime.tzinfo == None or endtime.tzinfo.utcoffset(endtime) == None: + endtime = endtime.replace(tzinfo=timezone.utc) + valid_idx = np.where((date_list_arr <= endtime) & (date_list_arr >= starttime))[0] before_startime = np.where(date_list_arr < starttime)[0] before_endtime = np.where(date_list_arr < endtime)[0] @@ -1606,7 +1612,7 @@ def get_mnemonic(mnemonic_identifier, start_time, end_time): data = service.get_values(mnemonic_identifier, start_time, end_time, include_obstime=True, include_bracket_values=bracket) - dates = [datetime.strptime(row.obstime.iso, "%Y-%m-%d %H:%M:%S.%f") for row in data] + dates = [datetime.strptime(row.obstime.iso, "%Y-%m-%d %H:%M:%S.%f").replace(tzinfo=timezone.utc) for row in data] values = [row.value for row in data] if bracket: diff --git a/jwql/instrument_monitors/common_monitors/edb_telemetry_monitor.py b/jwql/instrument_monitors/common_monitors/edb_telemetry_monitor.py index c28b6eba2..2557d1935 100755 --- a/jwql/instrument_monitors/common_monitors/edb_telemetry_monitor.py +++ b/jwql/instrument_monitors/common_monitors/edb_telemetry_monitor.py @@ -385,7 +385,6 @@ from bokeh.models.layouts import TabPanel, Tabs from bokeh.plotting import figure, output_file, save, show from bokeh.palettes import Turbo256 -from jwql.database import database_interface from jwql.edb import engineering_database as ed from jwql.instrument_monitors.common_monitors.edb_telemetry_monitor_utils import condition from jwql.instrument_monitors.common_monitors.edb_telemetry_monitor_utils import utils @@ -395,7 +394,7 @@ from jwql.utils.constants import EDB_DEFAULT_PLOT_RANGE, JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, MIRI_POS_RATIO_VALUES from jwql.utils.constants import ON_GITHUB_ACTIONS, ON_READTHEDOCS from jwql.utils.permissions import set_permissions -from jwql.utils.utils import ensure_dir_exists, get_config +from jwql.utils.utils import add_timezone_to_datetime, ensure_dir_exists, get_config if not ON_GITHUB_ACTIONS and not ON_READTHEDOCS: # Need to set up django apps before we can access the models @@ -516,10 +515,9 @@ def add_new_block_db_entry(self, mnem, query_time): logging.info(f"Adding new entry for {mnem.mnemonic_identifier} to history table.") times = mnem.data["dates"].data data = mnem.data["euvalues"].data - stdevs = mnem.stdev times = ensure_list(times) data = ensure_list(data) - stdevs = ensure_list(stdevs) + stdevs = ensure_list(mnem.stdev) medians = ensure_list(mnem.median) maxs = ensure_list(mnem.max) mins = ensure_list(mnem.min) @@ -537,6 +535,15 @@ def add_new_block_db_entry(self, mnem, query_time): if (isinstance(medians[0], int) | isinstance(medians[0], np.integer)): medians = [float(v) for v in medians] + + print('In add_new_block_db_entry:') + for ll, name in zip([times, data, stdevs, medians, maxs, mins],['times', 'data', 'stdevs', 'medians', 'maxs', 'mins']): + if isinstance(ll, np.ndarray): + print(f'{name} is a numpy array.') + print(ll) + + + db_entry = {'mnemonic': mnem.mnemonic_identifier, 'latest_query': query_time, 'times': times, @@ -1090,8 +1097,13 @@ def get_history(self, mnemonic, start_date, end_date, info={}, meta={}): # outside of the plot range. Return only the points inside the desired # plot range for row in data: - good = np.where((np.array(row.times) > self._plot_start) & (np.array(row.times) < self._plot_end))[0] - times = list(np.array(row.times)[good]) + # Make sure the data from the database has timezone info + time_vals = row.times + if time_vals[0].tzinfo == None or tie_vals[0].tzinfo.utcoffset(time_vals[0]) == None: + time_vals = [val.replace(tzinfo=datetime.timezone.utc) for val in time_vals] + + good = np.where((np.array(time_vals) > self._plot_start) & (np.array(time_vals) < self._plot_end))[0] + times = list(np.array(time_vals)[good]) data = list(np.array(row.data)[good]) medians = list(np.array(row.median)[good]) maxs = list(np.array(row.max)[good]) @@ -1273,8 +1285,8 @@ def identify_tables(self, inst, tel_type): Examples include "every_change", "daily", "all", etc """ mixed_case_name = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst] - if '_means' in tel_type: - tel_type = tel_type.strip('_means') + #if '_means' in tel_type: + # tel_type = tel_type.strip('_means') tel_type = tel_type.title().replace('_', '') self.history_table_name = f'{mixed_case_name}Edb{tel_type}Stats' self.history_table = eval(self.history_table_name) @@ -1299,11 +1311,13 @@ def most_recent_search(self, telem_name): if len(query) == 0: base_time = '2022-11-15 00:00:0.0' - query_result = datetime.datetime.strptime(base_time, '%Y-%m-%d %H:%M:%S.%f') + query_result = datetime.datetime.strptime(base_time, '%Y-%m-%d %H:%M:%S.%f').replace(tzinfo=datetime.timezone.utc) logging.info(f'\tNo query history for {telem_name}. Returning default "previous query" date of {base_time}.') else: # Negative indexing not allowed in QuerySet query_result = query[len(query) - 1].latest_query + if query_result.tzinfo == None or query_result.tzinfo.utcoffset(query_result) == None: + query_result = query_result.replace(tzinfo=datetime.timezone.utc) logging.info(f'For {telem_name}, the previous query time is {query_result}') return query_result @@ -1527,7 +1541,7 @@ def run(self, instrument, mnemonic_dict, plot_start=None, plot_end=None): # SPEED UP TESTING. REMOVE BEFORE MERGING plot_start = self._today - datetime.timedelta(days=3.) - plot_end = self._today + plot_end = self._today #- datetime.timedelta(days=56.) @@ -1609,7 +1623,7 @@ def run(self, instrument, mnemonic_dict, plot_start=None, plot_end=None): # For daily_means mnemonics, we force the search to always start at noon, and # have a 1 day cadence if telem_type == 'daily_means': - most_recent_search = datetime.datetime.combine(most_recent_search.date(), datetime.time(hour=12)) + most_recent_search = datetime.datetime.combine(most_recent_search.date(), datetime.time(hour=12)).replace(tzinfo=datetime.timezone.utc) logging.info(f'Most recent search is {most_recent_search}.') logging.info(f'Query cadence is {self.query_cadence}') @@ -2116,8 +2130,13 @@ def ensure_list(var): var : list var, translated into a list if necessary """ - if not isinstance(var, list) and not isinstance(var, np.ndarray): - return [var] + if not isinstance(var, list): + if not isinstance(var, np.ndarray): + # Here we assume var is a single float, int, str, etc. + return [var] + else: + # Here we convert a numpy array to a list + return var.tolist() else: return var diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index e0794cc1b..120a34e11 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -27,6 +27,7 @@ - JWST TR JWST-STScI-004800, SM-12 """ +import datetime import getpass import glob import itertools @@ -153,6 +154,25 @@ def _validate_config(config_file_dict): ) +def add_timezone_to_datetime(dt): + """Check to see if timezone information is present in the given + datetime.datetime object. If not, set it to UTC. + + Parameters + ---------- + dt : datetime.datetime + Datetime object + + Returns + ------- + dt : datetime.datetime + Datetime object with UTC timezone info added + """ + if dt.tzinfo == None or dt.tzinfo.utcoffset(dt) == None: + dt = dt.replace(tzinfo=datetime.timezone.utc) + return dt + + def create_png_from_fits(filename, outdir): """Create and save a png file of the provided file. The file will be saved with the same filename as the input file, but diff --git a/jwql/website/apps/jwql/bokeh_containers.py b/jwql/website/apps/jwql/bokeh_containers.py index f4569747a..8210fed3f 100644 --- a/jwql/website/apps/jwql/bokeh_containers.py +++ b/jwql/website/apps/jwql/bokeh_containers.py @@ -230,7 +230,7 @@ def edb_monitor_tabs(instrument): """ html_file_list = file_list[instrument] print('read in html files') - + #CLEARLY THIS IS NOT BEING USED def generic_telemetry_plot(times, values, name, nominal_value=None, yellow_limits=None, red_limits=None, save=True): diff --git a/jwql/website/apps/jwql/monitor_models/edb.py b/jwql/website/apps/jwql/monitor_models/edb.py index 01a617af6..26a8220ce 100644 --- a/jwql/website/apps/jwql/monitor_models/edb.py +++ b/jwql/website/apps/jwql/monitor_models/edb.py @@ -32,7 +32,7 @@ from jwql.utils.constants import MAX_LEN_DEPENDENCY_VALUE, MAX_LEN_MNEMONIC -class FGSEdbBlocksStats(models.Model): +class FGSEdbBlockMeansStats(models.Model): mnemonic = models.CharField(max_length=MAX_LEN_MNEMONIC, blank=True, null=True) latest_query = models.DateTimeField(blank=True, null=True) times = ArrayField(models.DateTimeField()) @@ -45,7 +45,7 @@ class FGSEdbBlocksStats(models.Model): class Meta: managed = True - db_table = 'fgs_edb_blocks_stats' + db_table = 'fgs_edb_block_means_stats' unique_together = (('id', 'entry_date'),) @@ -114,7 +114,7 @@ class Meta: unique_together = (('id', 'entry_date'),) -class MIRIEdbBlocksStats(models.Model): +class MIRIEdbBlockMeansStats(models.Model): mnemonic = models.CharField(max_length=MAX_LEN_MNEMONIC, blank=True, null=True) latest_query = models.DateTimeField(blank=True, null=True) times = ArrayField(models.DateTimeField()) @@ -127,7 +127,7 @@ class MIRIEdbBlocksStats(models.Model): class Meta: managed = True - db_table = 'miri_edb_blocks_stats' + db_table = 'miri_edb_block_means_stats' unique_together = (('id', 'entry_date'),) @@ -196,7 +196,7 @@ class Meta: unique_together = (('id', 'entry_date'),) -class NIRCamEdbBlocksStats(models.Model): +class NIRCamEdbBlockMeansStats(models.Model): mnemonic = models.CharField(max_length=MAX_LEN_MNEMONIC, blank=True, null=True) latest_query = models.DateTimeField(blank=True, null=True) times = ArrayField(models.DateTimeField()) @@ -209,7 +209,7 @@ class NIRCamEdbBlocksStats(models.Model): class Meta: managed = True - db_table = 'nircam_edb_blocks_stats' + db_table = 'nircam_edb_block_means_stats' unique_together = (('id', 'entry_date'),) @@ -278,7 +278,7 @@ class Meta: unique_together = (('id', 'entry_date'),) -class NIRISSEdbBlocksStats(models.Model): +class NIRISSEdbBlockMeansStats(models.Model): mnemonic = models.CharField(max_length=MAX_LEN_MNEMONIC, blank=True, null=True) latest_query = models.DateTimeField(blank=True, null=True) times = ArrayField(models.DateTimeField()) @@ -291,7 +291,7 @@ class NIRISSEdbBlocksStats(models.Model): class Meta: managed = True - db_table = 'niriss_edb_blocks_stats' + db_table = 'niriss_edb_block_means_stats' unique_together = (('id', 'entry_date'),) @@ -360,7 +360,7 @@ class Meta: unique_together = (('id', 'entry_date'),) -class NIRSpecEdbBlocksStats(models.Model): +class NIRSpecEdbBlockMeansStats(models.Model): mnemonic = models.CharField(max_length=MAX_LEN_MNEMONIC, blank=True, null=True) latest_query = models.DateTimeField(blank=True, null=True) times = ArrayField(models.DateTimeField()) @@ -373,7 +373,7 @@ class NIRSpecEdbBlocksStats(models.Model): class Meta: managed = True - db_table = 'nirspec_edb_blocks_stats' + db_table = 'nirspec_edb_block_means_stats' unique_together = (('id', 'entry_date'),)