From 54b0f9f6d8ee19a0935426ac6574b965d68198bb Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 16 Aug 2024 15:12:29 -0400 Subject: [PATCH 1/9] Make time limit a float for more flexibility. Add dry_run parameter --- jwql/website/apps/jwql/clean_old_log_files.py | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/jwql/website/apps/jwql/clean_old_log_files.py b/jwql/website/apps/jwql/clean_old_log_files.py index dd0a6b95c..18991c931 100644 --- a/jwql/website/apps/jwql/clean_old_log_files.py +++ b/jwql/website/apps/jwql/clean_old_log_files.py @@ -38,12 +38,14 @@ def define_options(): """ usage = 'clean_old_log_files.py -t 14' parser = argparse.ArgumentParser(usage=usage) - parser.add_argument('-t', '--time_limit', type=int, default=14, + parser.add_argument('-t', '--time_limit', type=float, default=14, help='Time limit in days. Log files older than this will be deleted.') + parser.add_argument('-d', '--dry_run', action="store_true", + help='If True, the log files that would be deleted are printed to the screen') return parser -def run(time_limit=timedelta(days=14)): +def run(time_limit=timedelta(days=14), dry_run=False): """Look through log directories and delete log files that are older than ``time_limit``. Have time_limit default to be 14 days. @@ -51,6 +53,10 @@ def run(time_limit=timedelta(days=14)): ------ time_limit : datetime.timdelta Files older than this time limit will be deleted + + dry_run : bool + If True, log files will not be deleted. Those that would be deleted are instead + printed to the screen """ now = datetime.now() @@ -62,7 +68,8 @@ def run(time_limit=timedelta(days=14)): # This should cover the dev server as well as local machines subdir = 'dev' - log_dir = os.path.join(LOG_BASE_DIR, subdir) + #log_dir = os.path.join(LOG_BASE_DIR, subdir) + log_dir = '/ifs/jwst/wit/nircam/hilbert/jwql/test_logs/dev' for logtype in os.scandir(log_dir): if logtype.is_dir(): for item in os.scandir(logtype): @@ -75,10 +82,12 @@ def run(time_limit=timedelta(days=14)): age = now - last_modified_time if age > time_limit: full_path = os.path.join(log_dir, logtype, item) - os.remove(full_path) - + if not dry_run: + os.remove(full_path) + else: + print(f'DELETE: {full_path}') if __name__ == '__main__': parser = define_options() args = parser.parse_args() - run(timedelta(days=args.time_limit)) + run(time_limit=timedelta(days=args.time_limit), dry_run=args.dry_run) From 4f36a6bdba1f67e865b8b232cd37018918304950 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 16 Aug 2024 15:13:27 -0400 Subject: [PATCH 2/9] Make file executable, so it will work in crontab --- jwql/website/apps/jwql/clean_old_log_files.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 jwql/website/apps/jwql/clean_old_log_files.py diff --git a/jwql/website/apps/jwql/clean_old_log_files.py b/jwql/website/apps/jwql/clean_old_log_files.py old mode 100644 new mode 100755 From f5c08e3919d93f63399513c379d7b0f99c0a54ac Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 16 Aug 2024 15:17:35 -0400 Subject: [PATCH 3/9] Fix dirname --- jwql/website/apps/jwql/clean_old_log_files.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jwql/website/apps/jwql/clean_old_log_files.py b/jwql/website/apps/jwql/clean_old_log_files.py index 18991c931..07140e710 100755 --- a/jwql/website/apps/jwql/clean_old_log_files.py +++ b/jwql/website/apps/jwql/clean_old_log_files.py @@ -68,8 +68,7 @@ def run(time_limit=timedelta(days=14), dry_run=False): # This should cover the dev server as well as local machines subdir = 'dev' - #log_dir = os.path.join(LOG_BASE_DIR, subdir) - log_dir = '/ifs/jwst/wit/nircam/hilbert/jwql/test_logs/dev' + log_dir = os.path.join(LOG_BASE_DIR, subdir) for logtype in os.scandir(log_dir): if logtype.is_dir(): for item in os.scandir(logtype): From c66dd7bff8be88ebd97a0fd0df1c93def403279d Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 16 Aug 2024 15:21:12 -0400 Subject: [PATCH 4/9] PEP8 --- jwql/website/apps/jwql/clean_old_log_files.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jwql/website/apps/jwql/clean_old_log_files.py b/jwql/website/apps/jwql/clean_old_log_files.py index 07140e710..2d1f3ccaf 100755 --- a/jwql/website/apps/jwql/clean_old_log_files.py +++ b/jwql/website/apps/jwql/clean_old_log_files.py @@ -86,6 +86,7 @@ def run(time_limit=timedelta(days=14), dry_run=False): else: print(f'DELETE: {full_path}') + if __name__ == '__main__': parser = define_options() args = parser.parse_args() From c675f6dedee9276dc319830fcb6c0f9debfb0c0b Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Sat, 17 Aug 2024 18:04:00 -0400 Subject: [PATCH 5/9] Specify circle radii in Bokeh plots --- .../apps/jwql/monitor_pages/monitor_readnoise_bokeh.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py index d78e64de5..43f86358f 100644 --- a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py @@ -154,7 +154,7 @@ def plot_readnoise_amplifers(self): ("ngroups", "@ngroups"), ("readnoise", "@readnoise")])) - amp_plot.circle(x='expstarts', y='readnoise', source=source) + amp_plot.circle(x='expstarts', y='readnoise', radius=0.01, radius_dimension='y', source=source) amp_plot.xaxis.axis_label = 'Date' amp_plot.yaxis.axis_label = 'Mean Readnoise [DN]' @@ -204,7 +204,7 @@ def plot_readnoise_histogram(self): self.readnoise_histogram.add_tools(HoverTool(tooltips=[("Data (x, y)", "(@x, @y)"), ])) - self.readnoise_histogram.circle(x='x', y='y', source=source) + self.readnoise_histogram.circle(x='x', y='y', radius=0.01, radius_dimension='x', source=source) self.readnoise_histogram.xaxis.axis_label = 'Readnoise Difference [DN]' self.readnoise_histogram.yaxis.axis_label = 'Number of Pixels' From fe80b95eebaffaffcb8516cc0e28d124e910e458 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 19 Aug 2024 13:50:39 -0400 Subject: [PATCH 6/9] Fix plot ranges and circle radius values --- .../jwql/monitor_pages/monitor_readnoise_bokeh.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py index 43f86358f..c190db3ed 100644 --- a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py @@ -154,7 +154,15 @@ def plot_readnoise_amplifers(self): ("ngroups", "@ngroups"), ("readnoise", "@readnoise")])) - amp_plot.circle(x='expstarts', y='readnoise', radius=0.01, radius_dimension='y', source=source) + min_rn = np.min(readnoise_vals) + max_rn = np.max(readnoise_vals) + delta_rn = max_rn - min_rn + plot_max = max_rn + 0.5 * delta_rn + plot_min = min_rn - 0.5 * delta_rn + circle_radius = 0.01 * (plot_max - plot_min) + + amp_plot.circle(x='expstarts', y='readnoise', radius=circle_radius, radius_dimension='y', source=source, + y_range=(plot_min, plot_max)) amp_plot.xaxis.axis_label = 'Date' amp_plot.yaxis.axis_label = 'Mean Readnoise [DN]' @@ -204,7 +212,8 @@ def plot_readnoise_histogram(self): self.readnoise_histogram.add_tools(HoverTool(tooltips=[("Data (x, y)", "(@x, @y)"), ])) - self.readnoise_histogram.circle(x='x', y='y', radius=0.01, radius_dimension='x', source=source) + circle_radius = 0.01 * (hist_xr_end - hist_xr_start) + self.readnoise_histogram.circle(x='x', y='y', radius=circle_radius, radius_dimension='x', source=source) self.readnoise_histogram.xaxis.axis_label = 'Readnoise Difference [DN]' self.readnoise_histogram.yaxis.axis_label = 'Number of Pixels' From 414808ed322cf4ab0c811476b4a2aad1deaf5d3b Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Tue, 20 Aug 2024 12:11:08 -0400 Subject: [PATCH 7/9] Update NRS TA monitors to not save plot and generate them live. --- .../ta_monitors/msata_monitor.py | 26 +++++++++---------- .../ta_monitors/wata_monitor.py | 23 +++++++++------- jwql/website/apps/jwql/monitor_views.py | 18 ++++++------- jwql/website/apps/jwql/static/js/jwql.js | 6 ++--- 4 files changed, 36 insertions(+), 37 deletions(-) diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py index 3f0fade13..52484721a 100755 --- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py +++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py @@ -1431,7 +1431,6 @@ def mk_plt_layout(self, plot_data): self.setup_date_range() # set the output html file name and create the plot grid - output_file(self.output_file_name) p1 = self.plt_status() p2 = self.plt_residual_offsets() p3 = self.plt_res_offsets_corrected() @@ -1452,11 +1451,9 @@ def mk_plt_layout(self, plot_data): merge_tools=False, ) box_layout = layout(children=[self.date_range, grid]) - save(box_layout) # return the needed components for embedding the results in the MSATA html template - script, div = components(box_layout) - return script, div + self.script, self.div = components(box_layout) def identify_tables(self): """Determine which database tables to use for a run of the TA monitor.""" @@ -1720,6 +1717,17 @@ def add_msata_data(self): logging.info("\tUpdated the MSATA statistics table") + def plots_for_app(self): + """Utility function to access div and script objects for + embedding bokeh in JWQL application. + """ + # Query results and convert into pandas df. + self.query_results = pd.DataFrame( + list(NIRSpecMsataStats.objects.all().values()) + ) + # Generate plot + self.mk_plt_layout(self.query_results) + @log_fail @log_info def run(self): @@ -1743,7 +1751,6 @@ def run(self): # Locate the record of most recent time the monitor was run self.query_start = self.most_recent_search() - self.output_file_name = os.path.join(self.output_dir, "msata_layout.html") # Use the current time as the end time for MAST query self.query_end = Time.now().mjd @@ -1827,17 +1834,10 @@ def run(self): # Generate plot self.mk_plt_layout(self.query_results) - logging.info( - "\tNew output plot file will be written as: {}".format( - self.output_file_name - ) - ) # Once data is added to database table and plots are made, the # monitor has run successfully. monitor_run = True - logging.info( - "\tOutput html plot file created: {}".format(self.output_file_name) - ) + msata_files_used4plots = len(self.msata_data["visit_id"]) logging.info( "\t{} MSATA files were used to make plots.".format( diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py index a2f3301e5..95f903aea 100755 --- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py +++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py @@ -820,7 +820,6 @@ def mk_plt_layout(self, plot_data): self.setup_date_range() # set the output html file name and create the plot grid - output_file(self.output_file_name) p1 = self.plt_status() p2 = self.plt_residual_offsets() p3 = self.plt_v2offset_time() @@ -831,11 +830,9 @@ def mk_plt_layout(self, plot_data): # make grid grid = gridplot([p1, p2, p3, p4, p5, p6], ncols=2, merge_tools=False) box_layout = layout(children=[self.date_range, grid]) - save(box_layout) # return the needed components for embeding the results in the WATA html template - script, div = components(box_layout) - return script, div + self.script, self.div = components(box_layout) def file_exists_in_database(self, filename): """Checks if an entry for filename exists in the wata stats @@ -1052,6 +1049,17 @@ def add_wata_data(self): logging.info("\tUpdated the WATA statistics table") + def plots_for_app(self): + """Utility function to access div and script objects for + embedding bokeh in JWQL application. + """ + # Query results and convert into pandas df. + self.query_results = pd.DataFrame( + list(NIRSpecWataStats.objects.all().values()) + ) + # Generate plot + self.mk_plt_layout(self.query_results) + @log_fail @log_info def run(self): @@ -1075,7 +1083,6 @@ def run(self): # Locate the record of most recent time the monitor was run self.query_start = self.most_recent_search() - self.output_file_name = os.path.join(self.output_dir, "wata_layout.html") # Use the current time as the end time for MAST query self.query_end = Time.now().mjd @@ -1153,11 +1160,7 @@ def run(self): self.query_results = pd.DataFrame(list(NIRSpecWataStats.objects.all().values())) # Generate plot. self.mk_plt_layout(self.query_results) - logging.info( - "\tNew output plot file will be written as: {}".format( - self.output_file_name - ) - ) + # Once data is added to database table and plots are made, the # monitor has run successfully. monitor_run = True diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py index f5408dd89..b0943c54a 100644 --- a/jwql/website/apps/jwql/monitor_views.py +++ b/jwql/website/apps/jwql/monitor_views.py @@ -342,13 +342,12 @@ def msata_monitoring_ajax(request): JsonResponse object Outgoing response sent to the webpage """ - # retrieve existing monitor html content + # Make plots and extract visualization components monitor = msata_monitor.MSATA() - div, script1, script2 = monitor.read_existing_html() + monitor.plots_for_app() - context = {'script1': script1, - 'script2': script2, - 'div': div} + context = {'script': monitor.script, + 'div': monitor.div} return JsonResponse(context, json_dumps_params={'indent': 2}) @@ -391,12 +390,11 @@ def wata_monitoring_ajax(request): JsonResponse object Outgoing response sent to the webpage """ - # retrieve existing monitor html content + # Make plots and extract visualization components monitor = wata_monitor.WATA() - div, script1, script2 = monitor.read_existing_html() + monitor.plots_for_app() - context = {'script1': script1, - 'script2': script2, - 'div': div} + context = {'script': monitor.script, + 'div': monitor.div} return JsonResponse(context, json_dumps_params={'indent': 2}) diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index 96896f4c3..895da4fea 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -1042,8 +1042,7 @@ function update_msata_page(base_url) { // Build div content var content = data["div"]; - content += data["script1"]; - content += data["script2"]; + content += data["script"]; /* Add the content to the div * Note: