Merge branch 'develop' into filename-parser-no-exceptions

jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py

@@ -1431,7 +1431,6 @@ def mk_plt_layout(self, plot_data):
         self.setup_date_range()
 
         # set the output html file name and create the plot grid
-        output_file(self.output_file_name)
         p1 = self.plt_status()
         p2 = self.plt_residual_offsets()
         p3 = self.plt_res_offsets_corrected()
@@ -1452,11 +1451,8 @@ def mk_plt_layout(self, plot_data):
             merge_tools=False,
         )
         box_layout = layout(children=[self.date_range, grid])
-        save(box_layout)
 
-        # return the needed components for embedding the results in the MSATA html template
-        script, div = components(box_layout)
-        return script, div
+        self.script, self.div = components(box_layout)
 
     def identify_tables(self):
         """Determine which database tables to use for a run of the TA monitor."""
@@ -1720,6 +1716,17 @@ def add_msata_data(self):
 
         logging.info("\tUpdated the MSATA statistics table")
 
+    def plots_for_app(self):
+        """Utility function to access div and script objects for
+        embedding bokeh in JWQL application.
+        """
+        # Query results and convert into pandas df.
+        self.query_results = pd.DataFrame(
+            list(NIRSpecMsataStats.objects.all().values())
+        )
+        # Generate plot
+        self.mk_plt_layout(self.query_results)
+
     @log_fail
     @log_info
     def run(self):
@@ -1730,20 +1737,8 @@ def run(self):
         # Identify which database tables to use
         self.identify_tables()
 
-        # Get the output directory and setup a directory to store the data
-        self.output_dir = os.path.join(get_config()["outputs"], "msata_monitor")
-        ensure_dir_exists(self.output_dir)
-        # Set up directory to store the data
-        ensure_dir_exists(os.path.join(self.output_dir, "data"))
-        self.data_dir = os.path.join(
-            self.output_dir,
-            "data/{}_{}".format(self.instrument.lower(), self.aperture.lower()),
-        )
-        ensure_dir_exists(self.data_dir)
-
         # Locate the record of most recent time the monitor was run
         self.query_start = self.most_recent_search()
-        self.output_file_name = os.path.join(self.output_dir, "msata_layout.html")
 
         # Use the current time as the end time for MAST query
         self.query_end = Time.now().mjd
@@ -1819,25 +1814,10 @@ def run(self):
             # Add MSATA data to stats table.
             self.add_msata_data()
 
-            # Query results and convert into pandas df.
-            self.query_results = pd.DataFrame(
-                list(NIRSpecMsataStats.objects.all().values())
-            )
-
-            # Generate plot
-            self.mk_plt_layout(self.query_results)
-
-            logging.info(
-                "\tNew output plot file will be written as: {}".format(
-                    self.output_file_name
-                )
-            )
             # Once data is added to database table and plots are made, the
             # monitor has run successfully.
             monitor_run = True
-            logging.info(
-                "\tOutput html plot file created: {}".format(self.output_file_name)
-            )
+
             msata_files_used4plots = len(self.msata_data["visit_id"])
             logging.info(
                 "\t{} MSATA files were used to make plots.".format(

jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py

@@ -820,7 +820,6 @@ def mk_plt_layout(self, plot_data):
         self.setup_date_range()
 
         # set the output html file name and create the plot grid
-        output_file(self.output_file_name)
         p1 = self.plt_status()
         p2 = self.plt_residual_offsets()
         p3 = self.plt_v2offset_time()
@@ -831,11 +830,8 @@ def mk_plt_layout(self, plot_data):
         # make grid
         grid = gridplot([p1, p2, p3, p4, p5, p6], ncols=2, merge_tools=False)
         box_layout = layout(children=[self.date_range, grid])
-        save(box_layout)
 
-        # return the needed components for embeding the results in the WATA html template
-        script, div = components(box_layout)
-        return script, div
+        self.script, self.div = components(box_layout)
 
     def file_exists_in_database(self, filename):
         """Checks if an entry for filename exists in the wata stats
@@ -1052,6 +1048,17 @@ def add_wata_data(self):
 
         logging.info("\tUpdated the WATA statistics table")
 
+    def plots_for_app(self):
+        """Utility function to access div and script objects for
+        embedding bokeh in JWQL application.
+        """
+        # Query results and convert into pandas df.
+        self.query_results = pd.DataFrame(
+            list(NIRSpecWataStats.objects.all().values())
+        )
+        # Generate plot
+        self.mk_plt_layout(self.query_results)
+
     @log_fail
     @log_info
     def run(self):
@@ -1062,20 +1069,8 @@ def run(self):
         # Identify which database tables to use
         self.identify_tables()
 
-        # Get the output directory and setup a directory to store the data
-        self.output_dir = os.path.join(get_config()["outputs"], "wata_monitor")
-        ensure_dir_exists(self.output_dir)
-        # Set up directories for the copied data
-        ensure_dir_exists(os.path.join(self.output_dir, "data"))
-        self.data_dir = os.path.join(
-            self.output_dir,
-            "data/{}_{}".format(self.instrument.lower(), self.aperture.lower()),
-        )
-        ensure_dir_exists(self.data_dir)
-
         # Locate the record of most recent time the monitor was run
         self.query_start = self.most_recent_search()
-        self.output_file_name = os.path.join(self.output_dir, "wata_layout.html")
 
         # Use the current time as the end time for MAST query
         self.query_end = Time.now().mjd
@@ -1149,15 +1144,6 @@ def run(self):
             # Add WATA data to stats table.
             self.add_wata_data()
 
-            # Get Results from database table
-            self.query_results = pd.DataFrame(list(NIRSpecWataStats.objects.all().values()))
-            # Generate plot.
-            self.mk_plt_layout(self.query_results)
-            logging.info(
-                "\tNew output plot file will be written as: {}".format(
-                    self.output_file_name
-                )
-            )
             # Once data is added to database table and plots are made, the
             # monitor has run successfully.
             monitor_run = True

jwql/website/apps/jwql/clean_old_log_files.py

@@ -38,19 +38,25 @@ def define_options():
     """
     usage = 'clean_old_log_files.py -t 14'
     parser = argparse.ArgumentParser(usage=usage)
-    parser.add_argument('-t', '--time_limit', type=int, default=14,
+    parser.add_argument('-t', '--time_limit', type=float, default=14,
                         help='Time limit in days. Log files older than this will be deleted.')
+    parser.add_argument('-d', '--dry_run', action="store_true",
+                        help='If True, the log files that would be deleted are printed to the screen')
     return parser
 
 
-def run(time_limit=timedelta(days=14)):
+def run(time_limit=timedelta(days=14), dry_run=False):
     """Look through log directories and delete log files that are older than ``time_limit``.
     Have time_limit default to be 14 days.
 
     Inputs
     ------
     time_limit : datetime.timdelta
         Files older than this time limit will be deleted
+
+    dry_run : bool
+        If True, log files will not be deleted. Those that would be deleted are instead
+        printed to the screen
     """
     now = datetime.now()
 
@@ -75,10 +81,13 @@ def run(time_limit=timedelta(days=14)):
                     age = now - last_modified_time
                     if age > time_limit:
                         full_path = os.path.join(log_dir, logtype, item)
-                        os.remove(full_path)
+                        if not dry_run:
+                            os.remove(full_path)
+                        else:
+                            print(f'DELETE: {full_path}')
 
 
 if __name__ == '__main__':
     parser = define_options()
     args = parser.parse_args()
-    run(timedelta(days=args.time_limit))
+    run(time_limit=timedelta(days=args.time_limit), dry_run=args.dry_run)

jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py

@@ -154,7 +154,15 @@ def plot_readnoise_amplifers(self):
                                                    ("ngroups", "@ngroups"),
                                                    ("readnoise", "@readnoise")]))
 
-            amp_plot.circle(x='expstarts', y='readnoise', source=source)
+            min_rn = np.min(readnoise_vals)
+            max_rn = np.max(readnoise_vals)
+            delta_rn = max_rn - min_rn
+            plot_max = max_rn + 0.5 * delta_rn
+            plot_min = min_rn - 0.5 * delta_rn
+            circle_radius = 0.01 * (plot_max - plot_min)
+
+            amp_plot.circle(x='expstarts', y='readnoise', radius=circle_radius, radius_dimension='y', source=source,
+                            y_range=(plot_min, plot_max))
 
             amp_plot.xaxis.axis_label = 'Date'
             amp_plot.yaxis.axis_label = 'Mean Readnoise [DN]'
@@ -204,7 +212,8 @@ def plot_readnoise_histogram(self):
 
         self.readnoise_histogram.add_tools(HoverTool(tooltips=[("Data (x, y)", "(@x, @y)"), ]))
 
-        self.readnoise_histogram.circle(x='x', y='y', source=source)
+        circle_radius = 0.01 * (hist_xr_end - hist_xr_start)
+        self.readnoise_histogram.circle(x='x', y='y', radius=circle_radius, radius_dimension='x', source=source)
 
         self.readnoise_histogram.xaxis.axis_label = 'Readnoise Difference [DN]'
         self.readnoise_histogram.yaxis.axis_label = 'Number of Pixels'

jwql/website/apps/jwql/monitor_views.py

@@ -342,13 +342,12 @@ def msata_monitoring_ajax(request):
     JsonResponse object
         Outgoing response sent to the webpage
     """
-    # retrieve existing monitor html content
+    # Make plots and extract visualization components
     monitor = msata_monitor.MSATA()
-    div, script1, script2 = monitor.read_existing_html()
+    monitor.plots_for_app()
 
-    context = {'script1': script1,
-               'script2': script2,
-               'div': div}
+    context = {'script': monitor.script,
+               'div': monitor.div}
 
     return JsonResponse(context, json_dumps_params={'indent': 2})
 
@@ -391,12 +390,11 @@ def wata_monitoring_ajax(request):
     JsonResponse object
         Outgoing response sent to the webpage
     """
-    # retrieve existing monitor html content
+    # Make plots and extract visualization components
     monitor = wata_monitor.WATA()
-    div, script1, script2 = monitor.read_existing_html()
+    monitor.plots_for_app()
 
-    context = {'script1': script1,
-               'script2': script2,
-               'div': div}
+    context = {'script': monitor.script,
+               'div': monitor.div}
 
     return JsonResponse(context, json_dumps_params={'indent': 2})

jwql/website/apps/jwql/static/js/jwql.js

@@ -1042,8 +1042,7 @@ function update_msata_page(base_url) {
 
             // Build div content
             var content = data["div"];
-            content += data["script1"];
-            content += data["script2"];
+            content += data["script"];
 
             /* Add the content to the div
             *    Note: <script> elements inserted via innerHTML are intentionally disabled/ignored by the browser.  Directly inserting script via jquery.
@@ -1075,8 +1074,7 @@ function update_wata_page(base_url) {
 
             // Build div content
             var content = data["div"];
-            content += data["script1"];
-            content += data["script2"];
+            content += data["script"];
 
             /* Add the content to the div
             *    Note: <script> elements inserted via innerHTML are intentionally disabled/ignored by the browser.  Directly inserting script via jquery.