Merge pull request #118 from laurenmarietta/embed-bokeh-components

Update monitor plots to use logging and Bokeh components

jwql/logging/logging_functions.py

@@ -66,7 +66,7 @@ def my_main_function():
 from functools import wraps
 
 from jwql.permissions.permissions import set_permissions
-from jwql.utils.utils import get_config
+from jwql.utils.utils import get_config, ensure_dir_exists
 
 LOG_FILE_LOC = ''
 PRODUCTION_BOOL = ''
@@ -144,6 +144,8 @@ def make_log_file(module, production_mode=True, path='./'):
     else:
         log_file = os.path.join(path, filename)
 
+    ensure_dir_exists(os.path.dirname(log_file))
+
     return log_file
 
 

jwql/monitor_filesystem/monitor_filesystem.py

@@ -24,7 +24,6 @@
     import statements:
 
     ::
-
         from monitor_filesystem import filesystem_monitor
         from monitor_filesystem import plot_system_stats
 
@@ -51,7 +50,7 @@
 Notes
 -----
 
-    The ``filesystem_monitor`` function queries the filesystem,
+    The ``monitor_filesystem`` function queries the filesystem,
     calculates the statistics and saves the output file(s) in the
     directory specified in the ``config.json`` file.
 
@@ -67,24 +66,25 @@
 import os
 import subprocess
 
-from bokeh.plotting import figure, output_file, save
+from bokeh.embed import components
 from bokeh.layouts import gridplot
+from bokeh.plotting import figure, output_file, save
 
-from jwql.logging.logging_functions import configure_logging
-from jwql.logging.logging_functions import log_info
-from jwql.logging.logging_functions import log_fail
+from jwql.logging.logging_functions import configure_logging, log_info, log_fail
 from jwql.permissions.permissions import set_permissions
 from jwql.utils.utils import filename_parser
 from jwql.utils.utils import get_config
 
 
 @log_fail
 @log_info
-def filesystem_monitor():
-    """ Get statistics on filesystem"""
+def monitor_filesystem():
+    """Tabulates the inventory of the JWST filesystem, saving
+    statistics to files, and generates plots.
+    """
 
     # Begin logging
-    logging.info("Beginning the script run: ")
+    logging.info('Beginning filesystem monitoring.')
 
     # Get path, directories and files in system and count files in all directories
     settings = get_config()
@@ -95,6 +95,7 @@ def filesystem_monitor():
     results_dict = defaultdict(int)
     size_dict = defaultdict(float)
     # Walk through all directories recursively and count files
+    logging.info('Searching filesystem...')
     for dirpath, dirs, files in os.walk(filesystem):
         results_dict['file_count'] += len(files)  # find number of all files
         for filename in files:
@@ -109,6 +110,7 @@ def filesystem_monitor():
                 instrument = detector[0:3]  # first three characters of detector specify instrument
                 results_dict[instrument] += 1
                 size_dict[instrument] += os.path.getsize(file_path)
+    logging.info('{} files found in filesystem'.format(results_dict['fits_files']))
 
     # Get df style stats on file system
     out = subprocess.check_output('df {}'.format(filesystem), shell=True)
@@ -130,6 +132,7 @@ def filesystem_monitor():
         f.write("{0} {1:15d} {2:15d} {3:15d} {4:15d} {5}\n".format(now, results_dict['file_count'],
                 total, available, used, percent_used))
     set_permissions(statsfile)
+    logging.info('Saved file statistics to: {}'.format(statsfile))
 
     # set up and read out stats on files by type
     filesbytype = os.path.join(outputs_dir, 'filesbytype.txt')
@@ -139,6 +142,7 @@ def filesystem_monitor():
                  results_dict['rateints'], results_dict['i2d'], results_dict['nrc'],
                  results_dict['nrs'], results_dict['nis'], results_dict['mir'], results_dict['gui']))
     set_permissions(filesbytype, verbose=False)
+    logging.info('Saved file statistics by type to {}'.format(filesbytype))
 
     # set up file size by type file
     sizebytype = os.path.join(outputs_dir, 'sizebytype.txt')
@@ -148,6 +152,12 @@ def filesystem_monitor():
                  size_dict['rateints'], size_dict['i2d'], size_dict['nrc'],
                  size_dict['nrs'], size_dict['nis'], size_dict['mir'], size_dict['gui']))
     set_permissions(sizebytype, verbose=False)
+    logging.info('Saved file sizes by type to {}'.format(sizebytype))
+
+    logging.info('Filesystem statistics calculation complete.')
+
+    # Create the plots
+    plot_system_stats(statsfile, filesbytype, sizebytype)
 
 
 def plot_system_stats(stats_file, filebytype, sizebytype):
@@ -169,9 +179,10 @@ def plot_system_stats(stats_file, filebytype, sizebytype):
     outputs_dir = os.path.join(settings['outputs'], 'monitor_filesystem')
 
     # read in file of statistics
-    date, f_count, sysize, frsize, used, percent = np.loadtxt(os.path.join(outputs_dir, stats_file), dtype=str, unpack=True)
-    fits_files, uncalfiles, calfiles, ratefiles, rateintsfiles, i2dfiles, nrcfiles, nrsfiles, nisfiles, mirfiles, fgsfiles = np.loadtxt(os.path.join(outputs_dir, filebytype), dtype=str, unpack=True)
-    fits_sz, uncal_sz, cal_sz, rate_sz, rateints_sz, i2d_sz, nrc_sz, nrs_sz, nis_sz, mir_sz, fgs_sz = np.loadtxt(os.path.join(outputs_dir, sizebytype), dtype=str, unpack=True)
+    date, f_count, sysize, frsize, used, percent = np.loadtxt(stats_file, dtype=str, unpack=True)
+    fits_files, uncalfiles, calfiles, ratefiles, rateintsfiles, i2dfiles, nrcfiles, nrsfiles, nisfiles, mirfiles, fgsfiles = np.loadtxt(filebytype, dtype=str, unpack=True)
+    fits_sz, uncal_sz, cal_sz, rate_sz, rateints_sz, i2d_sz, nrc_sz, nrs_sz, nis_sz, mir_sz, fgs_sz = np.loadtxt(sizebytype, dtype=str, unpack=True)
+    logging.info('Read in file statistics from {}, {}, {}'.format(stats_file, filebytype, sizebytype))
 
     # put in proper np array types and convert to GB sizes
     dates = np.array(date, dtype='datetime64')
@@ -277,25 +288,45 @@ def plot_system_stats(stats_file, filebytype, sizebytype):
     p4.line(dates, fgs_size, legend='fgs fits files', line_color='darkred')
     p4.x(dates, fgs_size, color='darkred')
 
-    # create a layout with a grid pattern
+    # create a layout with a grid pattern to save all plots
     grid = gridplot([[p1, p2], [p3, p4]])
     outfile = os.path.join(outputs_dir, "filesystem_monitor.html")
     output_file(outfile)
     save(grid)
     set_permissions(outfile)
+    logging.info('Saved plot of all statistics to {}'.format(outfile))
+
+    # Save each plot's components
+    plots = [p1, p2, p3, p4]
+    plot_names = ['filecount', 'system_stats', 'filecount_type', 'size_type']
+    for plot, name in zip(plots, plot_names):
+        plot.sizing_mode = 'stretch_both'
+        script, div = components(plot)
+
+        div_outfile = os.path.join(outputs_dir, "{}_component.html".format(name))
+        with open(div_outfile, 'w') as f:
+            f.write(div)
+            f.close()
+        set_permissions(div_outfile)
+
+        script_outfile = os.path.join(outputs_dir, "{}_component.js".format(name))
+        with open(script_outfile, 'w') as f:
+            f.write(script)
+            f.close()
+        set_permissions(script_outfile)
+
+        logging.info('Saved components files: {}_component.html and {}_component.js'.format(name, name))
+
+    logging.info('Filesystem statistics plotting complete.')
 
     # Begin logging:
     logging.info("Completed.")
 
 
 if __name__ == '__main__':
 
-    inputfile = 'statsfile.txt'
-    filebytype = 'filesbytype.txt'
-    sizebytype = 'sizebytype.txt'
-
+    # Configure logging
     module = os.path.basename(__file__).strip('.py')
     configure_logging(module)
 
-    filesystem_monitor()
-    plot_system_stats(inputfile, filebytype, sizebytype)
+    monitor_filesystem()

jwql/monitor_mast/monitor_mast.py

@@ -16,14 +16,17 @@
         inventory, keywords = monitor_mast.jwst_inventory()
 """
 
+import logging
 import os
 
 from astroquery.mast import Mast
 from bokeh.charts import Donut, save, output_file
+from bokeh.embed import components
 import pandas as pd
 
-from ..permissions.permissions import set_permissions
-from ..utils.utils import get_config, JWST_DATAPRODUCTS, JWST_INSTRUMENTS
+from jwql.logging.logging_functions import configure_logging, log_info, log_fail
+from jwql.permissions.permissions import set_permissions
+from jwql.utils.utils import get_config, JWST_DATAPRODUCTS, JWST_INSTRUMENTS
 
 
 def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS,
@@ -157,6 +160,7 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS,
     astropy.table.table.Table
         The table of record counts for each instrument and mode
     """
+    logging.info('Searching database...')
     # Iterate through instruments
     inventory, keywords = [], {}
     for instrument in instruments:
@@ -174,6 +178,9 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS,
         # Add the keywords to the dict
         keywords[instrument] = instrument_keywords(instrument, caom=caom)
 
+    logging.info('Completed database search for {} instruments and {} data products.'.
+                 format(instruments, dataproducts))
+
     # Make the table
     all_cols = ['instrument']+dataproducts+['total']
     table = pd.DataFrame(inventory, columns=all_cols)
@@ -185,20 +192,73 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS,
 
     # Plot it
     if plot:
+        # Determine plot location and names
+        output_dir = get_config()['outputs']
+
+        if caom:
+            output_filename = 'database_monitor_caom'
+        else:
+            output_filename = 'database_monitor_jwst'
 
         # Make the plot
         plt = Donut(table, label=['instrument', 'dataproduct'], values='files',
                     text_font_size='12pt', hover_text='files',
                     name="JWST Inventory", plot_width=600, plot_height=600)
 
-        # Save the plot
-        if caom:
-            output_filename = 'database_monitor_caom.html'
-        else:
-            output_filename = 'database_monitor_jwst.html'
-        outfile = os.path.join(get_config()['outputs'], 'database_monitor', output_filename)
+        # Save the plot as full html
+        html_filename = output_filename + '.html'
+        outfile = os.path.join(output_dir, 'monitor_mast', html_filename)
         output_file(outfile)
         save(plt)
         set_permissions(outfile)
 
+        logging.info('Saved Bokeh plots as HTML file: {}'.format(html_filename))
+
+        # Save the plot as components
+        plt.sizing_mode = 'stretch_both'
+        script, div = components(plt)
+
+        div_outfile = os.path.join(output_dir, 'monitor_mast', output_filename + "_component.html")
+        with open(div_outfile, 'w') as f:
+            f.write(div)
+            f.close()
+        set_permissions(div_outfile)
+
+        script_outfile = os.path.join(output_dir, 'monitor_mast', output_filename + "_component.js")
+        with open(script_outfile, 'w') as f:
+            f.write(script)
+            f.close()
+        set_permissions(script_outfile)
+
+        logging.info('Saved Bokeh components files: {}_component.html and {}_component.js'.format(output_filename, output_filename))
+
     return table, keywords
+
+
+@log_fail
+@log_info
+def monitor_mast():
+    """Tabulates the inventory of all JWST data products in the MAST
+    archive and generates plots.
+    """
+    logging.info('Beginning database monitoring.')
+
+    # Perform inventory of the JWST service
+    jwst_inventory(instruments=JWST_INSTRUMENTS,
+                   dataproducts=['image', 'spectrum', 'cube'],
+                   caom=False, plot=True)
+
+    # Perform inventory of the CAOM service
+    jwst_inventory(instruments=JWST_INSTRUMENTS,
+                   dataproducts=['image', 'spectrum', 'cube'],
+                   caom=True, plot=True)
+
+
+if __name__ == '__main__':
+
+    # Configure logging
+    module = os.path.basename(__file__).strip('.py')
+    configure_logging(module)
+
+    # Run the monitors
+    monitor_mast()

jwql/utils/utils.py

@@ -25,6 +25,8 @@
 import os
 import re
 
+from ..permissions import permissions
+
 __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
 
 JWST_INSTRUMENTS = sorted(['NIRISS', 'NIRCam', 'NIRSpec', 'MIRI', 'FGS'])
@@ -46,6 +48,15 @@
                 'Internal Lamp Monitor', 'Instrument Model Updates',
                 'Failed-open Shutter Monitor']}
 
+
+def ensure_dir_exists(fullpath):
+    """Creates dirs from ``fullpath`` if they do not already exist.
+    """
+    if not os.path.exists(fullpath):
+        os.makedirs(fullpath)
+        permissions.set_permissions(fullpath)
+
+
 def get_config():
     """Return a dictionary that holds the contents of the ``jwql``
     config file.

website/apps/jwql/data_containers.py

@@ -80,10 +80,10 @@ def get_dashboard_components():
 
     output_dir = get_config()['outputs']
     name_dict = {'': '',
-                 'database_monitor': 'Database Monitor',
+                 'monitor_mast': 'Database Monitor',
                  'database_monitor_jwst': 'JWST',
                  'database_monitor_caom': 'JWST (CAOM)',
-                 'filesystem_monitor': 'Filesystem Monitor',
+                 'monitor_filesystem': 'Filesystem Monitor',
                  'filecount_type': 'Total File Counts by Type',
                  'size_type': 'Total File Sizes by Type',
                  'filecount': 'Total File Counts',

website/apps/jwql/views.py

@@ -155,7 +155,7 @@ def dashboard(request):
                'inst_list': JWST_INSTRUMENTS,
                'tools': MONITORS,
                'outputs': output_dir,
-               'filesystem_html': os.path.join(output_dir, 'filesystem_monitor', 'filesystem_monitor.html'),
+               'filesystem_html': os.path.join(output_dir, 'monitor_filesystem', 'filesystem_monitor.html'),
                'dashboard_components': dashboard_components}
 
     return render(request, template, context)