From 9eaea9aa2788577a76efd434bef66c1f0f199819 Mon Sep 17 00:00:00 2001 From: Lauren Chambers Date: Mon, 6 Aug 2018 20:23:24 -0400 Subject: [PATCH 01/14] First try to write out plots as bokeh components; add logging to filesystem_monitor.py --- jwql/monitor_filesystem/monitor_filesystem.py | 44 ++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/jwql/monitor_filesystem/monitor_filesystem.py b/jwql/monitor_filesystem/monitor_filesystem.py index d8131cfbc..254de91fb 100755 --- a/jwql/monitor_filesystem/monitor_filesystem.py +++ b/jwql/monitor_filesystem/monitor_filesystem.py @@ -63,15 +63,20 @@ import numpy as np import os import subprocess +import logging from bokeh.plotting import figure, output_file, save from bokeh.layouts import gridplot +from bokeh.embed import components from jwql.permissions.permissions import set_permissions from jwql.utils.utils import filename_parser from jwql.utils.utils import get_config +from jwql.logging.logging_functions import configure_logging, log_info, log_fail +@log_fail +@log_info def filesystem_monitor(): """ Get statistics on filesystem""" @@ -84,6 +89,7 @@ def filesystem_monitor(): results_dict = defaultdict(int) size_dict = defaultdict(float) # Walk through all directories recursively and count files + logging.info('Searching filesystem...') for dirpath, dirs, files in os.walk(filesystem): results_dict['file_count'] += len(files) # find number of all files for filename in files: @@ -98,11 +104,13 @@ def filesystem_monitor(): instrument = detector[0:3] # first three characters of detector specify instrument results_dict[instrument] += 1 size_dict[instrument] += os.path.getsize(file_path) + logging.info('{} files found in filesystem'.format(results_dict['fits_files'])) # Get df style stats on file system out = subprocess.check_output('df {}'.format(filesystem), shell=True) outstring = out.decode("utf-8") # put into string for parsing from byte format parsed = outstring.split(sep=None) + logging.info(parsed) # Select desired elements from parsed string total = int(parsed[8]) # in blocks of 512 bytes @@ -119,6 +127,8 @@ def filesystem_monitor(): f.write("{0} {1:15d} {2:15d} {3:15d} {4:15d} {5}\n".format(now, results_dict['file_count'], total, available, used, percent_used)) set_permissions(statsfile) + logging.info('Saved file statistics to:') + logging.info('{}'.format(statsfile)) # set up and read out stats on files by type filesbytype = os.path.join(outputs_dir, 'filesbytype.txt') @@ -128,6 +138,7 @@ def filesystem_monitor(): results_dict['rateints'], results_dict['i2d'], results_dict['nrc'], results_dict['nrs'], results_dict['nis'], results_dict['mir'], results_dict['gui'])) set_permissions(filesbytype, verbose=False) + logging.info('Saved file statistics by type to {}'.format(filesbytype)) # set up file size by type file sizebytype = os.path.join(outputs_dir, 'sizebytype.txt') @@ -137,8 +148,12 @@ def filesystem_monitor(): size_dict['rateints'], size_dict['i2d'], size_dict['nrc'], size_dict['nrs'], size_dict['nis'], size_dict['mir'], size_dict['gui'])) set_permissions(sizebytype, verbose=False) + logging.info('Saved file sizes by type to {}'.format(sizebytype)) + logging.info('Filesystem statistics calculation complete.') +@log_fail +@log_info def plot_system_stats(stats_file, filebytype, sizebytype): """Read in the file of saved stats over time and plot them. @@ -161,6 +176,7 @@ def plot_system_stats(stats_file, filebytype, sizebytype): date, f_count, sysize, frsize, used, percent = np.loadtxt(os.path.join(outputs_dir, stats_file), dtype=str, unpack=True) fits_files, uncalfiles, calfiles, ratefiles, rateintsfiles, i2dfiles, nrcfiles, nrsfiles, nisfiles, mirfiles, fgsfiles = np.loadtxt(os.path.join(outputs_dir, filebytype), dtype=str, unpack=True) fits_sz, uncal_sz, cal_sz, rate_sz, rateints_sz, i2d_sz, nrc_sz, nrs_sz, nis_sz, mir_sz, fgs_sz = np.loadtxt(os.path.join(outputs_dir, sizebytype), dtype=str, unpack=True) + logging.info('Read in file statistics from {}, {}, {}'.format(stats_file, filebytype, sizebytype)) # put in proper np array types and convert to GB sizes dates = np.array(date, dtype='datetime64') @@ -266,19 +282,45 @@ def plot_system_stats(stats_file, filebytype, sizebytype): p4.line(dates, fgs_size, legend='fgs fits files', line_color='darkred') p4.x(dates, fgs_size, color='darkred') - # create a layout with a grid pattern + # create a layout with a grid pattern to save all plots grid = gridplot([[p1, p2], [p3, p4]]) outfile = os.path.join(outputs_dir, "filesystem_monitor.html") output_file(outfile) save(grid) set_permissions(outfile) + logging.info('Saved plot of all statistics to {}'.format(outfile)) + + # Save each plot's components + plots = [p1, p2, p3, p4] + plot_names = ['filecount', 'system_stats', 'filecount_type', 'size_type'] + for plot, name in zip(plots, plot_names): + script, div = components(plot) + + div_outfile = os.path.join(outputs_dir, "{}_component.html".format(name)) + with open(div_outfile, 'w') as f: + f.write(div) + f.close() + + script_outfile = os.path.join(outputs_dir, "{}_component.js".format(name)) + with open(script_outfile, 'w') as f: + f.write(script) + f.close() + + logging.info('Saved components files: {}, {}'.format(div_outfile, script_outfile)) + + logging.info('Filesystem statistics plotting complete.') if __name__ == '__main__': + # Configure logging + module = os.path.basename(__file__).strip('.py') + configure_logging(module) + inputfile = 'statsfile.txt' filebytype = 'filesbytype.txt' sizebytype = 'sizebytype.txt' + logging.info('Beginning filesystem monitoring.') filesystem_monitor() plot_system_stats(inputfile, filebytype, sizebytype) From b542c721dba0808472df4695981cfc113d158264 Mon Sep 17 00:00:00 2001 From: Lauren Chambers Date: Mon, 6 Aug 2018 20:35:38 -0400 Subject: [PATCH 02/14] Update logging output --- jwql/monitor_filesystem/monitor_filesystem.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/jwql/monitor_filesystem/monitor_filesystem.py b/jwql/monitor_filesystem/monitor_filesystem.py index 254de91fb..fff644614 100755 --- a/jwql/monitor_filesystem/monitor_filesystem.py +++ b/jwql/monitor_filesystem/monitor_filesystem.py @@ -110,7 +110,6 @@ def filesystem_monitor(): out = subprocess.check_output('df {}'.format(filesystem), shell=True) outstring = out.decode("utf-8") # put into string for parsing from byte format parsed = outstring.split(sep=None) - logging.info(parsed) # Select desired elements from parsed string total = int(parsed[8]) # in blocks of 512 bytes @@ -127,8 +126,7 @@ def filesystem_monitor(): f.write("{0} {1:15d} {2:15d} {3:15d} {4:15d} {5}\n".format(now, results_dict['file_count'], total, available, used, percent_used)) set_permissions(statsfile) - logging.info('Saved file statistics to:') - logging.info('{}'.format(statsfile)) + logging.info('Saved file statistics to: {}'.format(statsfile)) # set up and read out stats on files by type filesbytype = os.path.join(outputs_dir, 'filesbytype.txt') @@ -306,7 +304,7 @@ def plot_system_stats(stats_file, filebytype, sizebytype): f.write(script) f.close() - logging.info('Saved components files: {}, {}'.format(div_outfile, script_outfile)) + logging.info('Saved components files: {}_component.html and {}_component.js'.format(name, name)) logging.info('Filesystem statistics plotting complete.') From d63060522e9065fa3127eedb4bc176d7e1ae2e68 Mon Sep 17 00:00:00 2001 From: Lauren Chambers Date: Mon, 6 Aug 2018 21:49:55 -0400 Subject: [PATCH 03/14] Add responsive sizing --- jwql/monitor_filesystem/monitor_filesystem.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jwql/monitor_filesystem/monitor_filesystem.py b/jwql/monitor_filesystem/monitor_filesystem.py index fff644614..8759e7d53 100755 --- a/jwql/monitor_filesystem/monitor_filesystem.py +++ b/jwql/monitor_filesystem/monitor_filesystem.py @@ -292,6 +292,7 @@ def plot_system_stats(stats_file, filebytype, sizebytype): plots = [p1, p2, p3, p4] plot_names = ['filecount', 'system_stats', 'filecount_type', 'size_type'] for plot, name in zip(plots, plot_names): + plot.sizing_mode = 'stretch_both' script, div = components(plot) div_outfile = os.path.join(outputs_dir, "{}_component.html".format(name)) From 8473a1ff8e27f19b7ca961443d968223eba80530 Mon Sep 17 00:00:00 2001 From: Lauren Chambers Date: Tue, 7 Aug 2018 11:51:00 -0400 Subject: [PATCH 04/14] Add logging and bokeh components to monitor_mast --- jwql/monitor_filesystem/monitor_filesystem.py | 2 + jwql/monitor_mast/monitor_mast.py | 66 ++++++++++++++++--- 2 files changed, 60 insertions(+), 8 deletions(-) diff --git a/jwql/monitor_filesystem/monitor_filesystem.py b/jwql/monitor_filesystem/monitor_filesystem.py index 8759e7d53..2727bb570 100755 --- a/jwql/monitor_filesystem/monitor_filesystem.py +++ b/jwql/monitor_filesystem/monitor_filesystem.py @@ -299,11 +299,13 @@ def plot_system_stats(stats_file, filebytype, sizebytype): with open(div_outfile, 'w') as f: f.write(div) f.close() + set_permissions(div_outfile) script_outfile = os.path.join(outputs_dir, "{}_component.js".format(name)) with open(script_outfile, 'w') as f: f.write(script) f.close() + set_permissions(script_outfile) logging.info('Saved components files: {}_component.html and {}_component.js'.format(name, name)) diff --git a/jwql/monitor_mast/monitor_mast.py b/jwql/monitor_mast/monitor_mast.py index d74c7ec58..f7ee236d9 100644 --- a/jwql/monitor_mast/monitor_mast.py +++ b/jwql/monitor_mast/monitor_mast.py @@ -17,13 +17,17 @@ """ import os +import logging + from astroquery.mast import Mast from bokeh.charts import Donut, save, output_file +from bokeh.embed import components import pandas as pd -from ..permissions.permissions import set_permissions -from ..utils.utils import get_config, JWST_DATAPRODUCTS, JWST_INSTRUMENTS +from jwql.permissions.permissions import set_permissions +from jwql.utils.utils import get_config, JWST_DATAPRODUCTS, JWST_INSTRUMENTS +from jwql.logging.logging_functions import configure_logging, log_info, log_fail def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, @@ -135,6 +139,8 @@ def instrument_keywords(instrument, caom=False): return keywords +@log_fail +@log_info def jwst_inventory(instruments=JWST_INSTRUMENTS, dataproducts=['image', 'spectrum', 'cube'], caom=False, plot=False): @@ -157,6 +163,7 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, astropy.table.table.Table The table of record counts for each instrument and mode """ + logging.info('Searching database...') # Iterate through instruments inventory, keywords = [], {} for instrument in instruments: @@ -174,6 +181,9 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, # Add the keywords to the dict keywords[instrument] = instrument_keywords(instrument, caom=caom) + logging.info('Completed database search for {} instruments and {} data products.'. + format(instruments, dataproducts)) + # Make the table all_cols = ['instrument']+dataproducts+['total'] table = pd.DataFrame(inventory, columns=all_cols) @@ -185,20 +195,60 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, # Plot it if plot: + # Determine plot location and names + output_dir = get_config()['outputs'] + + if caom: + output_filename = 'database_monitor_caom' + else: + output_filename = 'database_monitor_jwst' # Make the plot plt = Donut(table, label=['instrument', 'dataproduct'], values='files', text_font_size='12pt', hover_text='files', name="JWST Inventory", plot_width=600, plot_height=600) - # Save the plot - if caom: - output_filename = 'database_monitor_caom.html' - else: - output_filename = 'database_monitor_jwst.html' - outfile = os.path.join(get_config()['outputs'], 'database_monitor', output_filename) + # Save the plot as full html + html_filename = output_filename + '.html' + outfile = os.path.join(output_dir, 'database_monitor', html_filename) output_file(outfile) save(plt) set_permissions(outfile) + logging.info('Saved Bokeh plots as HTML file: ', html_filename) + + # Save the plot as components + plt.sizing_mode = 'stretch_both' + script, div = components(plt) + + div_outfile = os.path.join(output_dir, output_filename + "_component.html") + with open(div_outfile, 'w') as f: + f.write(div) + f.close() + set_permissions(div_outfile) + + script_outfile = os.path.join(output_dir, output_filename + "_component.js") + with open(script_outfile, 'w') as f: + f.write(script) + f.close() + set_permissions(script_outfile) + + logging.info('Saved Bokeh components files: {}_component.html and {}_component.js'.format(output_filename, output_filename)) + return table, keywords + + +if __name__ == '__main__': + + # Configure logging + module = os.path.basename(__file__).strip('.py') + configure_logging(module) + + # Run the monitors + logging.info('Beginning database monitoring.') + jwst_inventory(instruments=JWST_INSTRUMENTS, + dataproducts=['image', 'spectrum', 'cube'], + caom=False, plot=True) + jwst_inventory(instruments=JWST_INSTRUMENTS, + dataproducts=['image', 'spectrum', 'cube'], + caom=True, plot=True) From d9b2a4b69cae823b728f036ef72441d124467df4 Mon Sep 17 00:00:00 2001 From: Lauren Chambers Date: Tue, 7 Aug 2018 11:57:11 -0400 Subject: [PATCH 05/14] Add ensure_dir_exists function to utils --- jwql/logging/logging_functions.py | 4 +++- jwql/utils/utils.py | 11 +++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/jwql/logging/logging_functions.py b/jwql/logging/logging_functions.py index d4c79f6e1..13e52e776 100644 --- a/jwql/logging/logging_functions.py +++ b/jwql/logging/logging_functions.py @@ -66,7 +66,7 @@ def my_main_function(): from functools import wraps from jwql.permissions.permissions import set_permissions -from jwql.utils.utils import get_config +from jwql.utils.utils import get_config, ensure_dir_exists LOG_FILE_LOC = '' PRODUCTION_BOOL = '' @@ -142,6 +142,8 @@ def make_log_file(module, production_mode=True, path='./'): else: log_file = os.path.join(path, filename) + ensure_dir_exists(log_file) + return log_file diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index 2249951f3..773a827df 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -25,12 +25,23 @@ import os import re +from ..permissions import permissions + __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) JWST_INSTRUMENTS = ['NIRISS', 'NIRCam', 'NIRSpec', 'MIRI', 'FGS'] JWST_DATAPRODUCTS = ['IMAGE', 'SPECTRUM', 'SED', 'TIMESERIES', 'VISIBILITY', 'EVENTLIST', 'CUBE', 'CATALOG', 'ENGINEERING', 'NULL'] + +def ensure_dir_exists(fullpath): + """Creates dirs from ``fullpath`` if they do not already exist. + """ + if not os.path.exists(fullpath): + os.makedirs(fullpath) + permissions.set_permissions(fullpath) + + def get_config(): """Return a dictionary that holds the contents of the ``jwql`` config file. From acca5c76ae79a3cd90fecc2581b62914068d13e1 Mon Sep 17 00:00:00 2001 From: Lauren Chambers Date: Tue, 7 Aug 2018 12:01:32 -0400 Subject: [PATCH 06/14] Fix implementation of ensure_dir_exists --- jwql/logging/logging_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/logging/logging_functions.py b/jwql/logging/logging_functions.py index 13e52e776..d8f2dda80 100644 --- a/jwql/logging/logging_functions.py +++ b/jwql/logging/logging_functions.py @@ -142,7 +142,7 @@ def make_log_file(module, production_mode=True, path='./'): else: log_file = os.path.join(path, filename) - ensure_dir_exists(log_file) + ensure_dir_exists(os.path.dirname(log_file)) return log_file From 7ab40b3404fb3a4cbc6b85721fef2570027b4fee Mon Sep 17 00:00:00 2001 From: Lauren Chambers Date: Tue, 7 Aug 2018 12:20:52 -0400 Subject: [PATCH 07/14] Fix logging and component output location --- jwql/monitor_mast/monitor_mast.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jwql/monitor_mast/monitor_mast.py b/jwql/monitor_mast/monitor_mast.py index f7ee236d9..c49650139 100644 --- a/jwql/monitor_mast/monitor_mast.py +++ b/jwql/monitor_mast/monitor_mast.py @@ -215,19 +215,19 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, save(plt) set_permissions(outfile) - logging.info('Saved Bokeh plots as HTML file: ', html_filename) + logging.info('Saved Bokeh plots as HTML file: {}'.format(html_filename)) # Save the plot as components plt.sizing_mode = 'stretch_both' script, div = components(plt) - div_outfile = os.path.join(output_dir, output_filename + "_component.html") + div_outfile = os.path.join(output_dir, 'database_monitor', output_filename + "_component.html") with open(div_outfile, 'w') as f: f.write(div) f.close() set_permissions(div_outfile) - script_outfile = os.path.join(output_dir, output_filename + "_component.js") + script_outfile = os.path.join(output_dir, 'database_monitor', output_filename + "_component.js") with open(script_outfile, 'w') as f: f.write(script) f.close() From 0cd1dccc0c5756ab290bccc4cbda1a2ccf81afd6 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Fri, 17 Aug 2018 14:12:58 -0400 Subject: [PATCH 08/14] Renamed filesystem_monitor function to monitor_filesystem so that it can act as a main function to the module. Placed plot_system_stats call within main function. Removed logging decorators around plotting function as they are no longer needed. --- jwql/monitor_filesystem/monitor_filesystem.py | 41 +++++++++---------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/jwql/monitor_filesystem/monitor_filesystem.py b/jwql/monitor_filesystem/monitor_filesystem.py index 2727bb570..c5906d53e 100755 --- a/jwql/monitor_filesystem/monitor_filesystem.py +++ b/jwql/monitor_filesystem/monitor_filesystem.py @@ -23,7 +23,7 @@ import statements: :: - from monitor_filesystem import filesystem_monitor + from monitor_filesystem import monitor_filesystem from monitor_filesystem import plot_system_stats @@ -49,7 +49,7 @@ Notes ----- - The ``filesystem_monitor`` function queries the filesystem, + The ``monitor_filesystem`` function queries the filesystem, calculates the statistics and saves the output file(s) in the directory specified in the ``config.json`` file. @@ -60,30 +60,30 @@ from collections import defaultdict import datetime +import logging import numpy as np import os import subprocess -import logging -from bokeh.plotting import figure, output_file, save -from bokeh.layouts import gridplot from bokeh.embed import components +from bokeh.layouts import gridplot +from bokeh.plotting import figure, output_file, save +from jwql.logging.logging_functions import configure_logging, log_info, log_fail from jwql.permissions.permissions import set_permissions from jwql.utils.utils import filename_parser from jwql.utils.utils import get_config -from jwql.logging.logging_functions import configure_logging, log_info, log_fail @log_fail @log_info -def filesystem_monitor(): - """ Get statistics on filesystem""" +def monitor_filesystem(): + """The main function of the ``monitor_filesystem`` module.""" # Get path, directories and files in system and count files in all directories settings = get_config() filesystem = settings['filesystem'] - outputs_dir = os.path.join(settings['outputs'], 'filesystem_monitor') + outputs_dir = os.path.join(settings['outputs'], 'monitor_filesystem') # set up dictionaries for output results_dict = defaultdict(int) @@ -150,8 +150,10 @@ def filesystem_monitor(): logging.info('Filesystem statistics calculation complete.') -@log_fail -@log_info + # Create the plots + plot_system_stats(statsfile, filesbytype, sizebytype) + + def plot_system_stats(stats_file, filebytype, sizebytype): """Read in the file of saved stats over time and plot them. @@ -168,12 +170,12 @@ def plot_system_stats(stats_file, filebytype, sizebytype): # get path for files settings = get_config() - outputs_dir = os.path.join(settings['outputs'], 'filesystem_monitor') + outputs_dir = os.path.join(settings['outputs'], 'monitor_filesystem') # read in file of statistics - date, f_count, sysize, frsize, used, percent = np.loadtxt(os.path.join(outputs_dir, stats_file), dtype=str, unpack=True) - fits_files, uncalfiles, calfiles, ratefiles, rateintsfiles, i2dfiles, nrcfiles, nrsfiles, nisfiles, mirfiles, fgsfiles = np.loadtxt(os.path.join(outputs_dir, filebytype), dtype=str, unpack=True) - fits_sz, uncal_sz, cal_sz, rate_sz, rateints_sz, i2d_sz, nrc_sz, nrs_sz, nis_sz, mir_sz, fgs_sz = np.loadtxt(os.path.join(outputs_dir, sizebytype), dtype=str, unpack=True) + date, f_count, sysize, frsize, used, percent = np.loadtxt(stats_file, dtype=str, unpack=True) + fits_files, uncalfiles, calfiles, ratefiles, rateintsfiles, i2dfiles, nrcfiles, nrsfiles, nisfiles, mirfiles, fgsfiles = np.loadtxt(filebytype, dtype=str, unpack=True) + fits_sz, uncal_sz, cal_sz, rate_sz, rateints_sz, i2d_sz, nrc_sz, nrs_sz, nis_sz, mir_sz, fgs_sz = np.loadtxt(sizebytype, dtype=str, unpack=True) logging.info('Read in file statistics from {}, {}, {}'.format(stats_file, filebytype, sizebytype)) # put in proper np array types and convert to GB sizes @@ -282,7 +284,7 @@ def plot_system_stats(stats_file, filebytype, sizebytype): # create a layout with a grid pattern to save all plots grid = gridplot([[p1, p2], [p3, p4]]) - outfile = os.path.join(outputs_dir, "filesystem_monitor.html") + outfile = os.path.join(outputs_dir, "monitor_filesystem.html") output_file(outfile) save(grid) set_permissions(outfile) @@ -318,10 +320,5 @@ def plot_system_stats(stats_file, filebytype, sizebytype): module = os.path.basename(__file__).strip('.py') configure_logging(module) - inputfile = 'statsfile.txt' - filebytype = 'filesbytype.txt' - sizebytype = 'sizebytype.txt' - logging.info('Beginning filesystem monitoring.') - filesystem_monitor() - plot_system_stats(inputfile, filebytype, sizebytype) + monitor_filesystem() From 823e775871c3844858bfe4afec2a2b8ed270a2ae Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Fri, 17 Aug 2018 14:32:56 -0400 Subject: [PATCH 09/14] Updated website code to reflect new location of monitor_filesystem outputs. --- jwql/monitor_filesystem/monitor_filesystem.py | 2 +- website/apps/jwql/data_containers.py | 2 +- website/apps/jwql/views.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/jwql/monitor_filesystem/monitor_filesystem.py b/jwql/monitor_filesystem/monitor_filesystem.py index c5906d53e..997c42327 100755 --- a/jwql/monitor_filesystem/monitor_filesystem.py +++ b/jwql/monitor_filesystem/monitor_filesystem.py @@ -284,7 +284,7 @@ def plot_system_stats(stats_file, filebytype, sizebytype): # create a layout with a grid pattern to save all plots grid = gridplot([[p1, p2], [p3, p4]]) - outfile = os.path.join(outputs_dir, "monitor_filesystem.html") + outfile = os.path.join(outputs_dir, "filesystem_monitor.html") output_file(outfile) save(grid) set_permissions(outfile) diff --git a/website/apps/jwql/data_containers.py b/website/apps/jwql/data_containers.py index 67f795cf6..f0c388332 100644 --- a/website/apps/jwql/data_containers.py +++ b/website/apps/jwql/data_containers.py @@ -83,7 +83,7 @@ def get_dashboard_components(): 'database_monitor': 'Database Monitor', 'database_monitor_jwst': 'JWST', 'database_monitor_caom': 'JWST (CAOM)', - 'filesystem_monitor': 'Filesystem Monitor', + 'monitor_filesystem': 'Filesystem Monitor', 'filecount_type': 'Total File Counts by Type', 'size_type': 'Total File Sizes by Type', 'filecount': 'Total File Counts', diff --git a/website/apps/jwql/views.py b/website/apps/jwql/views.py index 12a4ea780..b778658b2 100644 --- a/website/apps/jwql/views.py +++ b/website/apps/jwql/views.py @@ -155,7 +155,7 @@ def dashboard(request): 'inst_list': JWST_INSTRUMENTS, 'tools': MONITORS, 'outputs': output_dir, - 'filesystem_html': os.path.join(output_dir, 'filesystem_monitor', 'filesystem_monitor.html'), + 'filesystem_html': os.path.join(output_dir, 'monitor_filesystem', 'filesystem_monitor.html'), 'dashboard_components': dashboard_components} return render(request, template, context) From 910f2be3d7810ede02042b9201e96c70dd7a0f0b Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Fri, 17 Aug 2018 14:33:29 -0400 Subject: [PATCH 10/14] Created a main function as to enable more efficient logging. Also changed the directory to where output files are saved to be more consistent with naming scheme. --- jwql/monitor_mast/monitor_mast.py | 36 +++++++++++++++++++------------ 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/jwql/monitor_mast/monitor_mast.py b/jwql/monitor_mast/monitor_mast.py index c49650139..569daaef6 100644 --- a/jwql/monitor_mast/monitor_mast.py +++ b/jwql/monitor_mast/monitor_mast.py @@ -16,18 +16,17 @@ inventory, keywords = monitor_mast.jwst_inventory() """ -import os import logging - +import os from astroquery.mast import Mast from bokeh.charts import Donut, save, output_file from bokeh.embed import components import pandas as pd +from jwql.logging.logging_functions import configure_logging, log_info, log_fail from jwql.permissions.permissions import set_permissions from jwql.utils.utils import get_config, JWST_DATAPRODUCTS, JWST_INSTRUMENTS -from jwql.logging.logging_functions import configure_logging, log_info, log_fail def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, @@ -139,8 +138,6 @@ def instrument_keywords(instrument, caom=False): return keywords -@log_fail -@log_info def jwst_inventory(instruments=JWST_INSTRUMENTS, dataproducts=['image', 'spectrum', 'cube'], caom=False, plot=False): @@ -210,7 +207,7 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, # Save the plot as full html html_filename = output_filename + '.html' - outfile = os.path.join(output_dir, 'database_monitor', html_filename) + outfile = os.path.join(output_dir, 'monitor_mast', html_filename) output_file(outfile) save(plt) set_permissions(outfile) @@ -221,13 +218,13 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, plt.sizing_mode = 'stretch_both' script, div = components(plt) - div_outfile = os.path.join(output_dir, 'database_monitor', output_filename + "_component.html") + div_outfile = os.path.join(output_dir, 'monitor_mast', output_filename + "_component.html") with open(div_outfile, 'w') as f: f.write(div) f.close() set_permissions(div_outfile) - script_outfile = os.path.join(output_dir, 'database_monitor', output_filename + "_component.js") + script_outfile = os.path.join(output_dir, 'monitor_mast', output_filename + "_component.js") with open(script_outfile, 'w') as f: f.write(script) f.close() @@ -238,6 +235,22 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, return table, keywords +@log_fail +@log_info +def monitor_mast(): + """The main function of the ``monitor_mast`` module.""" + + # Perform inventory of the JWST service + jwst_inventory(instruments=JWST_INSTRUMENTS, + dataproducts=['image', 'spectrum', 'cube'], + caom=False, plot=True) + + # Perform inventory of the CAOM service + jwst_inventory(instruments=JWST_INSTRUMENTS, + dataproducts=['image', 'spectrum', 'cube'], + caom=True, plot=True) + + if __name__ == '__main__': # Configure logging @@ -246,9 +259,4 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, # Run the monitors logging.info('Beginning database monitoring.') - jwst_inventory(instruments=JWST_INSTRUMENTS, - dataproducts=['image', 'spectrum', 'cube'], - caom=False, plot=True) - jwst_inventory(instruments=JWST_INSTRUMENTS, - dataproducts=['image', 'spectrum', 'cube'], - caom=True, plot=True) + From 0fca28b37353ccaef86472542b0a91b82b286e7c Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Fri, 17 Aug 2018 14:39:10 -0400 Subject: [PATCH 11/14] Updated website code to reflect new location of monitor_mast outputs. --- website/apps/jwql/data_containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/apps/jwql/data_containers.py b/website/apps/jwql/data_containers.py index f0c388332..477d597a4 100644 --- a/website/apps/jwql/data_containers.py +++ b/website/apps/jwql/data_containers.py @@ -80,7 +80,7 @@ def get_dashboard_components(): output_dir = get_config()['outputs'] name_dict = {'': '', - 'database_monitor': 'Database Monitor', + 'monitor_mast': 'Database Monitor', 'database_monitor_jwst': 'JWST', 'database_monitor_caom': 'JWST (CAOM)', 'monitor_filesystem': 'Filesystem Monitor', From 883c020a4e561ef43c80889add1ec72a36a51048 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Fri, 17 Aug 2018 14:51:33 -0400 Subject: [PATCH 12/14] Added call to the main function --- jwql/monitor_mast/monitor_mast.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jwql/monitor_mast/monitor_mast.py b/jwql/monitor_mast/monitor_mast.py index 569daaef6..c692fb24c 100644 --- a/jwql/monitor_mast/monitor_mast.py +++ b/jwql/monitor_mast/monitor_mast.py @@ -259,4 +259,5 @@ def monitor_mast(): # Run the monitors logging.info('Beginning database monitoring.') + monitor_mast() From 2563304f89e8dd2cac01b75ea6247577a64a624e Mon Sep 17 00:00:00 2001 From: Lauren Chambers Date: Fri, 24 Aug 2018 09:30:57 -0400 Subject: [PATCH 13/14] Complete resolving merge conflict --- jwql/monitor_filesystem/monitor_filesystem.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/jwql/monitor_filesystem/monitor_filesystem.py b/jwql/monitor_filesystem/monitor_filesystem.py index ac197f700..6712f521c 100755 --- a/jwql/monitor_filesystem/monitor_filesystem.py +++ b/jwql/monitor_filesystem/monitor_filesystem.py @@ -24,14 +24,8 @@ import statements: :: - -<<<<<<< HEAD from monitor_filesystem import filesystem_monitor from monitor_filesystem import plot_system_stats -======= - from monitor_filesystem import monitor_filesystem - from monitor_filesystem import plot_system_stats ->>>>>>> 5095b34a45cfe52c098465f2f18ed8dc6bf598fe Required arguments (in a ``config.json`` file): From 9732fea397c0cb6f7b3829bd3cf6bcb9795d9794 Mon Sep 17 00:00:00 2001 From: Lauren Chambers Date: Fri, 24 Aug 2018 09:36:39 -0400 Subject: [PATCH 14/14] Updating docstrings from RE: Joe's review --- jwql/monitor_filesystem/monitor_filesystem.py | 4 +++- jwql/monitor_mast/monitor_mast.py | 7 ++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/jwql/monitor_filesystem/monitor_filesystem.py b/jwql/monitor_filesystem/monitor_filesystem.py index 6712f521c..f798dc552 100755 --- a/jwql/monitor_filesystem/monitor_filesystem.py +++ b/jwql/monitor_filesystem/monitor_filesystem.py @@ -79,7 +79,9 @@ @log_fail @log_info def monitor_filesystem(): - """The main function of the ``monitor_filesystem`` module.""" + """Tabulates the inventory of the JWST filesystem, saving + statistics to files, and generates plots. + """ # Begin logging logging.info('Beginning filesystem monitoring.') diff --git a/jwql/monitor_mast/monitor_mast.py b/jwql/monitor_mast/monitor_mast.py index c692fb24c..043b7f8c9 100644 --- a/jwql/monitor_mast/monitor_mast.py +++ b/jwql/monitor_mast/monitor_mast.py @@ -238,7 +238,10 @@ def jwst_inventory(instruments=JWST_INSTRUMENTS, @log_fail @log_info def monitor_mast(): - """The main function of the ``monitor_mast`` module.""" + """Tabulates the inventory of all JWST data products in the MAST + archive and generates plots. + """ + logging.info('Beginning database monitoring.') # Perform inventory of the JWST service jwst_inventory(instruments=JWST_INSTRUMENTS, @@ -258,6 +261,4 @@ def monitor_mast(): configure_logging(module) # Run the monitors - logging.info('Beginning database monitoring.') monitor_mast() -