From 9c9919ff8c797b1a69dcadbe454bc1883085821d Mon Sep 17 00:00:00 2001 From: Malcolm Ross Date: Tue, 12 Mar 2024 16:30:21 -0500 Subject: [PATCH 1/5] added rich HTML output for MC_GEOPHIRES. Also fixed bug for when run from command line --- src/geophires_monte_carlo/MC_GeoPHIRES3.py | 199 ++++++++++++++++++++- 1 file changed, 197 insertions(+), 2 deletions(-) diff --git a/src/geophires_monte_carlo/MC_GeoPHIRES3.py b/src/geophires_monte_carlo/MC_GeoPHIRES3.py index acd13500..adbddb46 100755 --- a/src/geophires_monte_carlo/MC_GeoPHIRES3.py +++ b/src/geophires_monte_carlo/MC_GeoPHIRES3.py @@ -22,8 +22,12 @@ import matplotlib.pyplot as plt import numpy as np import pandas as pd +from rich.console import Console +from rich.table import Table from geophires_monte_carlo.common import _get_logger +from geophires_x.Parameter import OutputParameter +from geophires_x.Parameter import floatParameter from geophires_x_client import GeophiresInputParameters from geophires_x_client import GeophiresXClient from geophires_x_client import GeophiresXResult @@ -33,6 +37,184 @@ from hip_ra_x import HipRaXClient +def Write_HTML_Output( + html_path: str, + df: pd.DataFrame, + outputs: list, + mins: list, + maxs: list, + medians: list, + averages: list, + means: list, + std: list, + full_names: set, + short_names: set, +) -> None: + """ + Write_HTML_Output - write the results of the Monte Carlo simulation to an HTML file + :param html_path: the path to the HTML file to write + :type html_path: str + :param df: the DataFrame with the results + :type df: pd.DataFrame + :param outputs: the list of output variable names + :type outputs: list + :param mins: the list of minimum values for each output variable + :type mins: list + :param maxs: the list of maximum values for each output variable + :type maxs: list + :param medians: the list of median values for each output variable + :type medians: list + :param averages: the list of average values for each output variable + :type averages: list + :param means: the list of mean values for each output variable + :type means: list + :param std: the list of standard deviation values for each output variable + :type std: list + :param full_names: the list of full names for each output variable + :type full_names: set + :param short_names: the list of short names for each output variable + :type short_names: set + """ + + # Build the tables that will hold those results, along with the columns for the input variables + results_table = Table(title='GEOPHIRES/HIR-RA Monte Carlo Results') + results_table.add_column('Iteration #', no_wrap=True, justify='center') + for output in df.axes[1]: + results_table.add_column(output.replace(',', ''), no_wrap=True, justify='center') + + statistics_table = Table(title='GEOPHIRES/HIR-RA Monte Carlo Statistics') + statistics_table.add_column('Output Parameter Name', no_wrap=True, justify='center') + statistics_table.add_column('minimum', no_wrap=True, justify='center') + statistics_table.add_column('maximum', no_wrap=True, justify='center') + statistics_table.add_column('median', no_wrap=True, justify='center') + statistics_table.add_column('average', no_wrap=True, justify='center') + statistics_table.add_column('mean', no_wrap=True, justify='center') + statistics_table.add_column('standard deviation', no_wrap=True, justify='center') + + # Iterate over the rows of the DataFrame and add them to the results table + for index, row in df.iterrows(): + data = row.values[0 : len(outputs)] + + # have to deal with the special case where thr last column is actually + # a compound string with multiple columns in it that looks like this: + # ' (Gradient 1:47.219846973456924;Reservoir Temperature:264.7789623351493;...)' + str_to_parse = str(row.values[len(outputs)]).strip().replace('(', '').replace(')', '') + fields = str_to_parse.split(';') + for field in fields: + if len(field) > 0: + key, value = field.split(':') + data = np.append(data, float(value)) + + results_table.add_row(str(int(index)), *[render_default(d) for d in data]) + + for i in range(len(outputs)): + statistics_table.add_row( + outputs[i], + render_default(mins[i]), + render_default(maxs[i]), + render_default(medians[i]), + render_default(averages[i]), + render_default(means[i]), + render_default(std[i]), + ) + + console = Console(style='bold white on black', force_terminal=True, record=True, width=500) + console.print(results_table) + console.print(' ') + console.print(statistics_table) + console.save_html(html_path) + + # Write a reference to the image(s) into the HTML file by inserting before the "" tag + # build the string to be inserted first + insert_string = '' + for _ in range(len(full_names)): + name_to_use = short_names.pop() + insert_string = insert_string + f'{name_to_use}\n' + + match_string = '' + with open(html_path, 'r+', encoding='UTF-8') as html_file: + contents = html_file.readlines() + if match_string in contents[-1]: # Handle last line to prevent IndexError + pass + else: + for index, line in enumerate(contents): + if match_string in line and insert_string not in contents[index + 1]: + contents.insert(index, insert_string) + break + html_file.seek(0) + html_file.writelines(contents) + + +def UpgradeSymbologyOfUnits(unit: str) -> str: + """ + UpgradeSymbologyOfUnits is a function that takes a string that represents a unit and replaces the **2 and **3 + with the appropriate unicode characters for superscript 2 and 3, and replaces "deg" with the unicode character + for degrees. + :param unit: a string that represents a unit + :return: a string that represents a unit with the appropriate unicode characters for superscript 2 and 3, and + replaces "deg" with the unicode character for degrees. + """ + return unit.replace('**2', '\u00b2').replace('**3', '\u00b3').replace('deg', '\u00b0') + + +def render_default(p: float, unit: str = '') -> str: + """ + RenderDefault - render a float as a string with 2 decimal places, or in scientific notation if it is greater than + 10,000 with the unit appended to it if it is not an empty string (the default) + :param p: the float to render + :type p: float + :param unit: the unit to append to the string + :type unit: str + :return: the string representation of the float + :rtype: str + """ + unit = UpgradeSymbologyOfUnits(unit) + # if the number is greater than 10,000, render it in scientific notation + if p > 10_000: + return f'{p:10.2e} {unit}'.strip() + # otherwise, render it with 2 decimal places + else: + return f'{p:10.2f} {unit}'.strip() + + +def render_scientific(p: float, unit: str = '') -> str: + """ + RenderScientific - render a float as a string in scientific notation with 2 decimal places + and the unit appended to it if it is not an empty string (the default) + :param p: the float to render + :type p: float + :param unit: the unit to append to the string + :type unit: str + :return: the string representation of the float + :rtype: str + """ + unit = UpgradeSymbologyOfUnits(unit) + return f'{p:10.2e} {unit}'.strip() + + +def render_Parameter_default(p: floatParameter | OutputParameter) -> str: + """ + RenderDefault - render a float as a string with 2 decimal places, or in scientific notation if it is greater than + 10,000 with the unit appended to it if it is not an empty string (the default) by calling the render_default base + function + :param p: the parameter to render + :type p: float + :return: the string representation of the float + """ + return render_default(p.value, p.CurrentUnits.value) + + +def render_parameter_scientific(p: floatParameter | OutputParameter) -> str: + """ + RenderScientific - render a float as a string in scientific notation with 2 decimal places + and the unit appended to it if it is not an empty string (the default) by calling the render_scientific base function + :param p: the parameter to render + :type p: float + :return: the string representation of the float + """ + return render_scientific(p.value, p.CurrentUnits.value) + + def check_and_replace_mean(input_value, args) -> list: """ CheckAndReplaceMean - check to see if the user has requested that a value be replaced by a mean value by specifying @@ -268,7 +450,9 @@ def main(command_line_args=None): if 'MC_OUTPUT_FILE' in args and args.MC_OUTPUT_FILE is not None else str(Path(Path(args.Input_file).parent, 'MC_Result.txt').absolute()) ) + args.MC_OUTPUT_FILE = output_file python_path = 'python' + html_path = '' for line in flist: clean = line.strip() @@ -284,6 +468,8 @@ def main(command_line_args=None): output_file = pair[1] elif pair[0].startswith('PYTHON_PATH'): python_path = pair[1] + elif pair[0].startswith('HTML_PATH'): + html_path = pair[1] # check to see if there is a "#" in an input, if so, use the results file to replace it with the value for input_value in inputs: @@ -375,6 +561,8 @@ def main(command_line_args=None): # write them out annotations = '' outputs_result: dict[str, dict] = {} + full_names: set = set() + short_names: set = set() with open(output_file, 'a') as f: if iterations != actual_records_count: f.write( @@ -408,10 +596,17 @@ def main(command_line_args=None): f.write(f'bin values (as percentage): {ret[0]!s}\n') f.write(f'bin edges: {ret[1]!s}\n') fname = df.columns[i].strip().replace('/', '-') - plt.savefig(Path(Path(output_file).parent, f'{fname}.png')) - + save_path = Path(Path(output_file).parent, f'{fname}.png') + if html_path: + save_path = Path(Path(html_path).parent, f'{fname}.png') + plt.savefig(save_path) + full_names.add(save_path) + short_names.add(fname) annotations = '' + if html_path: + Write_HTML_Output(html_path, df, outputs, mins, maxs, medians, averages, means, std, full_names, short_names) + with open(Path(output_file).with_suffix('.json'), 'w') as json_output_file: json_output_file.write(json.dumps(outputs_result)) logger.info(f'Wrote JSON results to {json_output_file.name}') From 060bd190cc3722f9a09229756612cc9387d72cc2 Mon Sep 17 00:00:00 2001 From: Malcolm Ross Date: Wed, 13 Mar 2024 09:44:04 -0500 Subject: [PATCH 2/5] added rich HTML output for MC_GEOPHIRES. Also fixed bug for when run from command line --- src/hip_ra/HIP_RA.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/hip_ra/HIP_RA.py b/src/hip_ra/HIP_RA.py index abead239..ad0fc2eb 100755 --- a/src/hip_ra/HIP_RA.py +++ b/src/hip_ra/HIP_RA.py @@ -26,6 +26,7 @@ from geophires_x.Parameter import ConvertUnitsBack from geophires_x.Parameter import LookupUnits from geophires_x.Parameter import OutputParameter +from geophires_x.Parameter import Parameter from geophires_x.Parameter import ReadParameter from geophires_x.Parameter import floatParameter from geophires_x.Parameter import intParameter @@ -742,10 +743,10 @@ def PrintOutputs(self): nl = '\n' outputfile = 'HIP.out' if len(sys.argv) <= 2 else sys.argv[2] - def render_default(p: floatParameter | OutputParameter) -> str: + def render_default(p: Parameter) -> str: return f'{p.value:10.2f} {p.CurrentUnits.value}' - def render_scientific(p: floatParameter | OutputParameter) -> str: + def render_scientific(p: Parameter) -> str: return f'{p.value:10.2e} {p.CurrentUnits.value}' summary_of_results = {} From b6af2ac25aeb50c0f80c3665e449855001f4db49 Mon Sep 17 00:00:00 2001 From: Malcolm Ross Date: Wed, 13 Mar 2024 10:13:52 -0500 Subject: [PATCH 3/5] added rich HTML output for MC_GEOPHIRES. Also fixed bug for when run from command line --- src/hip_ra_x/hip_ra_x.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hip_ra_x/hip_ra_x.py b/src/hip_ra_x/hip_ra_x.py index 5b35dbd1..9b841582 100644 --- a/src/hip_ra_x/hip_ra_x.py +++ b/src/hip_ra_x/hip_ra_x.py @@ -811,10 +811,10 @@ def PrintOutputs(self): try: outputfile = 'HIP.out' if len(sys.argv) <= 2 else sys.argv[2] - def render_default(p: floatParameter | OutputParameter) -> str: + def render_default(p: Parameter) -> str: return f'{p.value:10.2f} {p.CurrentUnits.value}' - def render_scientific(p: floatParameter | OutputParameter) -> str: + def render_scientific(p: Parameter) -> str: return f'{p.value:10.2e} {p.CurrentUnits.value}' summary_of_inputs = {} From 438880e47e94a2a64838e4d3c65c99b817abca9b Mon Sep 17 00:00:00 2001 From: Malcolm Ross Date: Wed, 13 Mar 2024 10:19:46 -0500 Subject: [PATCH 4/5] added rich HTML output for MC_GEOPHIRES. Also fixed bug for when run from command line --- src/geophires_monte_carlo/MC_GeoPHIRES3.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/geophires_monte_carlo/MC_GeoPHIRES3.py b/src/geophires_monte_carlo/MC_GeoPHIRES3.py index adbddb46..b571c783 100755 --- a/src/geophires_monte_carlo/MC_GeoPHIRES3.py +++ b/src/geophires_monte_carlo/MC_GeoPHIRES3.py @@ -27,6 +27,7 @@ from geophires_monte_carlo.common import _get_logger from geophires_x.Parameter import OutputParameter +from geophires_x.Parameter import Parameter from geophires_x.Parameter import floatParameter from geophires_x_client import GeophiresInputParameters from geophires_x_client import GeophiresXClient @@ -192,7 +193,7 @@ def render_scientific(p: float, unit: str = '') -> str: return f'{p:10.2e} {unit}'.strip() -def render_Parameter_default(p: floatParameter | OutputParameter) -> str: +def render_Parameter_default(p: Parameter) -> str: """ RenderDefault - render a float as a string with 2 decimal places, or in scientific notation if it is greater than 10,000 with the unit appended to it if it is not an empty string (the default) by calling the render_default base From 8238309fc2ec7398c9f8f5578c3f8edcf2bb696b Mon Sep 17 00:00:00 2001 From: Malcolm Ross Date: Wed, 13 Mar 2024 10:23:08 -0500 Subject: [PATCH 5/5] added rich HTML output for MC_GEOPHIRES. Also fixed bug for when run from command line --- src/geophires_monte_carlo/MC_GeoPHIRES3.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/geophires_monte_carlo/MC_GeoPHIRES3.py b/src/geophires_monte_carlo/MC_GeoPHIRES3.py index b571c783..1220cd17 100755 --- a/src/geophires_monte_carlo/MC_GeoPHIRES3.py +++ b/src/geophires_monte_carlo/MC_GeoPHIRES3.py @@ -26,9 +26,7 @@ from rich.table import Table from geophires_monte_carlo.common import _get_logger -from geophires_x.Parameter import OutputParameter from geophires_x.Parameter import Parameter -from geophires_x.Parameter import floatParameter from geophires_x_client import GeophiresInputParameters from geophires_x_client import GeophiresXClient from geophires_x_client import GeophiresXResult @@ -205,7 +203,7 @@ def render_Parameter_default(p: Parameter) -> str: return render_default(p.value, p.CurrentUnits.value) -def render_parameter_scientific(p: floatParameter | OutputParameter) -> str: +def render_parameter_scientific(p: Parameter) -> str: """ RenderScientific - render a float as a string in scientific notation with 2 decimal places and the unit appended to it if it is not an empty string (the default) by calling the render_scientific base function