From a91de7d0c6088ceb3c7dcea640c2f65bd3af0bae Mon Sep 17 00:00:00 2001 From: gkiar Date: Mon, 21 Feb 2022 15:44:58 -0500 Subject: [PATCH 1/4] add resource parser --- src/cpac/helpers/cpac_parse_resources.py | 93 ++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100755 src/cpac/helpers/cpac_parse_resources.py diff --git a/src/cpac/helpers/cpac_parse_resources.py b/src/cpac/helpers/cpac_parse_resources.py new file mode 100755 index 00000000..0398f53c --- /dev/null +++ b/src/cpac/helpers/cpac_parse_resources.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python +'''cpac_parse_resources.py + +`cpac_parse resources` is intended to be run outside a C-PAC container +''' + +from rich.console import Console +from rich.table import Table + +from argparse import ArgumentParser +import pandas as pd +import numpy as np +import json + + +runti = 'runtime_memory_gb' +estim = 'estimated_memory_gb' + +field = {'runtime': runti, + 'estimate': estim, + 'efficiency': 'efficiency'} + + +def display(df): + console = Console() + + table = Table(show_header=True, header_style="bold magenta") + table.add_column("Task ID", style="dim", width=40) + table.add_column("Memory Used") + table.add_column("Memory Estimated") + table.add_column("Memory Efficiency") + + for _, d in df.iterrows(): + tmp = list() + tmp += [d['id']] + tmp += [d[runti]] + tmp += [d[estim]] + tmp += ["{0:.2f} %".format(100*d[runti] * 1.0 / d[estim])] + + tmp = ["{0:.4f}".format(t) if isinstance(t, float) else str(t) + for t in tmp] + table.add_row(*tmp) + del tmp + + console.print(table) + + +def load_runtime_stats(callback): + with open(callback) as fhandle: + logs = [json.loads(log) for log in fhandle.readlines()] + + pruned_logs = [] + for log in logs: + if runti not in log.keys(): + continue + + tmp = {} + for k in ['id', runti, estim]: + tmp[k] = log[k] + tmp['efficiency'] = tmp[runti] / tmp[estim] * 100 + + pruned_logs += [tmp] + del tmp + + return pd.DataFrame.from_dict(pruned_logs) + + +def query(usage, f, g, c): + order = True if g == 'lowest' else False + usage.sort_values(by=field[f], ascending=order, inplace=True) + usage.reset_index(inplace=True, drop=True) + return usage[0:c] + + +if __name__ == '__main__': + parser = ArgumentParser(__file__) + parser.add_argument("callback", + help="callback.log file found in the 'log' " + "directory of the specified derivatives path") + parser.add_argument("--filter_field", "-f", action="store", + choices=['runtime', 'estimate', 'efficiency'], + default='efficiency') + parser.add_argument("--filter_group", "-g", action="store", + choices=['lowest', 'highest'], default='lowest') + parser.add_argument("--filter_count", "-c", action="store", type=int, + default=10) + + res = parser.parse_args() + usage = load_runtime_stats(res.callback) + + filtered_usage = query(usage, res.filter_field, res.filter_group, + res.filter_count) + display(filtered_usage) From ff6ebc532a96ec6e58d4dc3b6054fc97cdde10be Mon Sep 17 00:00:00 2001 From: gkiar Date: Mon, 21 Feb 2022 15:45:45 -0500 Subject: [PATCH 2/4] updated requirements.txt --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5b24cf58..3ad813ae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,5 @@ setuptools spython >= 0.0.81 tabulate >= 0.8.6 tornado -websocket-client \ No newline at end of file +websocket-client +rich From 6753c8de0283f99757764f58f31021ae0a8c8fa5 Mon Sep 17 00:00:00 2001 From: gkiar Date: Mon, 21 Feb 2022 15:54:32 -0500 Subject: [PATCH 3/4] updated changelog --- CHANGELOG.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8743bf51..d4c44edd 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,10 @@ ========= Changelog ========= +`Unreleased` +================================================================================================ +* 🧮 Evaluates memory usage for specific nodes from `callback.log` files + `Version 0.4.0: Goodbye Singularity Hub `_ ================================================================================================ * 👽 Drop call to now-deprecated Singularity Hub From db5d1ed9029ee6463f0ed0a76b0591b9b0441995 Mon Sep 17 00:00:00 2001 From: gkiar Date: Mon, 21 Feb 2022 15:55:12 -0500 Subject: [PATCH 4/4] made number flag more intuitively n than c --- src/cpac/helpers/cpac_parse_resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpac/helpers/cpac_parse_resources.py b/src/cpac/helpers/cpac_parse_resources.py index 0398f53c..8c825199 100755 --- a/src/cpac/helpers/cpac_parse_resources.py +++ b/src/cpac/helpers/cpac_parse_resources.py @@ -82,7 +82,7 @@ def query(usage, f, g, c): default='efficiency') parser.add_argument("--filter_group", "-g", action="store", choices=['lowest', 'highest'], default='lowest') - parser.add_argument("--filter_count", "-c", action="store", type=int, + parser.add_argument("--filter_count", "-n", action="store", type=int, default=10) res = parser.parse_args()