From 7d01efe6e4bb973c111f7e2a9a696a00ed0c14b0 Mon Sep 17 00:00:00 2001 From: Priya Kasimbeg Date: Mon, 29 Jul 2024 21:07:52 +0000 Subject: [PATCH 1/3] fixes --- scoring/performance_profile.py | 3 +++ scoring/score_submissions.py | 35 +++++++++++++++++++++++++++++----- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/scoring/performance_profile.py b/scoring/performance_profile.py index 31106f057..19157c2a0 100644 --- a/scoring/performance_profile.py +++ b/scoring/performance_profile.py @@ -227,6 +227,9 @@ def get_workloads_time_to_target(submission, else: time_val = float('inf') time_vals_per_study.append(time_val) + num_s = len(time_vals_per_study) + print(f'TIME VALS PER STUDY: {num_s}') + print(time_vals_per_study) workloads.append({ 'submission': submission_name, diff --git a/scoring/score_submissions.py b/scoring/score_submissions.py index 97264748f..5f711d6e4 100644 --- a/scoring/score_submissions.py +++ b/scoring/score_submissions.py @@ -32,7 +32,7 @@ 'Path to submission directory containing experiment directories.') flags.DEFINE_string('output_dir', 'scoring_results', - 'Path to save performance profile table and plot.') + 'Path to save performance profile artifacts, submission_summaries and results files.') flags.DEFINE_boolean('compute_performance_profiles', False, 'Whether or not to compute the performance profiles.') @@ -51,11 +51,16 @@ None, 'Filename to save the processed results that are fed into the performance profile functions.' ) -flags.DEFINE_boolean( +flags.DEFINE_string( 'load_results_from_filename', None, 'Filename to load processed results from that are fed into performance profile functions' ) +flags.DEFINE_string( + 'exclude_submissions', + '', + 'Optional comma seperated list of names of submissions to exclude from scoring.' +) FLAGS = flags.FLAGS @@ -128,6 +133,21 @@ def get_submission_summary(df, include_test_split=True): logging.info('\n' + tabulate(df, headers='keys', tablefmt='psql')) return df +def compute_leaderboard_score(df, normalize=False): + """Compute leaderboard score by taking integral of performance profile. + + Args: + df: pd.DataFrame returned from `compute_performance_profiles`. + normalize: divide by the range of the performance profile's tau. + + Returns: + pd.DataFrame with one column of scores indexed by submission. + """ + scores = np.trapz(df, x=df.columns) + if normalize: + scores /= df.columns.max() - df.columns.min() + return pd.DataFrame(scores, columns=['score'], index=df.index) + def main(_): results = {} @@ -144,6 +164,8 @@ def main(_): for submission in os.listdir( os.path.join(FLAGS.submission_directory, team)): print(submission) + if submission in FLAGS.exclude_submissions.split(','): + continue experiment_path = os.path.join(FLAGS.submission_directory, team, submission) @@ -185,10 +207,13 @@ def main(_): os.mkdir(FLAGS.output_dir) performance_profile.plot_performance_profiles( performance_profile_df, 'score', save_dir=FLAGS.output_dir) - perf_df = tabulate( + performance_profile_str = tabulate( performance_profile_df.T, headers='keys', tablefmt='psql') - logging.info(f'Performance profile:\n {perf_df}') - + logging.info(f'Performance profile:\n {performance_profile_str}') + scores = compute_leaderboard_score(performance_profile_df) + scores.to_csv(os.path.join(FLAGS.output_dir, 'scores.csv')) + scores_str = tabulate(scores, headers='keys', tablefmt='psql') + logging.info(f'Scores: \n {scores_str}') if __name__ == '__main__': # flags.mark_flag_as_required('submission_directory') From 006b07e4b44390ce1f69fb3c126250ce103d484c Mon Sep 17 00:00:00 2001 From: Priya Kasimbeg Date: Mon, 29 Jul 2024 21:10:26 +0000 Subject: [PATCH 2/3] remove logging --- scoring/performance_profile.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/scoring/performance_profile.py b/scoring/performance_profile.py index 19157c2a0..31106f057 100644 --- a/scoring/performance_profile.py +++ b/scoring/performance_profile.py @@ -227,9 +227,6 @@ def get_workloads_time_to_target(submission, else: time_val = float('inf') time_vals_per_study.append(time_val) - num_s = len(time_vals_per_study) - print(f'TIME VALS PER STUDY: {num_s}') - print(time_vals_per_study) workloads.append({ 'submission': submission_name, From 4238cb27b469c24d2c4c708ca6239de805b40900 Mon Sep 17 00:00:00 2001 From: Priya Kasimbeg Date: Mon, 29 Jul 2024 21:12:51 +0000 Subject: [PATCH 3/3] formatting --- scoring/score_submissions.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scoring/score_submissions.py b/scoring/score_submissions.py index 5f711d6e4..bbc23a1fc 100644 --- a/scoring/score_submissions.py +++ b/scoring/score_submissions.py @@ -30,9 +30,11 @@ 'submission_directory', None, 'Path to submission directory containing experiment directories.') -flags.DEFINE_string('output_dir', - 'scoring_results', - 'Path to save performance profile artifacts, submission_summaries and results files.') +flags.DEFINE_string( + 'output_dir', + 'scoring_results', + 'Path to save performance profile artifacts, submission_summaries and results files.' +) flags.DEFINE_boolean('compute_performance_profiles', False, 'Whether or not to compute the performance profiles.') @@ -133,6 +135,7 @@ def get_submission_summary(df, include_test_split=True): logging.info('\n' + tabulate(df, headers='keys', tablefmt='psql')) return df + def compute_leaderboard_score(df, normalize=False): """Compute leaderboard score by taking integral of performance profile. @@ -215,6 +218,7 @@ def main(_): scores_str = tabulate(scores, headers='keys', tablefmt='psql') logging.info(f'Scores: \n {scores_str}') + if __name__ == '__main__': # flags.mark_flag_as_required('submission_directory') app.run(main)