From 796f18c90f64910c3613947a90f5ddfea95de180 Mon Sep 17 00:00:00 2001 From: Priya Kasimbeg Date: Mon, 8 Apr 2024 20:08:19 +0000 Subject: [PATCH] sample held out workloads --- scoring/generate_held_out_workloads.py | 8 ++++++-- scoring/held_out_workloads_algoperf_v05.json | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 scoring/held_out_workloads_algoperf_v05.json diff --git a/scoring/generate_held_out_workloads.py b/scoring/generate_held_out_workloads.py index 14adb1692..d6b403a44 100644 --- a/scoring/generate_held_out_workloads.py +++ b/scoring/generate_held_out_workloads.py @@ -9,7 +9,8 @@ flags.DEFINE_integer('held_out_workloads_seed', None, - 'Random seed for scoring.') + 'Random seed for scoring.' + 'AlgoPerf v0.5 seed: 3438810845') flags.DEFINE_string('output_filename', 'held_out_workloads.json', 'Path to file to record sampled held_out workloads.') @@ -19,7 +20,10 @@ 'librispeech': [ 'librispeech_conformer_attention_temperature', 'librispeech_conformer_layernorm', - 'librispeech_conformer_gelu' + # 'librispeech_conformer_gelu', # Removed due to bug in target setting procedure + 'librispeech_deepspeech_no_resnet', + 'librispeech_deepspeech_norm_and_spec_aug', + 'librispeech_deepspeech_tanh' ], 'imagenet': [ 'imagenet_resnet_silu', diff --git a/scoring/held_out_workloads_algoperf_v05.json b/scoring/held_out_workloads_algoperf_v05.json new file mode 100644 index 000000000..f4d27b3ed --- /dev/null +++ b/scoring/held_out_workloads_algoperf_v05.json @@ -0,0 +1 @@ +["librispeech_conformer_layernorm", "imagenet_resnet_large_bn_init", "ogbg_model_size", "wmt_glu_tanh", "fastmri_tanh", "criteo1tb_embed_init"] \ No newline at end of file