diff --git a/scoring/generate_held_out_workloads.py b/scoring/generate_held_out_workloads.py index 14adb1692..d6b403a44 100644 --- a/scoring/generate_held_out_workloads.py +++ b/scoring/generate_held_out_workloads.py @@ -9,7 +9,8 @@ flags.DEFINE_integer('held_out_workloads_seed', None, - 'Random seed for scoring.') + 'Random seed for scoring.' + 'AlgoPerf v0.5 seed: 3438810845') flags.DEFINE_string('output_filename', 'held_out_workloads.json', 'Path to file to record sampled held_out workloads.') @@ -19,7 +20,10 @@ 'librispeech': [ 'librispeech_conformer_attention_temperature', 'librispeech_conformer_layernorm', - 'librispeech_conformer_gelu' + # 'librispeech_conformer_gelu', # Removed due to bug in target setting procedure + 'librispeech_deepspeech_no_resnet', + 'librispeech_deepspeech_norm_and_spec_aug', + 'librispeech_deepspeech_tanh' ], 'imagenet': [ 'imagenet_resnet_silu', diff --git a/scoring/held_out_workloads_algoperf_v05.json b/scoring/held_out_workloads_algoperf_v05.json new file mode 100644 index 000000000..f4d27b3ed --- /dev/null +++ b/scoring/held_out_workloads_algoperf_v05.json @@ -0,0 +1 @@ +["librispeech_conformer_layernorm", "imagenet_resnet_large_bn_init", "ogbg_model_size", "wmt_glu_tanh", "fastmri_tanh", "criteo1tb_embed_init"] \ No newline at end of file