sample held out workloads

mlcommons · Apr 8, 2024 · 796f18c · 796f18c
1 parent f463c1d
commit 796f18c
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 2 deletions.
diff --git a/scoring/generate_held_out_workloads.py b/scoring/generate_held_out_workloads.py
@@ -9,7 +9,8 @@
 
 flags.DEFINE_integer('held_out_workloads_seed',
                      None,
-                     'Random seed for scoring.')
+                     'Random seed for scoring.'
+                     'AlgoPerf v0.5 seed: 3438810845')
 flags.DEFINE_string('output_filename',
                     'held_out_workloads.json',
                     'Path to file to record sampled held_out workloads.')
@@ -19,7 +20,10 @@
     'librispeech': [
         'librispeech_conformer_attention_temperature',
         'librispeech_conformer_layernorm',
-        'librispeech_conformer_gelu'
+        # 'librispeech_conformer_gelu', # Removed due to bug in target setting procedure
+        'librispeech_deepspeech_no_resnet',
+        'librispeech_deepspeech_norm_and_spec_aug',
+        'librispeech_deepspeech_tanh'
     ],
     'imagenet': [
         'imagenet_resnet_silu',

diff --git a/scoring/held_out_workloads_algoperf_v05.json b/scoring/held_out_workloads_algoperf_v05.json
@@ -0,0 +1 @@
+["librispeech_conformer_layernorm", "imagenet_resnet_large_bn_init", "ogbg_model_size", "wmt_glu_tanh", "fastmri_tanh", "criteo1tb_embed_init"]
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		["librispeech_conformer_layernorm", "imagenet_resnet_large_bn_init", "ogbg_model_size", "wmt_glu_tanh", "fastmri_tanh", "criteo1tb_embed_init"]