only add --tokenizer_path to the command if this variable is set and …

…non-trivial (assuming the code can deal with it)
krai · Feb 14, 2024 · b2c30d6 · b2c30d6
1 parent ce895bb
commit b2c30d6
Showing 1 changed file with 6 additions and 1 deletion.
diff --git a/base_gptj_loadgen_experiment/data_axs.json b/base_gptj_loadgen_experiment/data_axs.json
@@ -17,6 +17,11 @@
         [ "get_path" ]
     ]] ],
     "accuracy_log_dtype": "int32",
+    "tokenizer_path": "",
+    "tokenizer_part": [ "^^", "case", [ [ "^^", "get", [ "tokenizer_path", "" ] ],
+        "EleutherAI/gpt-j-6B", "", "", "", null, "" ],
+        { "default_value": [ "^^", "substitute", " --tokenizer_path \"#{tokenizer_path}#\"" ] }
+    ],
 
     "extract_accuracy_report": [ "^^", "execute", [[
         [ "get_kernel" ],
@@ -30,7 +35,7 @@
                     [ "AS^IS", "^^", "python_sync_pip_package", "python_package,package_name=evaluate" ]
                 ],
                 "abs_script_path": ["^^", "get", "abs_script_path"],
-                "script_extra_params": [ "^^", "substitute", "--mlperf-accuracy-file #{abs_accuracy_log_path}# --dataset-file #{dataset_path}# --dtype #{accuracy_log_dtype}#" ],
+                "script_extra_params": [ "^^", "substitute", "--mlperf-accuracy-file #{abs_accuracy_log_path}# --dataset-file #{dataset_path}# --dtype #{accuracy_log_dtype}# #{tokenizer_part}#" ],
                 "capture_output": true
             } ],
         0,