Fix streaming case for openai (#499)

triton-inference-server · Mar 7, 2024 · 886a177 · 886a177
1 parent d1bb865
commit 886a177
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 1 deletion.
diff --git a/src/c++/perf_analyzer/genai-pa/genai_pa/llm_inputs/llm_inputs.py b/src/c++/perf_analyzer/genai-pa/genai_pa/llm_inputs/llm_inputs.py
@@ -653,7 +653,7 @@ def _add_optional_tags_to_openai_json(
         if add_model_name:
             pa_json["data"][index]["payload"][0]["model"] = model_name
         if add_stream:
-            pa_json["data"][index]["payload"][0]["stream"] = [True]
+            pa_json["data"][index]["payload"][0]["stream"] = True
 
         return pa_json
 

diff --git a/src/c++/perf_analyzer/genai-pa/genai_pa/wrapper.py b/src/c++/perf_analyzer/genai-pa/genai_pa/wrapper.py
@@ -44,6 +44,10 @@ def build_cmd(model, args, extra_args):
             "input_format",
             "output_format",
         ]
+        # OpenAI service-kind sets streaming via the input file, not via the CLI
+        if args.service_kind == "openai":
+            skip_args.append("streaming")
+
         if hasattr(args, "version") and args.version:
             cmd = f"perf_analyzer --version"
         else: