From 41821e5b78b0b4794ebde7de0a06b9c330551c72 Mon Sep 17 00:00:00 2001 From: Timothy Gerdes <50968584+tgerdesnv@users.noreply.github.com> Date: Thu, 7 Mar 2024 17:26:40 -0600 Subject: [PATCH] Fix streaming case for openai (#499) --- .../perf_analyzer/genai-pa/genai_pa/llm_inputs/llm_inputs.py | 2 +- src/c++/perf_analyzer/genai-pa/genai_pa/wrapper.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/c++/perf_analyzer/genai-pa/genai_pa/llm_inputs/llm_inputs.py b/src/c++/perf_analyzer/genai-pa/genai_pa/llm_inputs/llm_inputs.py index 743f87752..dd6a7b8f6 100644 --- a/src/c++/perf_analyzer/genai-pa/genai_pa/llm_inputs/llm_inputs.py +++ b/src/c++/perf_analyzer/genai-pa/genai_pa/llm_inputs/llm_inputs.py @@ -653,7 +653,7 @@ def _add_optional_tags_to_openai_json( if add_model_name: pa_json["data"][index]["payload"][0]["model"] = model_name if add_stream: - pa_json["data"][index]["payload"][0]["stream"] = [True] + pa_json["data"][index]["payload"][0]["stream"] = True return pa_json diff --git a/src/c++/perf_analyzer/genai-pa/genai_pa/wrapper.py b/src/c++/perf_analyzer/genai-pa/genai_pa/wrapper.py index 3caafebf3..3ce3a5744 100644 --- a/src/c++/perf_analyzer/genai-pa/genai_pa/wrapper.py +++ b/src/c++/perf_analyzer/genai-pa/genai_pa/wrapper.py @@ -44,6 +44,10 @@ def build_cmd(model, args, extra_args): "input_format", "output_format", ] + # OpenAI service-kind sets streaming via the input file, not via the CLI + if args.service_kind == "openai": + skip_args.append("streaming") + if hasattr(args, "version") and args.version: cmd = f"perf_analyzer --version" else: