Skip to content

Commit

Permalink
Fix setting of streaming option
Browse files Browse the repository at this point in the history
  • Loading branch information
tgerdesnv committed Mar 11, 2024
1 parent b49f48c commit 99e4dd8
Showing 1 changed file with 4 additions and 5 deletions.
9 changes: 4 additions & 5 deletions src/c++/perf_analyzer/genai-pa/genai_pa/wrapper.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ class Profiler:
def add_protocol_args(args):
cmd = ""
if args.service_kind == "triton":
cmd += f"-i grpc "
cmd += f"-i grpc --streaming "
if args.output_format == OutputFormat.TRTLLM:
cmd += f"--shape max_tokens:1 --shape text_input:1 "
elif args.service_kind == "openai":
Expand All @@ -55,17 +55,16 @@ def build_cmd(model, args, extra_args):
"input_type",
"input_format",
"output_format",
# The 'streaming' passed in to this script is to determine if the LLM response should be streaming. That is different than the 'streaming' that PA takes, which means something else (and is required for decoupled models)
"streaming",
]
# OpenAI service-kind sets streaming via the input file, not via the CLI
if args.service_kind == "openai":
skip_args.append("streaming")

if hasattr(args, "version") and args.version:
cmd = f"perf_analyzer --version"
else:
utils.remove_file(args.profile_export_file)

cmd = f"perf_analyzer -m {model} --async "
cmd = f"/home/tgerdes/Code/client/build/install/bin/perf_analyzer -m {model} --async "
for arg, value in vars(args).items():
if arg in skip_args:
pass
Expand Down

0 comments on commit 99e4dd8

Please sign in to comment.