From 2dc2111da6b9a86d481d01239868e23a39cba3c1 Mon Sep 17 00:00:00 2001 From: Izzy Putterman Date: Wed, 24 Jul 2024 16:12:03 -0700 Subject: [PATCH] Initial attempt to fix --- .../llm_profile_data_parser.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/c++/perf_analyzer/genai-perf/genai_perf/profile_data_parser/llm_profile_data_parser.py b/src/c++/perf_analyzer/genai-perf/genai_perf/profile_data_parser/llm_profile_data_parser.py index 183f21fd2..defa6307d 100755 --- a/src/c++/perf_analyzer/genai-perf/genai_perf/profile_data_parser/llm_profile_data_parser.py +++ b/src/c++/perf_analyzer/genai-perf/genai_perf/profile_data_parser/llm_profile_data_parser.py @@ -174,6 +174,14 @@ def _preprocess_response( if self._service_kind == "openai": # PA sometimes receives multiple SSE responses at once (as a single # response). Handle these responses by merging into a single response. + for i in range(len(res_outputs) - 1, -1, -1): + response = res_outputs[i]["response"] + if not response.startswith("data: "): + first_data = response.find("data: ") + res_outputs[i - 1]["response"] = ( + res_outputs[i - 1]["response"] + response[0:first_data].strip() + ) + res_outputs[i]["response"] = response[first_data:].strip() for i in range(len(res_outputs)): response = res_outputs[i]["response"] responses = response.strip().split("\n\n") @@ -193,7 +201,9 @@ def _preprocess_response( # Remove responses without any content indices_to_remove = [] for idx, out in enumerate(res_outputs): - if self._is_openai_empty_response(out["response"]): + if not out["response"] or self._is_openai_empty_response( + out["response"] + ): indices_to_remove.append(idx) indices_to_remove.sort(reverse=True) for index in indices_to_remove: @@ -268,7 +278,9 @@ def _extract_openai_text_output(self, response: str) -> str: """Extracts text/content of the OpenAI response object.""" response = remove_sse_prefix(response) - if response == "[DONE]": + if response == "[DONE]" or not ( + response.strip().startswith("{") and response.strip().endswith("}") + ): return "" data = load_json_str(response)