[benchmark] optimize benchmark: counting tokenlizer tokens and error requests

benchmark/profile_restful_api.py

@@ -5,6 +5,7 @@
 from queue import Queue
 from threading import Thread
 from typing import List, Optional, Tuple
+from urllib.parse import urlparse
 
 import fire
 import numpy as np
@@ -93,6 +94,7 @@ def _inference(self, req_queue: Queue, res_queue: Queue, session_id: int,
                 req_queue.get, [None, None, None]):
             timestamps = []
             timestamps.append(time.perf_counter())
+            full_output = ''
             for output in client.chat_completions_v1(
                     model=self.model_name,
                     messages=prompt,
@@ -103,17 +105,28 @@ def _inference(self, req_queue: Queue, res_queue: Queue, session_id: int,
                     stream=stream_output,
                     session_id=session_id,
                     ignore_eos=True):
+                # Here we ignore the index of the multiple outputs and
+                # just put all of them together to compute tokens.
+                for choice in output.get('choices', []):
+                    if stream_output:
+                        full_output += choice['delta']['content']
+                    else:
+                        full_output += choice['message']['content']
                 timestamps.append(time.perf_counter())
 
             first_token_latency = np.round(timestamps[1] - timestamps[0], 3)
             token_latency = np.round(timestamps[-1] - timestamps[0], 3)
             # assert output.pop('finish_reason') == 'length', \
             #     f'Error. session_id({session_id}) request {output_seqlen} ' \
             #     f'tokens, but `finish_reason` is not `length`'
-            total_tokens = input_seqlen + output_seqlen
+            tokenlizer_start = time.perf_counter()
+            real_output_seqlen = len(self.tokenizer(full_output).input_ids)
+            tokenlizer_finish = time.perf_counter()
+            tokenlizer_time = tokenlizer_finish - tokenlizer_start
+            total_tokens = input_seqlen + real_output_seqlen
             stats.append([
-                first_token_latency, output_seqlen, output_seqlen,
-                total_tokens, token_latency
+                first_token_latency, real_output_seqlen, output_seqlen,
+                total_tokens, token_latency, tokenlizer_time
             ])
             self.pbar.update(1)
 
@@ -157,7 +170,10 @@ def process_request(self,
             #       f'session {session_id} stats: \n{_stats}\n{"-" * 50}\n')
             stats.append(np.array(_stats))
 
-        stats = np.concatenate(stats).reshape(-1, 5)
+        stats = np.concatenate(stats).reshape(-1, 6)
+
+        tokenlizer_time = np.sum(stats[:, 5], axis=0) / concurrency
+        elapsed_time -= tokenlizer_time
 
         first_token_latency_min = np.min(stats[:, 0], axis=0)
         first_token_latency_max = np.max(stats[:, 0], axis=0)
@@ -166,6 +182,7 @@ def process_request(self,
         request_output_tokens = np.sum(stats[:, 2], axis=0)
         total_tokens = np.sum(stats[:, 3], axis=0)
         prompt_tokens = total_tokens - completion_tokens
+        local_tokenlizer_throughput = completion_tokens / tokenlizer_time
         completion_token_throughput = completion_tokens / elapsed_time
         total_token_throughput = total_tokens / elapsed_time
         rps = len(requests) / elapsed_time
@@ -186,6 +203,7 @@ def process_request(self,
         print(
             f'number of prompt tokens: {prompt_tokens:.0f}\n'
             f'number of completion tokens: {completion_tokens:.0f}\n'
+            f'local tokenlizer throughput (completion token): {local_tokenlizer_throughput:.3f} token/s\n'  # noqa
             f'token throughput (completion token): {completion_token_throughput:.3f} token/s\n'  # noqa
             f'token throughput (prompt + completion token): {total_token_throughput:.3f} token/s\n'  # noqa
             f'RPS (request per second): {rps:.3f} req/s\n'
@@ -241,10 +259,12 @@ def main(server_addr: str,
         csv (str, optional): The path to save the result.
         seed (int, optional): Seed used in sampling prompts from dataset. Defaults to 0.
     """    # noqa
-    if not server_addr.startswith('http://'):
+    addr_schem = urlparse(server_addr).scheme
+    if addr_schem not in ['http', 'https']:
         print(f'[WARNING] server_addr of the api_server should '
-              f'start with "http://", but got "{server_addr}"')
+              f'start with "http://" or "https://", but got "{server_addr}"')
         server_addr = 'http://' + server_addr.strip()
+    print(f'[INFO] using server_addr: {server_addr}')
 
     random.seed(seed)