Merge branch 'main' into dyas-generate-tests

genai-perf/README.md

@@ -293,6 +293,9 @@ options:
   filepaths to images to use for benchmarking as JSON objects.
 
 For any dataset, you can specify the following options:
+* `--num-prefix-prompts <int>`: The number of synthetic prefix prompts to
+  sample from. If this value is >0, synthetic prefix prompts will be prepended
+  to user prompts.
 * `--output-tokens-mean <int>`: The mean number of tokens in each output. Ensure
   the `--tokenizer` value is set correctly, >= 1.
 * `--output-tokens-stddev <int>`: The standard deviation of the number of tokens
@@ -303,6 +306,8 @@ For any dataset, you can specify the following options:
   Triton service-kind. Note that there is still some variability in the
   requested number of output tokens, but GenAi-Perf attempts its best effort
   with your model to get the right number of output tokens.
+* `--prefix-prompt-length <int>`: The number of tokens to include in each
+  prefix prompt. This value is only used if --num-prefix-prompts is positive.
 
 You can optionally set additional model inputs with the following option:
 * `--extra-inputs <input_name>:<value>`: An additional input for use with the
@@ -461,6 +466,12 @@ extensions. For example, 'synthetic:queries,passages'.
 The number of unique payloads to sample from. These will be reused until
 benchmarking is complete. (default: `100`)
 
+##### `--num-prefix-prompts <int>`
+
+The number of prefix prompts to select from. If this value is not zero, these
+are prompts that are prepended to input prompts. This is useful for
+benchmarking models that use a K-V cache. (default: `0`)
+
 ##### `--output-tokens-mean <int>`
 ##### `--osl`
 
@@ -502,6 +513,13 @@ data. (default: `550`)
 The standard deviation of number of tokens in the generated prompts when
 using synthetic data. (default: `0`)
 
+##### `--prefix-prompt-length <int>`
+
+The number of tokens in each prefix prompt. This value is only used if
+--num-prefix-prompts is positive. Note that due to the prefix and user prompts
+being concatenated, the number of tokens in the final prompt may be off by one.
+(default: `100`)
+
 ##### `--warmup-request-count <int>`
 
 The number of warmup requests to send before benchmarking. (default: `0`)

genai-perf/docs/customizable_frontends.md

@@ -123,6 +123,20 @@ converters = {
 }
 ```
 
+## Map an Endpoint Type to the Output Format
+
+Open `genai_perf/parser.py`.
+Add an mapping for your new endpoint type in the `_endpoint_type_map`.
+
+For example, see the code below.
+
+```python
+_endpoint_type_map = {
+    # Existing mappings
+    "NEW-ENDPOINT": EndpointConfig("v1/endpoint", "openai", ic.OutputFormat.NEW_ENDPOINT),
+}
+```
+
 ## Update the Metrics Parser
 
 GenAI-Perf needs to know which metrics format your API uses. Go to
@@ -142,7 +156,7 @@ After implementing your converter, you can run it against your server to
 ensure it works:
 
 ```bash
-genai-perf profile -m TEST_MODEL --endpoint NEW-ENDPOINT
+genai-perf profile -m TEST_MODEL --endpoint-type NEW-ENDPOINT
 ```
 
 You can also write unit tests to ensure it works as expected.

genai-perf/genai_perf/inputs/input_constants.py

@@ -85,6 +85,8 @@ def to_lowercase(self):
 DEFAULT_OUTPUT_TOKENS_MEAN = -1
 DEFAULT_OUTPUT_TOKENS_STDDEV = 0
 DEFAULT_NUM_DATASET_ENTRIES = 100
+DEFAULT_NUM_PREFIX_PROMPTS = 0
+DEFAULT_PREFIX_PROMPT_LENGTH = 100
 
 ###########################
 # Default Image Parameters

genai-perf/genai_perf/inputs/inputs_config.py

@@ -35,8 +35,10 @@
     DEFAULT_IMAGE_WIDTH_STDDEV,
     DEFAULT_LENGTH,
     DEFAULT_NUM_DATASET_ENTRIES,
+    DEFAULT_NUM_PREFIX_PROMPTS,
     DEFAULT_OUTPUT_TOKENS_MEAN,
     DEFAULT_OUTPUT_TOKENS_STDDEV,
+    DEFAULT_PREFIX_PROMPT_LENGTH,
     DEFAULT_PROMPT_TOKENS_MEAN,
     DEFAULT_PROMPT_TOKENS_STDDEV,
     DEFAULT_RANDOM_SEED,
@@ -142,3 +144,9 @@ class InputsConfig:
 
     # Seed used to generate random values
     random_seed: int = DEFAULT_RANDOM_SEED
+
+    # The number of prefix prompts to generate and pool from
+    num_prefix_prompts: int = DEFAULT_NUM_PREFIX_PROMPTS
+
+    # The length of the prefix prompts to generate
+    prefix_prompt_length: int = DEFAULT_PREFIX_PROMPT_LENGTH

genai-perf/genai_perf/inputs/retrievers/file_input_retriever.py

@@ -39,6 +39,9 @@
     GenericDataset,
 )
 from genai_perf.inputs.retrievers.synthetic_image_generator import ImageFormat
+from genai_perf.inputs.retrievers.synthetic_prompt_generator import (
+    SyntheticPromptGenerator,
+)
 from genai_perf.utils import load_json_str
 from PIL import Image
 
@@ -152,6 +155,15 @@ def _get_content_from_input_file(
         """
         prompts = []
         images = []
+
+        use_prefix_prompts = self.config.num_prefix_prompts > 0
+        if use_prefix_prompts:
+            SyntheticPromptGenerator.create_prefix_prompts_pool(
+                self.config.tokenizer,
+                self.config.num_prefix_prompts,
+                self.config.prefix_prompt_length,
+            )
+
         with open(filename, mode="r", newline=None) as file:
             for line in file:
                 if line.strip():
@@ -164,7 +176,13 @@ def _get_content_from_input_file(
                             "Each data entry must have only one of 'text_input' or 'text' key name."
                         )
                     prompt = prompt if prompt else prompt_alt
-                    prompts.append(prompt.strip() if prompt else prompt)
+                    if use_prefix_prompts:
+                        prefix_prompt = (
+                            SyntheticPromptGenerator.get_random_prefix_prompt()
+                        )
+                        prompt = f"{prefix_prompt} {prompt}"
+                    if prompt is not None:
+                        prompts.append(prompt.strip())
                     image = data.get("image")
                     if image is not None:
                         image = self._encode_image(image.strip())

genai-perf/genai_perf/inputs/retrievers/synthetic_data_retriever.py

@@ -51,17 +51,30 @@ def retrieve_data(self) -> GenericDataset:
         files = self.config.synthetic_input_filenames or [DEFAULT_SYNTHETIC_FILENAME]
         synthetic_dataset = GenericDataset(files_data={})
 
+        use_prefix_prompts = self.config.num_prefix_prompts > 0
+        if use_prefix_prompts:
+            SyntheticPromptGenerator.create_prefix_prompts_pool(
+                self.config.tokenizer,
+                self.config.num_prefix_prompts,
+                self.config.prefix_prompt_length,
+            )
+
         for file in files:
             data_rows: List[DataRow] = []
 
             for _ in range(self.config.num_dataset_entries):
                 row = DataRow(texts=[], images=[])
-                prompt = SyntheticPromptGenerator.create_synthetic_prompt(
-                    self.config.tokenizer,
-                    self.config.prompt_tokens_mean,
-                    self.config.prompt_tokens_stddev,
-                )
                 for _ in range(self.config.batch_size_text):
+                    prompt = SyntheticPromptGenerator.create_synthetic_prompt(
+                        self.config.tokenizer,
+                        self.config.prompt_tokens_mean,
+                        self.config.prompt_tokens_stddev,
+                    )
+                    if use_prefix_prompts:
+                        prefix_prompt = (
+                            SyntheticPromptGenerator.get_random_prefix_prompt()
+                        )
+                        prompt = f"{prefix_prompt} {prompt}"
                     row.texts.append(prompt)
 
                 for _ in range(self.config.batch_size_image):

genai-perf/genai_perf/inputs/retrievers/synthetic_prompt_generator.py

@@ -24,6 +24,7 @@
 class SyntheticPromptGenerator:
     _tokenized_corpus = None
     _corpus_length = 0
+    _prefix_prompts: List[str] = []
 
     @classmethod
     def create_synthetic_prompt(
@@ -106,3 +107,32 @@ def _generate_prompt(cls, tokenizer: Tokenizer, num_tokens: int) -> str:
             prompt_tokens += cls._tokenized_corpus[: end_idx - cls._corpus_length]
 
         return tokenizer.decode(prompt_tokens)
+
+    @classmethod
+    def create_prefix_prompts_pool(
+        cls, tokenizer: Tokenizer, num_prompts: int, prompt_length: int
+    ) -> None:
+        """
+        Generate a pool of prefix prompts.
+
+        Args:
+            tokenizer: Tokenizer instance.
+            num_prompts: Number of prefix prompts to generate.
+            prompt_length: Number of tokens per prefix prompt.
+        """
+        if cls._tokenized_corpus is None:
+            cls._initialize_corpus(tokenizer)
+
+        cls._prefix_prompts = [
+            cls._generate_prompt(tokenizer, prompt_length) for _ in range(num_prompts)
+        ]
+
+    @classmethod
+    def get_random_prefix_prompt(cls) -> str:
+        """
+        Fetch a random prefix prompt from the pool.
+
+        Returns:
+            A random prefix prompt.
+        """
+        return random.choice(cls._prefix_prompts)

genai-perf/genai_perf/main.py

@@ -96,6 +96,8 @@ def create_config_options(args: Namespace) -> InputsConfig:
         batch_size_image=args.batch_size_image,
         batch_size_text=args.batch_size_text,
         output_dir=args.artifact_dir,
+        num_prefix_prompts=args.num_prefix_prompts,
+        prefix_prompt_length=args.prefix_prompt_length,
     )
 
 

genai-perf/genai_perf/parser.py

@@ -506,6 +506,17 @@ def _add_input_args(parser):
         "These will be reused until benchmarking is complete.",
     )
 
+    input_group.add_argument(
+        "--num-prefix-prompts",
+        type=int,
+        default=ic.DEFAULT_NUM_PREFIX_PROMPTS,
+        required=False,
+        help=f"The number of prefix prompts to select from. "
+        "If this value is not zero, these are prompts that are "
+        "prepended to input prompts. This is useful for "
+        "benchmarking models that use a K-V cache.",
+    )
+
     input_group.add_argument(
         "--output-tokens-mean",
         "--osl",
@@ -573,6 +584,17 @@ def _add_input_args(parser):
         help=f"The standard deviation of number of tokens in the generated prompts when using synthetic data.",
     )
 
+    input_group.add_argument(
+        "--prefix-prompt-length",
+        type=int,
+        default=ic.DEFAULT_PREFIX_PROMPT_LENGTH,
+        required=False,
+        help=f"The number of tokens in each prefix prompt. This value is only "
+        "used if --num-prefix-prompts is positive. Note that due to "
+        "the prefix and user prompts being concatenated, the number of tokens "
+        "in the final prompt may be off by one.",
+    )
+
     input_group.add_argument(
         "--warmup-request-count",
         type=int,

genai-perf/genai_perf/wrapper.py

@@ -88,6 +88,7 @@ def build_cmd(args: Namespace, extra_args: Optional[List[str]] = None) -> List[s
             "model",
             "model_selection_strategy",
             "num_dataset_entries",
+            "num_prefix_prompts",
             "output_format",
             "output_tokens_mean",
             "output_tokens_mean_deterministic",
@@ -105,6 +106,7 @@ def build_cmd(args: Namespace, extra_args: Optional[List[str]] = None) -> List[s
             "synthetic_input_files",
             "synthetic_input_tokens_mean",
             "synthetic_input_tokens_stddev",
+            "prefix_prompt_length",
             "tokenizer",
             "tokenizer_trust_remote_code",
             "tokenizer_revision",

genai-perf/tests/test_cli.py

@@ -192,6 +192,7 @@ def test_help_version_arguments_output_and_exit(
             ),
             (["--num-dataset-entries", "101"], {"num_dataset_entries": 101}),
             (["--num-prompts", "101"], {"num_dataset_entries": 101}),
+            (["--num-prefix-prompts", "101"], {"num_prefix_prompts": 101}),
             (
                 ["--output-tokens-mean", "6"],
                 {"output_tokens_mean": 6},
@@ -245,6 +246,10 @@ def test_help_version_arguments_output_and_exit(
                 ["--synthetic-input-tokens-stddev", "7"],
                 {"synthetic_input_tokens_stddev": 7},
             ),
+            (
+                ["--prefix-prompt-length", "6"],
+                {"prefix_prompt_length": 6},
+            ),
             (
                 ["--image-width-mean", "123"],
                 {"image_width_mean": 123},

genai-perf/tests/test_file_input_retriever.py

@@ -346,3 +346,39 @@ def test_get_input_datasets_from_empty_dir(
         )
         with pytest.raises(ValueError, match="No JSONL files found in directory"):
             _ = file_retriever._get_input_datasets_from_dir()
+
+    @patch("builtins.open", side_effect=open_side_effect)
+    @patch(
+        "genai_perf.inputs.retrievers.file_input_retriever.SyntheticPromptGenerator.create_prefix_prompts_pool"
+    )
+    @patch(
+        "genai_perf.inputs.retrievers.file_input_retriever.SyntheticPromptGenerator.get_random_prefix_prompt",
+        return_value="prefix prompt",
+    )
+    @patch("pathlib.Path.exists", return_value=True)
+    def test_get_input_file_multiple_prompts_with_prefix_prompts(
+        self,
+        mock_exists,
+        mock_random_prefix_prompt,
+        mock_create_prefix_prompts_pool,
+        mock_file,
+    ):
+        file_retriever = FileInputRetriever(
+            InputsConfig(
+                tokenizer=get_empty_tokenizer(),
+                model_name=["test_model_A"],
+                model_selection_strategy=ModelSelectionStrategy.ROUND_ROBIN,
+                input_filename=Path("multiple_prompts.jsonl"),
+                num_prefix_prompts=3,
+                prefix_prompt_length=15,
+            )
+        )
+        file_data = file_retriever._get_input_dataset_from_file(
+            Path("multiple_prompts.jsonl")
+        )
+
+        assert file_data is not None
+        assert len(file_data.rows) == 3
+        mock_create_prefix_prompts_pool.assert_called_once()
+        for row in file_data.rows:
+            assert row.texts[0].startswith("prefix prompt ")

genai-perf/tests/test_json_exporter.py

@@ -224,6 +224,7 @@ class TestJsonExporter:
           "streaming": true,
           "u": null,
           "num_dataset_entries": 100,
+          "num_prefix_prompts": 0,
           "output_tokens_mean": -1,
           "output_tokens_mean_deterministic": false,
           "output_tokens_stddev": 0,
@@ -232,6 +233,7 @@ class TestJsonExporter:
           "synthetic_input_files": null,
           "synthetic_input_tokens_mean": 550,
           "synthetic_input_tokens_stddev": 0,
+          "prefix_prompt_length": 100,
           "warmup_request_count": 0,
           "image_width_mean": 100,
           "image_width_stddev": 0,