Support payload input in GAP

genai-perf/genai_perf/export_data/json_exporter.py

@@ -63,10 +63,13 @@ def export(self) -> None:
         with open(str(filename), "w") as f:
             f.write(json.dumps(self._stats_and_args, indent=2))
 
+    def _exclude_args(self, args_to_exclude) -> None:
+        for arg in args_to_exclude:
+            self._args.pop(arg, None)
+
     def _prepare_args_for_export(self) -> None:
-        self._args.pop("func", None)
-        self._args.pop("output_format", None)
-        self._args.pop("input_file", None)
+        args_to_exclude = ["func", "output_format", "input_file", "payload_input_file"]
+        self._exclude_args(args_to_exclude)
         self._args["profile_export_file"] = str(self._args["profile_export_file"])
         self._args["artifact_dir"] = str(self._args["artifact_dir"])
         for k, v in self._args.items():

genai-perf/genai_perf/inputs/converters/base_converter.py

@@ -30,7 +30,7 @@
 from genai_perf.exceptions import GenAIPerfException
 from genai_perf.inputs.input_constants import ModelSelectionStrategy
 from genai_perf.inputs.inputs_config import InputsConfig
-from genai_perf.inputs.retrievers.generic_dataset import GenericDataset
+from genai_perf.inputs.retrievers.generic_dataset import DataRow, GenericDataset
 
 
 class BaseConverter:
@@ -70,3 +70,26 @@ def _add_request_params(
     ) -> None:
         for key, value in config.extra_inputs.items():
             payload[key] = value
+
+    def _add_payload_params(self, payload: Dict[Any, Any], optional_data) -> None:
+        for key, value in optional_data.items():
+            payload[key] = value
+
+    def _add_extra_params(
+        self, payload: Dict[Any, Any], config: InputsConfig, row: DataRow
+    ) -> None:
+        self._add_request_params(payload, config)
+        self._add_payload_params(payload, row.optional_data)
+
+    def _finalize_payload(
+        self, payload: Dict[Any, Any], row, triton_format=False
+    ) -> Dict[str, Any]:
+        record: Dict[str, Any] = {}
+        if not triton_format:
+            record["payload"] = [payload]
+        else:
+            record.update(payload)
+        if row.timestamp:
+            record["timestamp"] = [row.timestamp]
+
+        return record

genai-perf/genai_perf/inputs/converters/image_retrieval_converter.py

@@ -56,6 +56,7 @@ def convert(
                 payload = {
                     "input": [{"type": "image_url", "url": img} for img in row.images]
                 }
-                request_body["data"].append({"payload": [payload]})
+                self._add_payload_params(payload, row.optional_data)
+                request_body["data"].append(self._finalize_payload(payload, row))
 
         return request_body

genai-perf/genai_perf/inputs/converters/nvclip_converter.py

@@ -60,7 +60,7 @@ def convert(
                     "input": input_items,
                 }
 
-                self._add_request_params(payload, config)
-                request_body["data"].append({"payload": [payload]})
+                self._add_extra_params(payload, config, row)
+                request_body["data"].append(self._finalize_payload(payload, row))
 
         return request_body

genai-perf/genai_perf/inputs/converters/openai_chat_completions_converter.py

@@ -62,7 +62,7 @@ def convert(
         for file_data in generic_dataset.files_data.values():
             for index, row in enumerate(file_data.rows):
                 payload = self._create_payload(index, row, config)
-                request_body["data"].append({"payload": [payload]})
+                request_body["data"].append(self._finalize_payload(payload, row))
 
         return request_body
 
@@ -82,7 +82,7 @@ def _create_payload(
             ],
         }
 
-        self._add_request_params(payload, config)
+        self._add_extra_params(payload, config, row)
         return payload
 
     def _retrieve_content(

genai-perf/genai_perf/inputs/converters/openai_completions_converter.py

@@ -49,8 +49,8 @@ def convert(
                     "model": model_name,
                     "prompt": prompt,
                 }
-                self._add_request_params(payload, config)
-                request_body["data"].append({"payload": [payload]})
+                self._add_extra_params(payload, config, row)
+                request_body["data"].append(self._finalize_payload(payload, row))
 
         return request_body
 

genai-perf/genai_perf/inputs/converters/openai_embeddings_converter.py

@@ -53,7 +53,8 @@ def convert(
                     "model": model_name,
                     "input": row.texts,
                 }
-                self._add_request_params(payload, config)
-                request_body["data"].append({"payload": [payload]})
+
+                self._add_extra_params(payload, config, row)
+                request_body["data"].append(self._finalize_payload(payload, row))
 
         return request_body

genai-perf/genai_perf/inputs/converters/rankings_converter.py

@@ -79,8 +79,8 @@ def convert(
                     "model": model_name,
                 }
 
-            self._add_request_params(payload, config)
-            request_body["data"].append({"payload": [payload]})
+            self._add_extra_params(payload, config, passage_entry)
+            request_body["data"].append(self._finalize_payload(payload, passage_entry))
 
         return request_body
 

genai-perf/genai_perf/inputs/converters/tensorrtllm_converter.py

@@ -61,8 +61,11 @@ def convert(
                     "text_input": [text],
                     "max_tokens": [DEFAULT_TENSORRTLLM_MAX_TOKENS],  # default
                 }
-                self._add_request_params(payload, config)
-                request_body["data"].append(payload)
+
+                self._add_extra_params(payload, config, row)
+                request_body["data"].append(
+                    self._finalize_payload(payload, row, triton_format=True)
+                )
 
         return request_body
 

genai-perf/genai_perf/inputs/converters/tensorrtllm_engine_converter.py

@@ -61,8 +61,11 @@ def convert(
                     "input_lengths": [len(token_ids)],
                     "request_output_len": [DEFAULT_TENSORRTLLM_MAX_TOKENS],
                 }
-                self._add_request_params(payload, config)
-                request_body["data"].append(payload)
+
+                self._add_extra_params(payload, config, row)
+                request_body["data"].append(
+                    self._finalize_payload(payload, row, triton_format=True)
+                )
 
         return request_body
 

genai-perf/genai_perf/inputs/converters/triton_generate_converter.py

@@ -55,8 +55,13 @@ def convert(
                 payload = {
                     "text_input": prompt,
                 }
-                self._add_request_params(payload, config)
-                request_body["data"].append({"payload": [payload]})
+                self._add_extra_params(payload, config, row)
+                request_body["data"].append(
+                    self._finalize_payload(
+                        payload,
+                        row,
+                    )
+                )
 
         return request_body
 

genai-perf/genai_perf/inputs/converters/vllm_converter.py

@@ -61,8 +61,10 @@ def convert(
                     "text_input": text,
                     "exclude_input_in_output": [True],  # default
                 }
-                self._add_request_params(payload, config)
-                request_body["data"].append(payload)
+                self._add_extra_params(payload, config, row)
+                request_body["data"].append(
+                    self._finalize_payload(payload, row, triton_format=True)
+                )
 
         return request_body
 

genai-perf/genai_perf/inputs/input_constants.py

@@ -25,7 +25,6 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 from enum import Enum, auto
-from typing import Dict
 
 
 class ModelSelectionStrategy(Enum):

genai-perf/genai_perf/inputs/inputs_config.py

@@ -82,6 +82,9 @@ class InputsConfig:
     # The filenames used for synthetic data generation
     synthetic_input_filenames: Optional[List[str]] = field(default_factory=list)
 
+    # The filename where payload input data is available
+    payload_input_filename: Optional[Path] = Path("")
+
     # The compression format of the images.
     image_format: ImageFormat = ImageFormat.PNG
 

genai-perf/genai_perf/inputs/retrievers/base_file_input_retriever.py

@@ -0,0 +1,79 @@
+# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+from pathlib import Path
+from typing import Any, Dict, List, Tuple, Union
+
+from genai_perf.inputs.retrievers.base_input_retriever import BaseInputRetriever
+from genai_perf.inputs.retrievers.generic_dataset import FileData, GenericDataset
+
+
+class BaseFileInputRetriever(BaseInputRetriever):
+    """
+    A base input retriever class that defines file input methods.
+    """
+
+    def _verify_file(self, filename: Path) -> None:
+        """
+        Verifies that the file exists.
+
+        Args
+        ----------
+        filename : Path
+            The file path to verify.
+
+        Raises
+        ------
+        FileNotFoundError
+            If the file does not exist.
+        """
+        if not filename.exists():
+            raise FileNotFoundError(f"The file '{filename}' does not exist.")
+
+    def _get_content_from_input_file(self, filename: Path) -> Union[
+        Tuple[List[str], List[str]],
+        Tuple[List[str], List[str], List[Dict[Any, Any]]],
+    ]:
+        """
+        Reads the content from a JSONL file and returns lists of each content type.
+
+        """
+        raise NotImplementedError("This method should be implemented by subclasses.")
+
+    def _get_input_dataset_from_file(self, filename: Path) -> FileData:
+        """
+        Retrieves the dataset from a specific JSONL file.
+
+        """
+
+        raise NotImplementedError("This method should be implemented by subclasses.")
+
+    def retrieve_data(self) -> GenericDataset:
+        """
+        Retrieves the dataset from a file or directory.
+        """
+        raise NotImplementedError("This method should be implemented by subclasses.")

genai-perf/genai_perf/inputs/retrievers/file_input_retriever.py

@@ -31,8 +31,9 @@
 from genai_perf import utils
 from genai_perf.exceptions import GenAIPerfException
 from genai_perf.inputs.input_constants import DEFAULT_BATCH_SIZE
-from genai_perf.inputs.inputs_config import InputsConfig
-from genai_perf.inputs.retrievers.base_input_retriever import BaseInputRetriever
+from genai_perf.inputs.retrievers.base_file_input_retriever import (
+    BaseFileInputRetriever,
+)
 from genai_perf.inputs.retrievers.generic_dataset import (
     DataRow,
     FileData,
@@ -46,7 +47,7 @@
 from PIL import Image
 
 
-class FileInputRetriever(BaseInputRetriever):
+class FileInputRetriever(BaseFileInputRetriever):
     """
     A input retriever class that handles input data provided by the user through
     file and directories.
@@ -118,24 +119,7 @@ def _get_input_dataset_from_file(self, filename: Path) -> FileData:
         """
         self._verify_file(filename)
         prompts, images = self._get_content_from_input_file(filename)
-        return self._convert_content_to_data_file(prompts, images, filename)
-
-    def _verify_file(self, filename: Path) -> None:
-        """
-        Verifies that the file exists.
-
-        Args
-        ----------
-        filename : Path
-            The file path to verify.
-
-        Raises
-        ------
-        FileNotFoundError
-            If the file does not exist.
-        """
-        if not filename.exists():
-            raise FileNotFoundError(f"The file '{filename}' does not exist.")
+        return self._convert_content_to_data_file(prompts, filename, images)
 
     def _get_content_from_input_file(
         self, filename: Path
@@ -224,7 +208,7 @@ def _encode_image(self, filename: str) -> str:
         return payload
 
     def _convert_content_to_data_file(
-        self, prompts: List[str], images: List[str], filename: Path
+        self, prompts: List[str], filename: Path, images: List[str] = []
     ) -> FileData:
         """
         Converts the content to a DataFile.

genai-perf/genai_perf/inputs/retrievers/generic_dataset.py

@@ -25,25 +25,37 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 from dataclasses import dataclass, field
-from typing import Dict, List, TypeAlias
+from typing import Any, Dict, List, TypeAlias, Union
 
 Filename: TypeAlias = str
 TypeOfData: TypeAlias = str
 ListOfData: TypeAlias = List[str]
-DataRowDict: TypeAlias = Dict[TypeOfData, ListOfData]
+DataRowDict: TypeAlias = Dict[str, Union[List[str], Dict[str, Any], str]]
 GenericDatasetDict: TypeAlias = Dict[Filename, List[DataRowDict]]
 
 
 @dataclass
 class DataRow:
     texts: List[str] = field(default_factory=list)
     images: List[str] = field(default_factory=list)
+    timestamp: str = ""
+    optional_data: Dict[str, Any] = field(default_factory=dict)
 
     def to_dict(self) -> DataRowDict:
         """
         Converts the DataRow object to a dictionary.
         """
-        return {"texts": self.texts, "images": self.images}
+        datarow_dict: DataRowDict = {}
+
+        if self.texts:
+            datarow_dict["texts"] = self.texts
+        if self.images:
+            datarow_dict["images"] = self.images
+        if self.timestamp:
+            datarow_dict["timestamp"] = self.timestamp
+        if self.optional_data:
+            datarow_dict["optional_data"] = self.optional_data
+        return datarow_dict
 
 
 @dataclass
@@ -55,8 +67,8 @@ def to_list(self) -> List[DataRowDict]:
         Converts the FileData object to a list.
         Output format example for two payloads from a file:
         [
-            {'texts': ['text1', 'text2'], 'images': ['image1', 'image2']},
-            {'texts': ['text3', 'text4'], 'images': ['image3', 'image4']}
+            {'texts': ['text1', 'text2'], 'images': ['image1', 'image2'], 'timestamp': 'timestamp1', 'optional_data': {}},
+            {'texts': ['text3', 'text4'], 'images': ['image3', 'image4'], 'timestamp': 'timestamp2', 'optional_data': {}},
         ]
         """
         return [row.to_dict() for row in self.rows]
@@ -71,8 +83,8 @@ def to_dict(self) -> GenericDatasetDict:
         Converts the entire DataStructure object to a dictionary.
         Output format example for one payload from two files:
         {
-            'file_0': [{'texts': ['text1', 'text2'], 'images': ['image1', 'image2']}],
-            'file_1': [{'texts': ['text1', 'text2'], 'images': ['image1', 'image2']}]
+            'file_0': [{'texts': ['text1', 'text2'], 'images': ['image1', 'image2'],  'timestamp': 'timestamp1', 'optional_data': {}}],
+            'file_1': [{'texts': ['text1', 'text2'], 'images': ['image1', 'image2'],  'timestamp': 'timestamp2', 'optional_data': {}}],
         }
         """
         return {