Fixing merge conflicts. Updating num_prompts -> num_dataset_entries

genai-perf/genai_perf/config/generate/genai_perf_config.py

@@ -83,8 +83,8 @@ def get_obj_args(self) -> Namespace:
             obj_args.synthetic_input_tokens_mean = self._parameters[
                 "input_sequence_length"
             ]
-        if "num_prompts" in self._parameters:
-            obj_args.num_prompts = self._parameters["num_prompts"]
+        if "num_dataset_entries" in self._parameters:
+            obj_args.num_dataset_entries = self._parameters["num_dataset_entries"]
 
         return obj_args
 

genai-perf/genai_perf/config/generate/perf_analyzer_config.py

@@ -20,7 +20,11 @@
 from typing import Any, Dict, List, Optional
 
 from genai_perf.config.generate.search_parameter import SearchUsage
-from genai_perf.config.input.config_command import ConfigCommand, ConfigPerfAnalyzer
+from genai_perf.config.input.config_command import (
+    ConfigCommand,
+    ConfigPerfAnalyzer,
+    RunConfigDefaults,
+)
 from genai_perf.constants import DEFAULT_ARTIFACT_DIR
 from genai_perf.exceptions import GenAIPerfException
 from genai_perf.inputs.input_constants import DEFAULT_INPUT_DATA_JSON
@@ -58,7 +62,9 @@
     "input_format",
     "model",
     "model_selection_strategy",
-    "num_prompts",
+    "num_dataset_entries",
+    "request_count",
+    "warmup_request_count",
     "output_format",
     "output_tokens_mean",
     "output_tokens_mean_deterministic",
@@ -90,6 +96,7 @@
 
 
 class InferenceType(Enum):
+    NONE = auto()
     CONCURRENCY = auto()
     REQUEST_RATE = auto()
 
@@ -228,11 +235,11 @@ def _get_artifact_stimulus_type(
                 parameters["input_sequence_length"].get_value_based_on_category()
             )
             stimulus = [f"input_sequence_length{input_sequence_length}"]
-        elif "num_prompts" in parameters:
+        elif "num_dataset_entries" in parameters:
             input_sequence_length = str(
-                parameters["num_prompts"].get_value_based_on_category()
+                parameters["num_dataset_entries"].get_value_based_on_category()
             )
-            stimulus = [f"num_prompts{input_sequence_length}"]
+            stimulus = [f"num_dataset_entries{input_sequence_length}"]
 
         return stimulus
 
@@ -327,24 +334,26 @@ def get_inference_type(self) -> InferenceType:
         elif "--request-rate-range" in cmd:
             return InferenceType.REQUEST_RATE
         else:
-            raise GenAIPerfException(
-                f"An inference type (either concurrency or request-rate) was not found."
-            )
+            return InferenceType.NONE
 
     def get_inference_value(self) -> int:
         """
         Returns the value that we are inferencing
         """
         infer_type = self.get_inference_type()
-        infer_cmd_option = (
-            "--concurrency-range"
-            if infer_type == InferenceType.CONCURRENCY
-            else "--request-rate-range"
-        )
 
-        cmd = self.create_command()
-        infer_value_index = cmd.index(infer_cmd_option) + 1
-        infer_value = int(cmd[infer_value_index])
+        if infer_type == InferenceType.NONE:
+            infer_value = RunConfigDefaults.MIN_CONCURRENCY
+        else:
+            infer_cmd_option = (
+                "--concurrency-range"
+                if infer_type == InferenceType.CONCURRENCY
+                else "--request-rate-range"
+            )
+
+            cmd = self.create_command()
+            infer_value_index = cmd.index(infer_cmd_option) + 1
+            infer_value = int(cmd[infer_value_index])
 
         return infer_value
 

genai-perf/genai_perf/config/generate/search_parameters.py

@@ -41,7 +41,7 @@ class SearchParameters:
         "input_sequence_length",
     ]
 
-    linear_range_parameters = ["instance_count", "num_prompts"]
+    linear_range_parameters = ["instance_count", "num_dataset_entries"]
 
     model_parameters = [
         "model_batch_size",
@@ -51,7 +51,7 @@ class SearchParameters:
 
     runtime_pa_parameters = ["runtime_batch_size", "concurrency", "request_rate"]
 
-    runtime_gap_parameters = ["num_prompts", "input_sequence_length"]
+    runtime_gap_parameters = ["num_dataset_entries", "input_sequence_length"]
 
     all_parameters = model_parameters + runtime_pa_parameters + runtime_gap_parameters
 
@@ -198,20 +198,20 @@ def _populate_request_rate(self) -> None:
     # GenAI Perf Parameters
     ###########################################################################
     def _populate_genai_perf_parameters(self) -> None:
-        self._populate_genai_perf_num_prompts()
+        self._populate_genai_perf_num_dataset_entries()
 
-    def _populate_genai_perf_num_prompts(self) -> None:
-        if isinstance(self._config.genai_perf.num_prompts, list):
+    def _populate_genai_perf_num_dataset_entries(self) -> None:
+        if isinstance(self._config.genai_perf.num_dataset_entries, list):
             self._populate_list_parameter(
-                parameter_name="num_prompts",
-                parameter_list=list(self._config.genai_perf.num_prompts),
+                parameter_name="num_dataset_entries",
+                parameter_list=list(self._config.genai_perf.num_dataset_entries),
                 parameter_category=SearchCategory.INT_LIST,
             )
-        elif isinstance(self._config.genai_perf.num_prompts, Range):
+        elif isinstance(self._config.genai_perf.num_dataset_entries, Range):
             self._populate_range_parameter(
-                parameter_name="num_prompts",
-                parameter_min_value=self._config.genai_perf.num_prompts.min,
-                parameter_max_value=self._config.genai_perf.num_prompts.max,
+                parameter_name="num_dataset_entries",
+                parameter_min_value=self._config.genai_perf.num_dataset_entries.min,
+                parameter_max_value=self._config.genai_perf.num_dataset_entries.max,
             )
 
     ###########################################################################

genai-perf/genai_perf/config/input/config_command.py

@@ -149,7 +149,9 @@ def is_request_rate_specified(self) -> bool:
 
 @dataclass
 class ConfigOptimizeGenAIPerf:
-    num_prompts: ConfigRangeOrList = default_field([RunConfigDefaults.NUM_PROMPTS])
+    num_dataset_entries: ConfigRangeOrList = default_field(
+        [RunConfigDefaults.NUM_PROMPTS]
+    )
 
 
 @dataclass
@@ -200,7 +202,7 @@ class ConfigSyntheticTokens:
 class ConfigInput:
     dataset: str = default_field(RunConfigDefaults.DATASET)
     file: str = default_field(RunConfigDefaults.FILE)
-    num_prompts: int = default_field(RunConfigDefaults.NUM_PROMPTS)
+    num_dataset_entries: int = default_field(RunConfigDefaults.NUM_PROMPTS)
     seed: int = default_field(RunConfigDefaults.SEED)
     synthetic_tokens: ConfigSyntheticTokens = ConfigSyntheticTokens()
 

genai-perf/genai_perf/parser.py

@@ -916,7 +916,12 @@ def _add_analyze_args(parser):
         "--sweep-type",
         type=str,
         default=RunConfigDefaults.STIMULUS_TYPE,
-        choices=["concurrency", "num_prompts", "input_sequence_length", "request_rate"],
+        choices=[
+            "concurrency",
+            "num_dataset_entries",
+            "input_sequence_length",
+            "request_rate",
+        ],
         required=False,
         help=f"The stimulus type that GAP will sweep.",
     )

genai-perf/genai_perf/subcommand/analyze.py

@@ -294,7 +294,7 @@ def _create_stimulus_header(self) -> List[str]:
         infer_type = self._determine_infer_type()
         infer_header = "Concurrency" if infer_type == "concurrency" else "Request Rate"
 
-        stimulus_header = ["Config Name", infer_header, "ISL", "Num Prompts"]
+        stimulus_header = ["Config Name", infer_header, "ISL", "Num Dataset Entries"]
 
         return stimulus_header
 
@@ -311,14 +311,14 @@ def _write_perf_metrics_body(self, csv_writer) -> None:
                     self._model_name, InputSequenceLengthP99.tag
                 )
             )
-            num_prompts = run_config.genai_perf_config.input.num_prompts
+            num_dataset_entries = run_config.genai_perf_config.input.num_dataset_entries
 
             metrics = []
             for tag in Analyze.PERF_METRICS_TAGS:
                 metric = run_config.get_model_perf_metric_value(self._model_name, tag)
                 metrics.append(f"{metric:.2f}")
 
-            row = [run_config.name, infer_value, isl, num_prompts] + metrics
+            row = [run_config.name, infer_value, isl, num_dataset_entries] + metrics
             csv_writer.writerow(row)
 
     ###########################################################################
@@ -368,7 +368,7 @@ def _determine_infer_mode_and_load_level(
             load_level = f"{float(objectives[model_name][infer_mode].get_value_based_on_category())}"
         elif (
             args.sweep_type == "input_sequence_length"
-            or args.sweep_type == "num_prompts"
+            or args.sweep_type == "num_dataset_entries"
         ):
             if args.concurrency:
                 infer_mode = "concurrency"

genai-perf/genai_perf/subcommand/common.py

@@ -162,7 +162,7 @@ def create_config_options(args: Namespace) -> InputsConfig:
         input_filename=args.input_file,
         synthetic_input_filenames=args.synthetic_input_files,
         starting_index=DEFAULT_STARTING_INDEX,
-        length=args.num_prompts,
+        length=args.num_dataset_entries,
         prompt_tokens_mean=args.synthetic_input_tokens_mean,
         prompt_tokens_stddev=args.synthetic_input_tokens_stddev,
         output_tokens_mean=args.output_tokens_mean,
@@ -174,7 +174,7 @@ def create_config_options(args: Namespace) -> InputsConfig:
         image_height_stddev=args.image_height_stddev,
         image_format=args.image_format,
         random_seed=args.random_seed,
-        num_prompts=args.num_prompts,
+        num_dataset_entries=args.num_dataset_entries,
         add_stream=args.streaming,
         tokenizer=get_tokenizer(
             args.tokenizer, args.tokenizer_trust_remote_code, args.tokenizer_revision

genai-perf/tests/test_genai_perf_config.py

@@ -39,7 +39,7 @@ def setUp(self):
 
         self._objective_parameters = {
             "test_model": {
-                "num_prompts": ObjectiveParameter(
+                "num_dataset_entries": ObjectiveParameter(
                     SearchUsage.RUNTIME_GAP,
                     ObjectiveCategory.INTEGER,
                     50,
@@ -64,7 +64,7 @@ def test_default_config_and_objective_capture(self):
         at __init__
         """
         expected_input_config = ConfigInput()
-        expected_input_config.num_prompts = 50
+        expected_input_config.num_dataset_entries = 50
 
         self.assertEqual(expected_input_config, self._default_genai_perf_config.input)
 
@@ -82,7 +82,10 @@ def test_representation(self):
         Test that the representation is created correctly
         """
         expected_representation = " ".join(
-            [ConfigInput(num_prompts=50).__str__(), ConfigOutputTokens().__str__()]
+            [
+                ConfigInput(num_dataset_entries=50).__str__(),
+                ConfigOutputTokens().__str__(),
+            ]
         )
         representation = self._default_genai_perf_config.representation()
 

genai-perf/tests/test_optuna_objective_generator.py

@@ -156,7 +156,7 @@ def test_create_trial_objectives(self):
                 "model_batch_size": 4,
                 "runtime_batch_size": 1,
                 "instance_count": 2,
-                "num_prompts": 100,
+                "num_dataset_entries": 100,
                 "concurrency": 6,
             }
         }
@@ -186,7 +186,7 @@ def test_create_objective_parameters(self):
                     ObjectiveCategory.INTEGER,
                     RunConfigDefaults.PA_BATCH_SIZE,
                 ),
-                "num_prompts": ObjectiveParameter(
+                "num_dataset_entries": ObjectiveParameter(
                     SearchUsage.RUNTIME_GAP,
                     ObjectiveCategory.INTEGER,
                     RunConfigDefaults.NUM_PROMPTS,

genai-perf/tests/test_search_parameters.py

@@ -328,7 +328,7 @@ def test_custom_analyze_config(self):
         """
         config = deepcopy(self.config)
         config.analyze.sweep_parameters = {
-            "num_prompts": [10, 50, 100],
+            "num_dataset_entries": [10, 50, 100],
             "request_rate": Range(
                 min=RunConfigDefaults.MIN_REQUEST_RATE,
                 max=RunConfigDefaults.MAX_REQUEST_RATE,
@@ -338,10 +338,10 @@ def test_custom_analyze_config(self):
             config=config, subcommand=Subcommand.ANALYZE
         )
 
-        num_prompts = search_parameters.get_parameter("num_prompts")
-        self.assertEqual(SearchUsage.RUNTIME_GAP, num_prompts.usage)
-        self.assertEqual(SearchCategory.INT_LIST, num_prompts.category)
-        self.assertEqual([10, 50, 100], num_prompts.enumerated_list)
+        num_dataset_entries = search_parameters.get_parameter("num_dataset_entries")
+        self.assertEqual(SearchUsage.RUNTIME_GAP, num_dataset_entries.usage)
+        self.assertEqual(SearchCategory.INT_LIST, num_dataset_entries.category)
+        self.assertEqual([10, 50, 100], num_dataset_entries.enumerated_list)
 
         request_rate = search_parameters.get_parameter("request_rate")
         self.assertEqual(SearchUsage.RUNTIME_PA, request_rate.usage)