diff --git a/genai-perf/genai_perf/export_data/exporter_config.py b/genai-perf/genai_perf/export_data/exporter_config.py index 452e9947..2fb8803e 100644 --- a/genai-perf/genai_perf/export_data/exporter_config.py +++ b/genai-perf/genai_perf/export_data/exporter_config.py @@ -25,62 +25,19 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import argparse as args +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, Optional + from genai_perf.metrics import Metrics +@dataclass class ExporterConfig: - def __init__(self): - self._stats = None - self._telemetry_stats = None - self._metrics = None - self._args = None - self._extra_inputs = None - self._artifact_dir = None - - @property - def stats(self): - return self._stats - - @stats.setter - def stats(self, stats_value): - self._stats = stats_value - - @property - def telemetry_stats(self): - return self._telemetry_stats - - @telemetry_stats.setter - def telemetry_stats(self, stats_value): - self._telemetry_stats = stats_value - - @property - def metrics(self): - return self._metrics - - @metrics.setter - def metrics(self, metrics: Metrics): - self._metrics = metrics - - @property - def args(self): - return self._args - - @args.setter - def args(self, args_value): - self._args = args_value - - @property - def extra_inputs(self): - return self._extra_inputs - - @extra_inputs.setter - def extra_inputs(self, extra_inputs_value): - self._extra_inputs = extra_inputs_value - - @property - def artifact_dir(self): - return self._artifact_dir - - @artifact_dir.setter - def artifact_dir(self, artifact_dir_value): - self._artifact_dir = artifact_dir_value + stats: Dict[Any, Any] + metrics: Metrics + args: args.Namespace + extra_inputs: Dict[str, Any] + artifact_dir: Path + telemetry_stats: Optional[Dict[str, Any]] = None diff --git a/genai-perf/genai_perf/export_data/output_reporter.py b/genai-perf/genai_perf/export_data/output_reporter.py index 9876b712..82807d5f 100644 --- a/genai-perf/genai_perf/export_data/output_reporter.py +++ b/genai-perf/genai_perf/export_data/output_reporter.py @@ -29,7 +29,7 @@ from genai_perf.export_data.data_exporter_factory import DataExporterFactory from genai_perf.export_data.exporter_config import ExporterConfig -from genai_perf.metrics import Statistics, TelemetryStatistics +from genai_perf.metrics import Metrics, Statistics, TelemetryStatistics from genai_perf.subcommand.common import get_extra_inputs_as_dict @@ -60,13 +60,18 @@ def report_output(self) -> None: exporter.export() def _create_exporter_config(self) -> ExporterConfig: - config = ExporterConfig() - config.stats = self.stats.stats_dict - config.telemetry_stats = ( + assert isinstance(self.stats.metrics, Metrics) + extra_inputs = get_extra_inputs_as_dict(self.args) + telemetry_stats = ( self.telemetry_stats.stats_dict if self.telemetry_stats else None ) - config.metrics = self.stats.metrics - config.args = self.args - config.artifact_dir = self.args.artifact_dir - config.extra_inputs = get_extra_inputs_as_dict(self.args) + config = ExporterConfig( + self.stats.stats_dict, + self.stats.metrics, + self.args, + extra_inputs, + self.args.artifact_dir, + telemetry_stats, + ) + return config diff --git a/genai-perf/genai_perf/metrics/telemetry_statistics.py b/genai-perf/genai_perf/metrics/telemetry_statistics.py index 79c0164d..511b2d41 100755 --- a/genai-perf/genai_perf/metrics/telemetry_statistics.py +++ b/genai-perf/genai_perf/metrics/telemetry_statistics.py @@ -48,11 +48,18 @@ def __init__(self, metrics: TelemetryMetrics): for attr, data in self._metrics.data.items(): if self._should_skip(data): continue + + gpu_index = None + gpu_data = None + for gpu_index, gpu_data in data.items(): self._stats_dict[attr][gpu_index]["avg"] = ( self._statistics._calculate_mean(gpu_data) ) if not self._is_constant_metric(attr): + if gpu_data is None or gpu_index is None: + continue + percentile_results = self._statistics._calculate_percentiles(gpu_data) for percentile_label, percentile_value in percentile_results.items(): self._stats_dict[attr][gpu_index][ @@ -121,5 +128,5 @@ def _is_constant_metric(self, attr: str) -> bool: return attr in ["gpu_power_limit", "total_gpu_memory"] @property - def stats_dict(self) -> Dict: + def stats_dict(self) -> Dict[str, Any]: return self._stats_dict diff --git a/genai-perf/genai_perf/subcommand/common.py b/genai-perf/genai_perf/subcommand/common.py index 4b310c7a..21a2e8db 100644 --- a/genai-perf/genai_perf/subcommand/common.py +++ b/genai-perf/genai_perf/subcommand/common.py @@ -27,7 +27,7 @@ import os import subprocess # nosec from argparse import Namespace -from typing import List, Optional +from typing import Any, Dict, List, Optional import genai_perf.logging as logging from genai_perf.config.generate.perf_analyzer_config import PerfAnalyzerConfig @@ -81,7 +81,7 @@ def calculate_metrics(args: Namespace, tokenizer: Tokenizer) -> ProfileDataParse ) -def get_extra_inputs_as_dict(args: Namespace) -> dict: +def get_extra_inputs_as_dict(args: Namespace) -> Dict[str, Any]: request_inputs = {} if args.extra_inputs: for input_str in args.extra_inputs: diff --git a/genai-perf/tests/test_exporters/test_console_exporter.py b/genai-perf/tests/test_exporters/test_console_exporter.py index 6d1d8b0c..37de151c 100644 --- a/genai-perf/tests/test_exporters/test_console_exporter.py +++ b/genai-perf/tests/test_exporters/test_console_exporter.py @@ -39,6 +39,7 @@ TelemetryMetrics, TelemetryStatistics, ) +from tests.test_utils import create_default_exporter_config class TestConsoleExporter: @@ -69,11 +70,10 @@ def test_streaming_llm_output(self, monkeypatch, capsys) -> None: input_sequence_lengths=[5, 6, 7], ) stats = Statistics(metrics=metrics) - - config = ExporterConfig() - config.stats = stats.stats_dict - config.metrics = stats.metrics - config.args = args + assert isinstance(stats.metrics, Metrics) + config = create_default_exporter_config( + stats=stats.stats_dict, metrics=stats.metrics, args=args + ) exporter = ConsoleExporter(config) exporter.export() @@ -124,10 +124,10 @@ def test_nonstreaming_llm_output(self, monkeypatch, capsys) -> None: ) stats = Statistics(metrics=metrics) - config = ExporterConfig() - config.stats = stats.stats_dict - config.metrics = stats.metrics - config.args = args + assert isinstance(stats.metrics, Metrics) + config = create_default_exporter_config( + stats=stats.stats_dict, metrics=stats.metrics, args=args + ) exporter = ConsoleExporter(config) exporter.export() @@ -170,10 +170,10 @@ def test_embedding_output(self, monkeypatch, capsys) -> None: ) stats = Statistics(metrics=metrics) - config = ExporterConfig() - config.stats = stats.stats_dict - config.metrics = stats.metrics - config.args = args + assert isinstance(stats.metrics, Metrics) + config = create_default_exporter_config( + stats=stats.stats_dict, metrics=stats.metrics, args=args + ) exporter = ConsoleExporter(config) exporter.export() @@ -222,10 +222,10 @@ def test_valid_goodput(self, monkeypatch, capsys) -> None: ) stats = Statistics(metrics=metrics) - config = ExporterConfig() - config.stats = stats.stats_dict - config.metrics = stats.metrics - config.args = args + assert isinstance(stats.metrics, Metrics) + config = create_default_exporter_config( + stats=stats.stats_dict, metrics=stats.metrics, args=args + ) exporter = ConsoleExporter(config) exporter.export() @@ -281,10 +281,10 @@ def test_invalid_goodput_output(self, monkeypatch, capsys) -> None: stats = Statistics(metrics=metrics) - config = ExporterConfig() - config.stats = stats.stats_dict - config.metrics = stats.metrics - config.args = args + assert isinstance(stats.metrics, Metrics) + config = create_default_exporter_config( + stats=stats.stats_dict, metrics=stats.metrics, args=args + ) exporter = ConsoleExporter(config) exporter.export() @@ -345,10 +345,10 @@ def test_console_title( stats = Statistics(metrics=metrics) - config = ExporterConfig() - config.stats = stats.stats_dict - config.metrics = stats.metrics - config.args = args + assert isinstance(stats.metrics, Metrics) + config = create_default_exporter_config( + stats=stats.stats_dict, metrics=stats.metrics, args=args + ) exporter = ConsoleExporter(config) exporter.export() @@ -395,11 +395,13 @@ def test_valid_telemetry_verbose(self, monkeypatch, capsys) -> None: stats = Statistics(metrics=metrics) telemetry_stats = TelemetryStatistics(telemetry_metrics) - config = ExporterConfig() - config.stats = stats.stats_dict - config.telemetry_stats = telemetry_stats.stats_dict - config.metrics = stats.metrics - config.args = args + assert isinstance(stats.metrics, Metrics) + config = create_default_exporter_config( + stats=stats.stats_dict, + metrics=stats.metrics, + args=args, + telemetry_stats=telemetry_stats.stats_dict, + ) exporter = ConsoleExporter(config) exporter.export() @@ -501,10 +503,10 @@ def test_missing_data(self, monkeypatch, capsys) -> None: ) stats = Statistics(metrics=metrics) - config = ExporterConfig() - config.stats = stats.stats_dict - config.metrics = stats.metrics - config.args = args + assert isinstance(stats.metrics, Metrics) + config = create_default_exporter_config( + stats=stats.stats_dict, args=args, metrics=stats.metrics + ) # Missing data del config.stats["request_latency"]["avg"] diff --git a/genai-perf/tests/test_exporters/test_csv_exporter.py b/genai-perf/tests/test_exporters/test_csv_exporter.py index 88d3f1b7..c9b0dd3b 100644 --- a/genai-perf/tests/test_exporters/test_csv_exporter.py +++ b/genai-perf/tests/test_exporters/test_csv_exporter.py @@ -40,6 +40,7 @@ TelemetryMetrics, TelemetryStatistics, ) +from tests.test_utils import create_default_exporter_config class TestCsvExporter: @@ -104,11 +105,10 @@ def test_streaming_llm_csv_output( stats = Statistics(metrics=llm_metrics) - config = ExporterConfig() - config.stats = stats.stats_dict - config.metrics = stats.metrics - config.artifact_dir = Path(".") - config.args = args + assert isinstance(stats.metrics, Metrics) + config = create_default_exporter_config( + stats=stats.stats_dict, metrics=stats.metrics, args=args + ) exporter = CsvExporter(config) exporter.export() @@ -160,11 +160,10 @@ def test_nonstreaming_llm_csv_output( stats = Statistics(metrics=llm_metrics) - config = ExporterConfig() - config.stats = stats.stats_dict - config.metrics = stats.metrics - config.artifact_dir = Path(".") - config.args = args + assert isinstance(stats.metrics, Metrics) + config = create_default_exporter_config( + stats=stats.stats_dict, metrics=stats.metrics, args=args + ) exporter = CsvExporter(config) exporter.export() @@ -209,11 +208,10 @@ def test_embedding_csv_output( ) stats = Statistics(metrics=metrics) - config = ExporterConfig() - config.stats = stats.stats_dict - config.metrics = stats.metrics - config.artifact_dir = Path(".") - config.args = args + assert isinstance(stats.metrics, Metrics) + config = create_default_exporter_config( + stats=stats.stats_dict, metrics=stats.metrics, args=args + ) exporter = CsvExporter(config) exporter.export() @@ -261,11 +259,10 @@ def test_valid_goodput_csv_output( ) stats = Statistics(metrics=metrics) - config = ExporterConfig() - config.stats = stats.stats_dict - config.metrics = stats.metrics - config.artifact_dir = Path(".") - config.args = args + assert isinstance(stats.metrics, Metrics) + config = create_default_exporter_config( + stats=stats.stats_dict, metrics=stats.metrics, args=args + ) exporter = CsvExporter(config) exporter.export() @@ -312,11 +309,10 @@ def test_invalid_goodput_csv_output( ) stats = Statistics(metrics=metrics) - config = ExporterConfig() - config.stats = stats.stats_dict - config.metrics = stats.metrics - config.artifact_dir = Path(".") - config.args = args + assert isinstance(stats.metrics, Metrics) + config = create_default_exporter_config( + stats=stats.stats_dict, metrics=stats.metrics, args=args + ) exporter = CsvExporter(config) exporter.export() @@ -361,12 +357,13 @@ def test_triton_telemetry_output( stats = Statistics(metrics=llm_metrics) telemetry_stats = TelemetryStatistics(telemetry_metrics) - config = ExporterConfig() - config.stats = stats.stats_dict - config.telemetry_stats = telemetry_stats.stats_dict - config.metrics = stats.metrics - config.artifact_dir = Path(".") - config.args = args + assert isinstance(stats.metrics, Metrics) + config = create_default_exporter_config( + stats=stats.stats_dict, + metrics=stats.metrics, + args=args, + telemetry_stats=telemetry_stats.stats_dict, + ) exporter = CsvExporter(config) exporter.export() @@ -428,11 +425,10 @@ def test_missing_data( stats = Statistics(metrics=llm_metrics) - config = ExporterConfig() - config.stats = stats.stats_dict - config.metrics = stats.metrics - config.artifact_dir = Path(".") - config.args = args + assert isinstance(stats.metrics, Metrics) + config = create_default_exporter_config( + stats=stats.stats_dict, args=args, metrics=stats.metrics + ) # Missing data del config.stats["request_latency"]["avg"] diff --git a/genai-perf/tests/test_exporters/test_data_exporter_factory.py b/genai-perf/tests/test_exporters/test_data_exporter_factory.py index 18f6c639..e3c603e4 100644 --- a/genai-perf/tests/test_exporters/test_data_exporter_factory.py +++ b/genai-perf/tests/test_exporters/test_data_exporter_factory.py @@ -33,6 +33,7 @@ from genai_perf.export_data.exporter_config import ExporterConfig from genai_perf.export_data.json_exporter import JsonExporter from genai_perf.subcommand.common import get_extra_inputs_as_dict +from tests.test_utils import create_default_exporter_config class TestOutputReporter: @@ -64,11 +65,12 @@ class TestOutputReporter: } args_namespace = Namespace(**args) - config = ExporterConfig() - config.stats = stats - config.args = args_namespace - config.artifact_dir = args_namespace.artifact_dir - config.extra_inputs = get_extra_inputs_as_dict(args_namespace) + config = create_default_exporter_config( + stats=stats, + args=args_namespace, + artifact_dir=args_namespace.artifact_dir, + extra_inputs=get_extra_inputs_as_dict(args_namespace), + ) f = factory.DataExporterFactory() def test_return_json_exporter(self) -> None: diff --git a/genai-perf/tests/test_exporters/test_json_exporter.py b/genai-perf/tests/test_exporters/test_json_exporter.py index bc7e4835..02fe0b58 100644 --- a/genai-perf/tests/test_exporters/test_json_exporter.py +++ b/genai-perf/tests/test_exporters/test_json_exporter.py @@ -34,6 +34,7 @@ from genai_perf.export_data.exporter_config import ExporterConfig from genai_perf.export_data.json_exporter import JsonExporter from genai_perf.subcommand.common import get_extra_inputs_as_dict +from tests.test_utils import create_default_exporter_config class TestJsonExporter: @@ -304,11 +305,12 @@ def test_generate_json( ] monkeypatch.setattr("sys.argv", cli_cmd) args, _ = parser.parse_args() - config = ExporterConfig() - config.stats = self.stats - config.args = args - config.extra_inputs = get_extra_inputs_as_dict(args) - config.artifact_dir = args.artifact_dir + config = create_default_exporter_config( + stats=self.stats, + args=args, + extra_inputs=get_extra_inputs_as_dict(args), + artifact_dir=args.artifact_dir, + ) json_exporter = JsonExporter(config) assert json_exporter._stats_and_args == json.loads(self.expected_json_output) json_exporter.export() @@ -346,11 +348,12 @@ def test_generate_json_custom_export( ] monkeypatch.setattr("sys.argv", cli_cmd) args, _ = parser.parse_args() - config = ExporterConfig() - config.stats = self.stats - config.args = args - config.extra_inputs = get_extra_inputs_as_dict(args) - config.artifact_dir = args.artifact_dir + config = create_default_exporter_config( + stats=self.stats, + args=args, + extra_inputs=get_extra_inputs_as_dict(args), + artifact_dir=args.artifact_dir, + ) json_exporter = JsonExporter(config) json_exporter.export() written_data = [ @@ -491,11 +494,12 @@ def test_valid_goodput_json_output( ] monkeypatch.setattr("sys.argv", cli_cmd) args, _ = parser.parse_args() - config = ExporterConfig() - config.stats = valid_goodput_stats - config.args = args - config.extra_inputs = get_extra_inputs_as_dict(args) - config.artifact_dir = args.artifact_dir + config = create_default_exporter_config( + stats=valid_goodput_stats, + args=args, + extra_inputs=get_extra_inputs_as_dict(args), + artifact_dir=args.artifact_dir, + ) json_exporter = JsonExporter(config) assert json_exporter._stats_and_args["request_goodput"] == json.loads( expected_valid_goodput_json_output @@ -648,11 +652,12 @@ def test_invalid_goodput_json_output( ] monkeypatch.setattr("sys.argv", cli_cmd) args, _ = parser.parse_args() - config = ExporterConfig() - config.stats = invalid_goodput_stats - config.args = args - config.extra_inputs = get_extra_inputs_as_dict(args) - config.artifact_dir = args.artifact_dir + config = create_default_exporter_config( + stats=invalid_goodput_stats, + args=args, + extra_inputs=get_extra_inputs_as_dict(args), + artifact_dir=args.artifact_dir, + ) json_exporter = JsonExporter(config) assert json_exporter._stats_and_args["request_goodput"] == json.loads( expected_invalid_goodput_json_output @@ -833,12 +838,13 @@ def test_triton_telemetry_output( monkeypatch.setattr("sys.argv", cli_cmd) args, _ = parser.parse_args() - config = ExporterConfig() - config.stats = self.stats - config.telemetry_stats = telemetry_stats - config.args = args - config.extra_inputs = get_extra_inputs_as_dict(args) - config.artifact_dir = args.artifact_dir + config = create_default_exporter_config( + stats=self.stats, + telemetry_stats=telemetry_stats, + args=args, + extra_inputs=get_extra_inputs_as_dict(args), + artifact_dir=args.artifact_dir, + ) json_exporter = JsonExporter(config) assert json_exporter._stats_and_args["telemetry_stats"] == json.loads( expected_telemetry_json_output diff --git a/genai-perf/tests/test_utils.py b/genai-perf/tests/test_utils.py index 114177f1..e9847e6a 100644 --- a/genai-perf/tests/test_utils.py +++ b/genai-perf/tests/test_utils.py @@ -24,7 +24,9 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from typing import Optional, Union +from argparse import Namespace +from pathlib import Path +from typing import Any, Dict, Optional, Union import pytest from genai_perf import utils @@ -32,7 +34,9 @@ from genai_perf.config.generate.perf_analyzer_config import PerfAnalyzerConfig from genai_perf.config.input.config_command import ConfigCommand from genai_perf.config.run.run_config import RunConfig +from genai_perf.export_data.exporter_config import ExporterConfig from genai_perf.measurements.run_config_measurement import RunConfigMeasurement +from genai_perf.metrics.metrics import Metrics from genai_perf.metrics.statistics import Statistics from genai_perf.record.types.gpu_power_usage_p99 import GPUPowerUsageP99 from genai_perf.record.types.gpu_utilization_p99 import GPUUtilizationP99 @@ -60,6 +64,27 @@ def check_statistics(s1: Statistics, s2: Statistics) -> None: assert s2_dict[metric][stat_name] == pytest.approx(value) +########################################################################### +# ExporterConfig Constructor +########################################################################### +def create_default_exporter_config( + stats: Optional[Dict[Any, Any]] = None, + metrics: Optional[Metrics] = None, + args: Optional[Namespace] = None, + extra_inputs: Optional[Dict[str, Any]] = None, + artifact_dir: Optional[Path] = None, + telemetry_stats: Optional[Dict[str, Any]] = None, +) -> ExporterConfig: + return ExporterConfig( + stats=stats or {}, + metrics=metrics or Metrics(), + args=args or Namespace(), + extra_inputs=extra_inputs or {}, + artifact_dir=artifact_dir or Path("."), + telemetry_stats=telemetry_stats, + ) + + ########################################################################### # Perf Metrics Constructor ###########################################################################