From 4383bdc1b9781f9dca3d43870aac785eb188f325 Mon Sep 17 00:00:00 2001 From: Brian Raf Date: Sat, 4 May 2024 02:29:36 +0000 Subject: [PATCH] Moving unit testing into SearchParameters test class --- tests/test_config.py | 181 ------------------------------ tests/test_search_parameters.py | 190 ++++++++++++++++++++++++++++++++ 2 files changed, 190 insertions(+), 181 deletions(-) diff --git a/tests/test_config.py b/tests/test_config.py index c012e7cca..a8e8cb3b6 100755 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -32,12 +32,6 @@ from model_analyzer.config.input.config_command_report import ConfigCommandReport from model_analyzer.config.input.config_defaults import ( DEFAULT_LLM_INFERENCE_OUTPUT_FIELDS, - DEFAULT_RUN_CONFIG_MAX_CONCURRENCY, - DEFAULT_RUN_CONFIG_MAX_INSTANCE_COUNT, - DEFAULT_RUN_CONFIG_MAX_MODEL_BATCH_SIZE, - DEFAULT_RUN_CONFIG_MIN_CONCURRENCY, - DEFAULT_RUN_CONFIG_MIN_INSTANCE_COUNT, - DEFAULT_RUN_CONFIG_MIN_MODEL_BATCH_SIZE, ) from model_analyzer.config.input.config_enum import ConfigEnum from model_analyzer.config.input.config_list_generic import ConfigListGeneric @@ -2400,181 +2394,6 @@ def test_model_type_llm(self): config.inference_output_fields, DEFAULT_LLM_INFERENCE_OUTPUT_FIELDS ) - def test_search_parameter_creation_default(self): - """ - Test that search parameters are correctly created in default optuna case - """ - - args = [ - "model-analyzer", - "profile", - "--model-repository", - "cli-repository", - "-f", - "path-to-config-file", - "--run-config-search-mode", - "optuna", - ] - - yaml_content = """ - profile_models: add_sub - """ - - config = self._evaluate_config(args, yaml_content) - - analyzer = Analyzer(config, MagicMock(), MagicMock(), MagicMock()) - analyzer._populate_search_parameters() - - # batch_sizes - batch_sizes = analyzer._search_parameters["add_sub"].get_parameter( - "batch_sizes" - ) - self.assertEqual(ParameterUsage.MODEL, batch_sizes.usage) - self.assertEqual(ParameterCategory.EXPONENTIAL, batch_sizes.category) - self.assertEqual( - log2(DEFAULT_RUN_CONFIG_MIN_MODEL_BATCH_SIZE), batch_sizes.min_range - ) - self.assertEqual( - log2(DEFAULT_RUN_CONFIG_MAX_MODEL_BATCH_SIZE), batch_sizes.max_range - ) - - # concurrency - concurrency = analyzer._search_parameters["add_sub"].get_parameter( - "concurrency" - ) - self.assertEqual(ParameterUsage.RUNTIME, concurrency.usage) - self.assertEqual(ParameterCategory.EXPONENTIAL, concurrency.category) - self.assertEqual( - log2(DEFAULT_RUN_CONFIG_MIN_CONCURRENCY), concurrency.min_range - ) - self.assertEqual( - log2(DEFAULT_RUN_CONFIG_MAX_CONCURRENCY), concurrency.max_range - ) - - # instance_group - instance_group = analyzer._search_parameters["add_sub"].get_parameter( - "instance_group" - ) - self.assertEqual(ParameterUsage.MODEL, instance_group.usage) - self.assertEqual(ParameterCategory.INTEGER, instance_group.category) - self.assertEqual( - DEFAULT_RUN_CONFIG_MIN_INSTANCE_COUNT, instance_group.min_range - ) - self.assertEqual( - DEFAULT_RUN_CONFIG_MAX_INSTANCE_COUNT, instance_group.max_range - ) - - def test_search_parameter_creation_multi_model_non_default(self): - """ - Test that search parameters are correctly created in - a multi-model non-default optuna case - """ - - args = [ - "model-analyzer", - "profile", - "--model-repository", - "cli-repository", - "-f", - "path-to-config-file", - "--run-config-search-mode", - "optuna", - ] - - yaml_content = """ - run_config_search_mode: optuna - profile_models: - add_sub: - parameters: - batch_sizes: [16, 32, 64] - model_config_parameters: - dynamic_batching: - max_queue_delay_microseconds: [100, 200, 300] - instance_group: - - kind: KIND_GPU - count: [1, 2, 3, 4] - mult_div: - parameters: - concurrency: [1, 8, 64, 256] - """ - - config = self._evaluate_config(args, yaml_content) - - analyzer = Analyzer(config, MagicMock(), MagicMock(), MagicMock()) - analyzer._populate_search_parameters() - - # ADD_SUB - # batch_sizes - batch_sizes = analyzer._search_parameters["add_sub"].get_parameter( - "batch_sizes" - ) - self.assertEqual(ParameterUsage.MODEL, batch_sizes.usage) - self.assertEqual(ParameterCategory.LIST, batch_sizes.category) - self.assertEqual([16, 32, 64], batch_sizes.enumerated_list) - - # concurrency - concurrency = analyzer._search_parameters["add_sub"].get_parameter( - "concurrency" - ) - self.assertEqual(ParameterUsage.RUNTIME, concurrency.usage) - self.assertEqual(ParameterCategory.EXPONENTIAL, concurrency.category) - self.assertEqual( - log2(DEFAULT_RUN_CONFIG_MIN_CONCURRENCY), concurrency.min_range - ) - self.assertEqual( - log2(DEFAULT_RUN_CONFIG_MAX_CONCURRENCY), concurrency.max_range - ) - - # instance_group - instance_group = analyzer._search_parameters["add_sub"].get_parameter( - "instance_group" - ) - self.assertEqual(ParameterUsage.MODEL, instance_group.usage) - self.assertEqual(ParameterCategory.LIST, instance_group.category) - self.assertEqual([1, 2, 3, 4], instance_group.enumerated_list) - - instance_group = analyzer._search_parameters["add_sub"].get_parameter( - "max_queue_delay_microseconds" - ) - self.assertEqual(ParameterUsage.MODEL, instance_group.usage) - self.assertEqual(ParameterCategory.LIST, instance_group.category) - self.assertEqual([100, 200, 300], instance_group.enumerated_list) - - # MULT_DIV - # batch_sizes - batch_sizes = analyzer._search_parameters["mult_div"].get_parameter( - "batch_sizes" - ) - self.assertEqual(ParameterUsage.MODEL, batch_sizes.usage) - self.assertEqual(ParameterCategory.EXPONENTIAL, batch_sizes.category) - self.assertEqual( - log2(DEFAULT_RUN_CONFIG_MIN_MODEL_BATCH_SIZE), batch_sizes.min_range - ) - self.assertEqual( - log2(DEFAULT_RUN_CONFIG_MAX_MODEL_BATCH_SIZE), batch_sizes.max_range - ) - - # concurrency - concurrency = analyzer._search_parameters["mult_div"].get_parameter( - "concurrency" - ) - self.assertEqual(ParameterUsage.RUNTIME, concurrency.usage) - self.assertEqual(ParameterCategory.LIST, concurrency.category) - self.assertEqual([1, 8, 64, 256], concurrency.enumerated_list) - - # instance_group - instance_group = analyzer._search_parameters["mult_div"].get_parameter( - "instance_group" - ) - self.assertEqual(ParameterUsage.MODEL, instance_group.usage) - self.assertEqual(ParameterCategory.INTEGER, instance_group.category) - self.assertEqual( - DEFAULT_RUN_CONFIG_MIN_INSTANCE_COUNT, instance_group.min_range - ) - self.assertEqual( - DEFAULT_RUN_CONFIG_MAX_INSTANCE_COUNT, instance_group.max_range - ) - def _test_request_rate_config_conflicts( self, base_args: List[Any], yaml_content: str ) -> None: diff --git a/tests/test_search_parameters.py b/tests/test_search_parameters.py index ff5012f40..64c154912 100755 --- a/tests/test_search_parameters.py +++ b/tests/test_search_parameters.py @@ -15,8 +15,11 @@ # limitations under the License. import unittest +from math import log2 from unittest.mock import MagicMock, patch +import model_analyzer.config.input.config_defaults as default +from model_analyzer.analyzer import Analyzer from model_analyzer.config.generate.search_parameters import ( ParameterCategory, ParameterUsage, @@ -181,6 +184,193 @@ def test_illegal_inputs(self): max_range=10, ) + def test_search_parameter_creation_default(self): + """ + Test that search parameters are correctly created in default optuna case + """ + + args = [ + "model-analyzer", + "profile", + "--model-repository", + "cli-repository", + "-f", + "path-to-config-file", + "--run-config-search-mode", + "optuna", + ] + + yaml_content = """ + profile_models: add_sub + """ + + config = TestConfig()._evaluate_config(args=args, yaml_content=yaml_content) + + analyzer = Analyzer(config, MagicMock(), MagicMock(), MagicMock()) + analyzer._populate_search_parameters() + + # batch_sizes + batch_sizes = analyzer._search_parameters["add_sub"].get_parameter( + "batch_sizes" + ) + self.assertEqual(ParameterUsage.MODEL, batch_sizes.usage) + self.assertEqual(ParameterCategory.EXPONENTIAL, batch_sizes.category) + self.assertEqual( + log2(default.DEFAULT_RUN_CONFIG_MIN_MODEL_BATCH_SIZE), batch_sizes.min_range + ) + self.assertEqual( + log2(default.DEFAULT_RUN_CONFIG_MAX_MODEL_BATCH_SIZE), batch_sizes.max_range + ) + + # concurrency + concurrency = analyzer._search_parameters["add_sub"].get_parameter( + "concurrency" + ) + self.assertEqual(ParameterUsage.RUNTIME, concurrency.usage) + self.assertEqual(ParameterCategory.EXPONENTIAL, concurrency.category) + self.assertEqual( + log2(default.DEFAULT_RUN_CONFIG_MIN_CONCURRENCY), concurrency.min_range + ) + self.assertEqual( + log2(default.DEFAULT_RUN_CONFIG_MAX_CONCURRENCY), concurrency.max_range + ) + + # instance_group + instance_group = analyzer._search_parameters["add_sub"].get_parameter( + "instance_group" + ) + self.assertEqual(ParameterUsage.MODEL, instance_group.usage) + self.assertEqual(ParameterCategory.INTEGER, instance_group.category) + self.assertEqual( + default.DEFAULT_RUN_CONFIG_MIN_INSTANCE_COUNT, instance_group.min_range + ) + self.assertEqual( + default.DEFAULT_RUN_CONFIG_MAX_INSTANCE_COUNT, instance_group.max_range + ) + + def test_search_parameter_creation_multi_model_non_default(self): + """ + Test that search parameters are correctly created in + a multi-model non-default optuna case + """ + + args = [ + "model-analyzer", + "profile", + "--model-repository", + "cli-repository", + "-f", + "path-to-config-file", + "--run-config-search-mode", + "optuna", + ] + + yaml_content = """ + run_config_search_mode: optuna + profile_models: + add_sub: + parameters: + batch_sizes: [16, 32, 64] + model_config_parameters: + dynamic_batching: + max_queue_delay_microseconds: [100, 200, 300] + instance_group: + - kind: KIND_GPU + count: [1, 2, 3, 4] + mult_div: + parameters: + concurrency: [1, 8, 64, 256] + """ + + config = TestConfig()._evaluate_config(args, yaml_content) + + analyzer = Analyzer(config, MagicMock(), MagicMock(), MagicMock()) + analyzer._populate_search_parameters() + + # =================================================================== + # ADD_SUB + # =================================================================== + + # batch_sizes + # =================================================================== + batch_sizes = analyzer._search_parameters["add_sub"].get_parameter( + "batch_sizes" + ) + self.assertEqual(ParameterUsage.MODEL, batch_sizes.usage) + self.assertEqual(ParameterCategory.LIST, batch_sizes.category) + self.assertEqual([16, 32, 64], batch_sizes.enumerated_list) + + # concurrency + # =================================================================== + concurrency = analyzer._search_parameters["add_sub"].get_parameter( + "concurrency" + ) + self.assertEqual(ParameterUsage.RUNTIME, concurrency.usage) + self.assertEqual(ParameterCategory.EXPONENTIAL, concurrency.category) + self.assertEqual( + log2(default.DEFAULT_RUN_CONFIG_MIN_CONCURRENCY), concurrency.min_range + ) + self.assertEqual( + log2(default.DEFAULT_RUN_CONFIG_MAX_CONCURRENCY), concurrency.max_range + ) + + # instance_group + # =================================================================== + instance_group = analyzer._search_parameters["add_sub"].get_parameter( + "instance_group" + ) + self.assertEqual(ParameterUsage.MODEL, instance_group.usage) + self.assertEqual(ParameterCategory.LIST, instance_group.category) + self.assertEqual([1, 2, 3, 4], instance_group.enumerated_list) + + instance_group = analyzer._search_parameters["add_sub"].get_parameter( + "max_queue_delay_microseconds" + ) + self.assertEqual(ParameterUsage.MODEL, instance_group.usage) + self.assertEqual(ParameterCategory.LIST, instance_group.category) + self.assertEqual([100, 200, 300], instance_group.enumerated_list) + + # =================================================================== + # MULT_DIV + # =================================================================== + + # batch_sizes + # =================================================================== + batch_sizes = analyzer._search_parameters["mult_div"].get_parameter( + "batch_sizes" + ) + self.assertEqual(ParameterUsage.MODEL, batch_sizes.usage) + self.assertEqual(ParameterCategory.EXPONENTIAL, batch_sizes.category) + self.assertEqual( + log2(default.DEFAULT_RUN_CONFIG_MIN_MODEL_BATCH_SIZE), batch_sizes.min_range + ) + self.assertEqual( + log2(default.DEFAULT_RUN_CONFIG_MAX_MODEL_BATCH_SIZE), batch_sizes.max_range + ) + + # concurrency + # =================================================================== + concurrency = analyzer._search_parameters["mult_div"].get_parameter( + "concurrency" + ) + self.assertEqual(ParameterUsage.RUNTIME, concurrency.usage) + self.assertEqual(ParameterCategory.LIST, concurrency.category) + self.assertEqual([1, 8, 64, 256], concurrency.enumerated_list) + + # instance_group + # =================================================================== + instance_group = analyzer._search_parameters["mult_div"].get_parameter( + "instance_group" + ) + self.assertEqual(ParameterUsage.MODEL, instance_group.usage) + self.assertEqual(ParameterCategory.INTEGER, instance_group.category) + self.assertEqual( + default.DEFAULT_RUN_CONFIG_MIN_INSTANCE_COUNT, instance_group.min_range + ) + self.assertEqual( + default.DEFAULT_RUN_CONFIG_MAX_INSTANCE_COUNT, instance_group.max_range + ) + if __name__ == "__main__": unittest.main()