From 03d45eac9b238efbd35a91c664ab2c867d0912ee Mon Sep 17 00:00:00 2001 From: Hyunjae Woo Date: Thu, 31 Aug 2023 14:54:34 -0700 Subject: [PATCH 01/24] Add macros to reuse it for checking range options --- .../perf_analyzer/test_command_line_parser.cc | 251 +++++++++--------- 1 file changed, 119 insertions(+), 132 deletions(-) diff --git a/src/c++/perf_analyzer/test_command_line_parser.cc b/src/c++/perf_analyzer/test_command_line_parser.cc index 86a5a9175..d000c7145 100644 --- a/src/c++/perf_analyzer/test_command_line_parser.cc +++ b/src/c++/perf_analyzer/test_command_line_parser.cc @@ -47,6 +47,105 @@ CHECK_STRING(std::string act, std::string exp) !act.compare(exp), "Expecting: '", exp, "', Found: '", act, "'"); } +#define CHECK_RANGE_PASS(option_name, using_range, start, end, step) \ + SUBCASE("only start provided") \ + { \ + int argc = 5; \ + char* argv[argc] = {app_name, "-m", model_name, option_name, "100"}; \ + \ + REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); \ + CHECK(!parser.UsageCalled()); \ + \ + using_range = true; \ + start = 100; \ + } \ + \ + SUBCASE("start and end provided") \ + { \ + int argc = 5; \ + char* argv[argc] = {app_name, "-m", model_name, option_name, "100:400"}; \ + \ + REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); \ + CHECK(!parser.UsageCalled()); \ + \ + using_range = true; \ + start = 100; \ + end = 400; \ + } \ + \ + SUBCASE("start, end, and step provided") \ + { \ + int argc = 5; \ + char* argv[argc] = { \ + app_name, "-m", model_name, option_name, "100:400:10"}; \ + \ + REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); \ + CHECK(!parser.UsageCalled()); \ + \ + using_range = true; \ + start = 100; \ + end = 400; \ + step = 10; \ + } + +#define CHECK_RANGE_FAIL(option_name, msg) \ + SUBCASE("too many input values") \ + { \ + int argc = 5; \ + char* argv[argc] = { \ + app_name, "-m", model_name, option_name, "200:100:25:10"}; \ + \ + REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); \ + CHECK(parser.UsageCalled()); \ + \ + check_params = false; \ + } + +#define CHECK_RANGE_START_VALUE(option_name, msg) \ + SUBCASE("invalid start value") \ + { \ + int argc = 5; \ + char* argv[argc] = { \ + app_name, "-m", model_name, option_name, "bad:400:10"}; \ + \ + REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); \ + CHECK(parser.UsageCalled()); \ + \ + CHECK_STRING("Usage Message", parser.GetUsageMessage(), expected_msg); \ + \ + check_params = false; \ + } + +#define CHECK_RANGE_END_VALUE(option_name, msg) \ + SUBCASE("invalid end value") \ + { \ + int argc = 5; \ + char* argv[argc] = { \ + app_name, "-m", model_name, option_name, "100:bad:10"}; \ + \ + REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); \ + CHECK(parser.UsageCalled()); \ + \ + CHECK_STRING("Usage Message", parser.GetUsageMessage(), expected_msg); \ + \ + check_params = false; \ + } + +#define CHECK_RANGE_STEP_VALUE(option_name, msg) \ + SUBCASE("invalid step value") \ + { \ + int argc = 5; \ + char* argv[argc] = { \ + app_name, "-m", model_name, option_name, "100:400:bad"}; \ + \ + REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); \ + CHECK(parser.UsageCalled()); \ + \ + CHECK_STRING("Usage Message", parser.GetUsageMessage(), expected_msg); \ + \ + check_params = false; \ + } + std::string CreateUsageMessage(const std::string& option_name, const std::string& msg) { @@ -967,49 +1066,26 @@ TEST_CASE("Testing Command Line Parser") SUBCASE("Option : --concurrency-range") { - SUBCASE("expected use") - { - int argc = 5; - char* argv[argc] = { - app_name, "-m", model_name, "--concurrency-range", "100:400:10"}; + CHECK_RANGE_PASS( + "--concurrency-range", exp->using_concurrency_range, + exp->concurrency_range.start, exp->concurrency_range.end, + exp->concurrency_range.step); - REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); - CHECK(!parser.UsageCalled()); - - exp->using_concurrency_range = true; - exp->concurrency_range.start = 100; - exp->concurrency_range.end = 400; - exp->concurrency_range.step = 10; - } - - SUBCASE("only two options") - { - int argc = 5; - char* argv[argc] = { - app_name, "-m", model_name, "--concurrency-range", "100:400"}; - - REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); - CHECK(!parser.UsageCalled()); - - exp->using_concurrency_range = true; - exp->concurrency_range.start = 100; - exp->concurrency_range.end = 400; - } + expected_msg = CreateUsageMessage( + "--concurrency-range", "The value does not match ."); + CHECK_RANGE_FAIL("--concurrency-range", expected_msg); - SUBCASE("only one options") - { - int argc = 5; - char* argv[argc] = { - app_name, "-m", model_name, "--concurrency-range", "100"}; + expected_msg = CreateUsageMessage( + "--concurrency-range", "Invalid value provided: bad:400:10"); + CHECK_RANGE_START_VALUE("--concurrency-range", expected_msg); - // QUESTION: What does this mean? Why pass only one? - // - REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); - CHECK(!parser.UsageCalled()); + expected_msg = CreateUsageMessage( + "--concurrency-range", "Invalid value provided: 100:bad:10"); + CHECK_RANGE_END_VALUE("--concurrency-range", expected_msg); - exp->using_concurrency_range = true; - exp->concurrency_range.start = 100; - } + expected_msg = CreateUsageMessage( + "--concurrency-range", "Invalid value provided: 100:400:bad"); + CHECK_RANGE_STEP_VALUE("--concurrency-range", expected_msg); SUBCASE("no options") { @@ -1021,52 +1097,13 @@ TEST_CASE("Testing Command Line Parser") REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); CHECK(parser.UsageCalled()); - // BUG: Usage message does not contain error. Error statement + // BUG (TMA-1307): Usage message does not contain error. Error statement // "option '--concurrency-range' requires an argument" written directly // to std::out // CHECK_STRING("Usage Message", parser.GetUsageMessage(), ""); } - SUBCASE("too many options") - { - int argc = 5; - char* argv[argc] = { - app_name, "-m", model_name, "--concurrency-range", "200:100:25:10"}; - - REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); - CHECK(parser.UsageCalled()); - - expected_msg = CreateUsageMessage( - "--concurrency-range", "The value does not match ."); - CHECK_STRING("Usage Message", parser.GetUsageMessage(), expected_msg); - - exp->using_concurrency_range = true; - exp->concurrency_range.start = 200; - exp->concurrency_range.end = 100; - exp->concurrency_range.step = 25; - } - - SUBCASE("way too many options") - { - int argc = 5; - char* argv[argc] = { - app_name, "-m", model_name, "--concurrency-range", - "200:100:25:10:20:30"}; - - REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); - CHECK(parser.UsageCalled()); - - expected_msg = CreateUsageMessage( - "--concurrency-range", "The value does not match ."); - CHECK_STRING("Usage Message", parser.GetUsageMessage(), expected_msg); - - exp->using_concurrency_range = true; - exp->concurrency_range.start = 200; - exp->concurrency_range.end = 100; - exp->concurrency_range.step = 25; - } - SUBCASE("wrong separator") { int argc = 5; @@ -1076,65 +1113,15 @@ TEST_CASE("Testing Command Line Parser") REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); CHECK(!parser.UsageCalled()); - // BUG: Should detect this and through an error. User will enter this and - // have no clue why the end and step sizes are not used correctly. + // BUG (TMA-1307): Should detect this and through an error. User will + // enter this and have no clue why the end and step sizes are not used + // correctly. // exp->using_concurrency_range = true; exp->concurrency_range.start = 100; } - SUBCASE("bad start value") - { - int argc = 5; - char* argv[argc] = { - app_name, "-m", model_name, "--concurrency-range", "bad:400:10"}; - - REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); - CHECK(parser.UsageCalled()); - - expected_msg = CreateUsageMessage( - "--concurrency-range", "Invalid value provided: bad:400:10"); - CHECK_STRING("Usage Message", parser.GetUsageMessage(), expected_msg); - - exp->using_concurrency_range = true; - } - - SUBCASE("bad end value") - { - int argc = 5; - char* argv[argc] = { - app_name, "-m", model_name, "--concurrency-range", "100:bad:10"}; - - REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); - CHECK(parser.UsageCalled()); - - expected_msg = CreateUsageMessage( - "--concurrency-range", "Invalid value provided: 100:bad:10"); - CHECK_STRING("Usage Message", parser.GetUsageMessage(), expected_msg); - - exp->using_concurrency_range = true; - exp->concurrency_range.start = 100; - } - - SUBCASE("bad step value") - { - int argc = 5; - char* argv[argc] = { - app_name, "-m", model_name, "--concurrency-range", "100:400:bad"}; - - REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); - CHECK(parser.UsageCalled()); - - expected_msg = CreateUsageMessage( - "--concurrency-range", "Invalid value provided: 100:400:bad"); - CHECK_STRING("Usage Message", parser.GetUsageMessage(), expected_msg); - - exp->using_concurrency_range = true; - exp->concurrency_range.start = 100; - exp->concurrency_range.end = 400; - } - SUBCASE("invalid condition - end and latency threshold are 0") { int argc = 7; From 842470dd6a4d3423641bf78b2233d7ad138b21bc Mon Sep 17 00:00:00 2001 From: Hyunjae Woo Date: Thu, 31 Aug 2023 15:08:50 -0700 Subject: [PATCH 02/24] Add tests for periodic-concurrency-range option --- .../perf_analyzer/test_command_line_parser.cc | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/c++/perf_analyzer/test_command_line_parser.cc b/src/c++/perf_analyzer/test_command_line_parser.cc index d000c7145..aa0b40fe7 100644 --- a/src/c++/perf_analyzer/test_command_line_parser.cc +++ b/src/c++/perf_analyzer/test_command_line_parser.cc @@ -1145,6 +1145,32 @@ TEST_CASE("Testing Command Line Parser") } } + SUBCASE("Option: --periodic-concurrency-range") + { + CHECK_RANGE_PASS( + "--periodic-concurrency-range", exp->using_periodic_concurrency_range, + exp->periodic_concurrency_range.start, + exp->periodic_concurrency_range.end, + exp->periodic_concurrency_range.step); + + expected_msg = CreateUsageMessage( + "--periodic-concurrency-range", + "The value does not match ."); + CHECK_RANGE_FAIL("--periodic-concurrency-range", expected_msg); + + expected_msg = CreateUsageMessage( + "--periodic-concurrency-range", "Invalid value provided: bad:400:10"); + CHECK_RANGE_START_VALUE("--periodic-concurrency-range", expected_msg); + + expected_msg = CreateUsageMessage( + "--periodic-concurrency-range", "Invalid value provided: 100:bad:10"); + CHECK_RANGE_END_VALUE("--periodic-concurrency-range", expected_msg); + + expected_msg = CreateUsageMessage( + "--periodic-concurrency-range", "Invalid value provided: 100:400:bad"); + CHECK_RANGE_STEP_VALUE("--periodic-concurrency-range", expected_msg); + } + SUBCASE("Option : --latency-threshold") { expected_msg = CreateUsageMessage( From 8e07edf553abc592aeb33950760d7e96e5603636 Mon Sep 17 00:00:00 2001 From: Hyunjae Woo Date: Thu, 31 Aug 2023 15:13:47 -0700 Subject: [PATCH 03/24] Add periodic-concurrency-range and request-period options --- src/c++/perf_analyzer/command_line_parser.cc | 79 ++++++++++++++++++++ src/c++/perf_analyzer/command_line_parser.h | 3 + 2 files changed, 82 insertions(+) diff --git a/src/c++/perf_analyzer/command_line_parser.cc b/src/c++/perf_analyzer/command_line_parser.cc index 7398455cb..b3d08fad2 100644 --- a/src/c++/perf_analyzer/command_line_parser.cc +++ b/src/c++/perf_analyzer/command_line_parser.cc @@ -88,6 +88,8 @@ CLParser::Usage(const std::string& msg) std::cerr << "\t--measurement-interval (-p) " << std::endl; std::cerr << "\t--concurrency-range " << std::endl; + std::cerr << "\t--periodic-concurrency-range " << std::endl; + std::cerr << "\t--request-period " << std::endl; std::cerr << "\t--request-rate-range " << std::endl; std::cerr << "\t--request-distribution <\"poisson\"|\"constant\">" << std::endl; @@ -274,6 +276,31 @@ CLParser::Usage(const std::string& msg) "not be 0 for sequence models while using asynchronous mode.", 18) << std::endl; + std::cerr + << FormatMessage( + "--periodic-concurrency-range : Determines the " + "range of concurrency levels in the similar manner as the " + "--concurrency-range. The perf_analyzer will start from the " + "concurrency level of 'start' and go until it reaches 'end' with " + "a stride of 'step'. Unlike --concurrency-range, the user can " + "specify *when* to periodically increase the concurrency level " + "using the --request-period option. The concurrency level will " + "periodically increase for every n-th response specified by " + "--request-period. Since this disables stability check in " + "perf_analyzer and reports response timestamps only, the user " + "must provide --profile-export-file to specify where to dump all " + "the measured timestamps. The default values of 'start', 'end', " + "and 'step' are 1.", + 18) + << std::endl; + std::cerr + << FormatMessage( + "--request-period: Indicates the number of responses that each " + "request will wait until it launches a new, concurrent " + "request when --periodic-concurrency-range is specified. " + "Default value is 10.", + 18) + << std::endl; std::cerr << FormatMessage( " --request-rate-range : Determines the range of " @@ -806,6 +833,8 @@ CLParser::ParseCommandLine(int argc, char** argv) {"output-tensor-format", required_argument, 0, 56}, {"version", no_argument, 0, 57}, {"profile-export-file", required_argument, 0, 58}, + {"periodic-concurrency-range", required_argument, 0, 59}, + {"request-period", required_argument, 0, 60}, {0, 0, 0, 0}}; // Parse commandline... @@ -1482,6 +1511,51 @@ CLParser::ParseCommandLine(int argc, char** argv) params_->profile_export_file = profile_export_file; break; } + case 59: { + params_->using_periodic_concurrency_range = true; + std::string arg = optarg; + size_t pos = 0; + int index = 0; + while (pos != std::string::npos) { + size_t colon_pos = arg.find(":", pos); + if (index > 2) { + Usage( + "Failed to parse --periodic-concurrency-range. The value " + "does not match ."); + } + int64_t val; + if (colon_pos == std::string::npos) { + val = std::stoull(arg.substr(pos, colon_pos)); + pos = colon_pos; + } else { + val = std::stoull(arg.substr(pos, colon_pos - pos)); + pos = colon_pos + 1; + } + switch (index) { + case 0: + params_->periodic_concurrency_range.start = val; + break; + case 1: + params_->periodic_concurrency_range.end = val; + break; + case 2: + params_->periodic_concurrency_range.step = val; + break; + } + index++; + } + + break; + } + case 60: { + std::string request_period{optarg}; + if (std::stoi(request_period) > 0) { + params_->request_period = std::stoull(request_period); + } else { + Usage("Failed to parse --request-period. The value must be > 0"); + } + break; + } case 'v': params_->extra_verbose = params_->verbose; params_->verbose = true; @@ -1645,6 +1719,11 @@ CLParser::VerifyOptions() "simultaneously."); } + // TODO: + // - check if two or more mode is specified + // - check if --profile-export-file is specified + // - check if (end - start) % step == 0 + if (params_->using_request_rate_range && params_->mpi_driver->IsMPIRun() && (params_->request_rate_range[SEARCH_RANGE::kEND] != 1.0 || params_->request_rate_range[SEARCH_RANGE::kSTEP] != 1.0)) { diff --git a/src/c++/perf_analyzer/command_line_parser.h b/src/c++/perf_analyzer/command_line_parser.h index a0706525c..e92f638ab 100644 --- a/src/c++/perf_analyzer/command_line_parser.h +++ b/src/c++/perf_analyzer/command_line_parser.h @@ -58,6 +58,9 @@ struct PerfAnalyzerParameters { uint64_t measurement_window_ms = 5000; bool using_concurrency_range = false; Range concurrency_range{1, 1, 1}; + bool using_periodic_concurrency_range = false; + Range periodic_concurrency_range{1, 1, 1}; + uint64_t request_period = 10; uint64_t latency_threshold_ms = NO_LIMIT; double stability_threshold = 0.1; size_t max_trials = 10; From 4006100c8faa27b5e73e67277c5b627a17af26a3 Mon Sep 17 00:00:00 2001 From: Hyunjae Woo Date: Thu, 31 Aug 2023 15:15:01 -0700 Subject: [PATCH 04/24] Add doc for periodic-concurrency-range and request-period --- src/c++/perf_analyzer/docs/cli.md | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/c++/perf_analyzer/docs/cli.md b/src/c++/perf_analyzer/docs/cli.md index 3fae93692..602a453a0 100644 --- a/src/c++/perf_analyzer/docs/cli.md +++ b/src/c++/perf_analyzer/docs/cli.md @@ -173,13 +173,35 @@ Specifies the range of concurrency levels covered by Perf Analyzer. Perf Analyzer will start from the concurrency level of 'start' and go until 'end' with a stride of 'step'. -Default of 'end' and 'step' are `1`. If 'end' is not specified then Perf -Analyzer will run for a single concurrency level determined by 'start'. If +Default of 'start', 'end', and 'step' are `1`. If 'end' is not specified then +Perf Analyzer will run for a single concurrency level determined by 'start'. If 'end' is set as `0`, then the concurrency limit will be incremented by 'step' until the latency threshold is met. 'end' and `--latency-threshold` cannot both be `0`. 'end' cannot be `0` for sequence models while using asynchronous mode. +#### `--periodic-concurrency-range=` + +Determines the range of concurrency levels in the similar manner as the +`--concurrency-range`. The perf_analyzer will start from the concurrency level +of 'start' and go until it reaches 'end' with a stride of 'step'. +Unlike `--concurrency-range`, the user can specify *when* to periodically +increase the concurrency level using the `--request-period` option. +The concurrency level will periodically increase for every `n`-th response +specified by `--request-period`. +Since this disables stability check in perf_analyzer and reports response +timestamps only, the user must provide `--profile-export-file` to specify where +to dump all the measured response timestamps. + +The default values of 'start', 'end', and 'step' are `1`. + +#### `--request-period ` + +Indicates the number of responses that each request will wait until it launches +a new, concurrent request when `--periodic-concurrency-range` is specified. + +Default value is `10`. + #### `--request-rate-range=` Specifies the range of request rates for load generated by Perf Analyzer. This From 65593325a140ac0acca903754f3722286d421e71 Mon Sep 17 00:00:00 2001 From: Hyunjae Woo Date: Thu, 31 Aug 2023 15:40:28 -0700 Subject: [PATCH 05/24] Add test for request-period option --- .../perf_analyzer/test_command_line_parser.cc | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/c++/perf_analyzer/test_command_line_parser.cc b/src/c++/perf_analyzer/test_command_line_parser.cc index aa0b40fe7..c3facc8f4 100644 --- a/src/c++/perf_analyzer/test_command_line_parser.cc +++ b/src/c++/perf_analyzer/test_command_line_parser.cc @@ -1171,6 +1171,24 @@ TEST_CASE("Testing Command Line Parser") CHECK_RANGE_STEP_VALUE("--periodic-concurrency-range", expected_msg); } + SUBCASE("Option : --request-period") + { + expected_msg = + CreateUsageMessage("--request-period", "The value must be > 0"); + CHECK_INT_OPTION("--request-period", exp->request_period, expected_msg); + + SUBCASE("set to 0") + { + int argc = 5; + char* argv[argc] = {app_name, "-m", model_name, "--request-period", "0"}; + + REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); + CHECK(parser.UsageCalled()); + + CHECK_STRING("Usage Message", parser.GetUsageMessage(), expected_msg); + } + } + SUBCASE("Option : --latency-threshold") { expected_msg = CreateUsageMessage( From 778385357d6fa3c5920112233e3cbdbe811d3547 Mon Sep 17 00:00:00 2001 From: Hyunjae Woo Date: Thu, 31 Aug 2023 16:56:33 -0700 Subject: [PATCH 06/24] Revert macro and add reusable test function --- .../perf_analyzer/test_command_line_parser.cc | 337 +++++++++--------- 1 file changed, 163 insertions(+), 174 deletions(-) diff --git a/src/c++/perf_analyzer/test_command_line_parser.cc b/src/c++/perf_analyzer/test_command_line_parser.cc index c3facc8f4..3ba99f1b7 100644 --- a/src/c++/perf_analyzer/test_command_line_parser.cc +++ b/src/c++/perf_analyzer/test_command_line_parser.cc @@ -47,105 +47,6 @@ CHECK_STRING(std::string act, std::string exp) !act.compare(exp), "Expecting: '", exp, "', Found: '", act, "'"); } -#define CHECK_RANGE_PASS(option_name, using_range, start, end, step) \ - SUBCASE("only start provided") \ - { \ - int argc = 5; \ - char* argv[argc] = {app_name, "-m", model_name, option_name, "100"}; \ - \ - REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); \ - CHECK(!parser.UsageCalled()); \ - \ - using_range = true; \ - start = 100; \ - } \ - \ - SUBCASE("start and end provided") \ - { \ - int argc = 5; \ - char* argv[argc] = {app_name, "-m", model_name, option_name, "100:400"}; \ - \ - REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); \ - CHECK(!parser.UsageCalled()); \ - \ - using_range = true; \ - start = 100; \ - end = 400; \ - } \ - \ - SUBCASE("start, end, and step provided") \ - { \ - int argc = 5; \ - char* argv[argc] = { \ - app_name, "-m", model_name, option_name, "100:400:10"}; \ - \ - REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); \ - CHECK(!parser.UsageCalled()); \ - \ - using_range = true; \ - start = 100; \ - end = 400; \ - step = 10; \ - } - -#define CHECK_RANGE_FAIL(option_name, msg) \ - SUBCASE("too many input values") \ - { \ - int argc = 5; \ - char* argv[argc] = { \ - app_name, "-m", model_name, option_name, "200:100:25:10"}; \ - \ - REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); \ - CHECK(parser.UsageCalled()); \ - \ - check_params = false; \ - } - -#define CHECK_RANGE_START_VALUE(option_name, msg) \ - SUBCASE("invalid start value") \ - { \ - int argc = 5; \ - char* argv[argc] = { \ - app_name, "-m", model_name, option_name, "bad:400:10"}; \ - \ - REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); \ - CHECK(parser.UsageCalled()); \ - \ - CHECK_STRING("Usage Message", parser.GetUsageMessage(), expected_msg); \ - \ - check_params = false; \ - } - -#define CHECK_RANGE_END_VALUE(option_name, msg) \ - SUBCASE("invalid end value") \ - { \ - int argc = 5; \ - char* argv[argc] = { \ - app_name, "-m", model_name, option_name, "100:bad:10"}; \ - \ - REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); \ - CHECK(parser.UsageCalled()); \ - \ - CHECK_STRING("Usage Message", parser.GetUsageMessage(), expected_msg); \ - \ - check_params = false; \ - } - -#define CHECK_RANGE_STEP_VALUE(option_name, msg) \ - SUBCASE("invalid step value") \ - { \ - int argc = 5; \ - char* argv[argc] = { \ - app_name, "-m", model_name, option_name, "100:400:bad"}; \ - \ - REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); \ - CHECK(parser.UsageCalled()); \ - \ - CHECK_STRING("Usage Message", parser.GetUsageMessage(), expected_msg); \ - \ - check_params = false; \ - } - std::string CreateUsageMessage(const std::string& option_name, const std::string& msg) { @@ -441,6 +342,158 @@ class TestCLParser : public CLParser { } }; +void +CheckValidRange( + char* app_name, char* model_name, char* option_name, TestCLParser& parser, + PAParamsPtr& act, bool& using_range, Range& range) +{ + SUBCASE("only start provided") + { + int argc = 5; + char* argv[argc] = {app_name, "-m", model_name, option_name, "100"}; + + REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); + CHECK(!parser.UsageCalled()); + + using_range = true; + range.start = 100; + } + + SUBCASE("start and end provided") + { + int argc = 5; + char* argv[argc] = {app_name, "-m", model_name, option_name, "100:400"}; + + REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); + CHECK(!parser.UsageCalled()); + + using_range = true; + range.start = 100; + range.end = 400; + } + + SUBCASE("start, end, and step provided") + { + int argc = 5; + char* argv[argc] = {app_name, "-m", model_name, option_name, "100:400:10"}; + + REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); + CHECK(!parser.UsageCalled()); + + using_range = true; + range.start = 100; + range.end = 400; + range.step = 10; + } +} + +void +CheckInvalidRange( + char* app_name, char* model_name, char* option_name, TestCLParser& parser, + PAParamsPtr& act, bool& check_params) +{ + std::string expected_msg; + + SUBCASE("too many input values") + { + int argc = 5; + char* argv[argc] = { + app_name, "-m", model_name, option_name, "200:100:25:10"}; + + REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); + CHECK(parser.UsageCalled()); + + expected_msg = CreateUsageMessage( + option_name, "The value does not match ."); + CHECK_STRING("Usage Message", parser.GetUsageMessage(), expected_msg); + + check_params = false; + } + + SUBCASE("invalid start value") + { + int argc = 5; + char* argv[argc] = {app_name, "-m", model_name, option_name, "bad:400:10"}; + + REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); + CHECK(parser.UsageCalled()); + + expected_msg = + CreateUsageMessage(option_name, "Invalid value provided: bad:400:10"); + CHECK_STRING("Usage Message", parser.GetUsageMessage(), expected_msg); + + check_params = false; + } + + SUBCASE("invalid end value") + { + int argc = 5; + char* argv[argc] = {app_name, "-m", model_name, option_name, "100:bad:10"}; + + REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); + CHECK(parser.UsageCalled()); + + expected_msg = + CreateUsageMessage(option_name, "Invalid value provided: 100:bad:10"); + CHECK_STRING("Usage Message", parser.GetUsageMessage(), expected_msg); + + check_params = false; + } + + SUBCASE("invalid step value") + { + int argc = 5; + char* argv[argc] = {app_name, "-m", model_name, option_name, "100:400:bad"}; + + REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); + CHECK(parser.UsageCalled()); + + expected_msg = + CreateUsageMessage(option_name, "Invalid value provided: 100:400:bad"); + CHECK_STRING("Usage Message", parser.GetUsageMessage(), expected_msg); + + check_params = false; + } + + SUBCASE("no options") + { + int argc = 4; + char* argv[argc] = {app_name, "-m", model_name, option_name}; + + opterr = 0; // Disable error output for GetOpt library for this case + + REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); + CHECK(parser.UsageCalled()); + + // BUG (TMA-1307): Usage message does not contain error. Error statement + // "option '--concurrency-range' requires an argument" written directly + // to std::out + // + CHECK_STRING("Usage Message", parser.GetUsageMessage(), ""); + + check_params = false; + } + + SUBCASE("wrong separator") + { + int argc = 5; + char* argv[argc] = {app_name, "-m", model_name, option_name, "100,400,10"}; + + REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); + CHECK(!parser.UsageCalled()); + + // BUG (TMA-1307): Should detect this and through an error. User will + // enter this and have no clue why the end and step sizes are not used + // correctly. + // + + // using_range = true; + // range.start = 100; + check_params = false; + } +} + + TEST_CASE("Testing Command Line Parser") { char* model_name = "my_model"; @@ -1066,61 +1119,12 @@ TEST_CASE("Testing Command Line Parser") SUBCASE("Option : --concurrency-range") { - CHECK_RANGE_PASS( - "--concurrency-range", exp->using_concurrency_range, - exp->concurrency_range.start, exp->concurrency_range.end, - exp->concurrency_range.step); - - expected_msg = CreateUsageMessage( - "--concurrency-range", "The value does not match ."); - CHECK_RANGE_FAIL("--concurrency-range", expected_msg); - - expected_msg = CreateUsageMessage( - "--concurrency-range", "Invalid value provided: bad:400:10"); - CHECK_RANGE_START_VALUE("--concurrency-range", expected_msg); - - expected_msg = CreateUsageMessage( - "--concurrency-range", "Invalid value provided: 100:bad:10"); - CHECK_RANGE_END_VALUE("--concurrency-range", expected_msg); - - expected_msg = CreateUsageMessage( - "--concurrency-range", "Invalid value provided: 100:400:bad"); - CHECK_RANGE_STEP_VALUE("--concurrency-range", expected_msg); - - SUBCASE("no options") - { - int argc = 4; - char* argv[argc] = {app_name, "-m", model_name, "--concurrency-range"}; - - opterr = 0; // Disable error output for GetOpt library for this case - - REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); - CHECK(parser.UsageCalled()); - - // BUG (TMA-1307): Usage message does not contain error. Error statement - // "option '--concurrency-range' requires an argument" written directly - // to std::out - // - CHECK_STRING("Usage Message", parser.GetUsageMessage(), ""); - } - - SUBCASE("wrong separator") - { - int argc = 5; - char* argv[argc] = { - app_name, "-m", model_name, "--concurrency-range", "100,400,10"}; - - REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); - CHECK(!parser.UsageCalled()); - - // BUG (TMA-1307): Should detect this and through an error. User will - // enter this and have no clue why the end and step sizes are not used - // correctly. - // + CheckValidRange( + app_name, model_name, "--concurrency-range", parser, act, + exp->using_concurrency_range, exp->concurrency_range); - exp->using_concurrency_range = true; - exp->concurrency_range.start = 100; - } + CheckInvalidRange( + app_name, model_name, "--concurrency-range", parser, act, check_params); SUBCASE("invalid condition - end and latency threshold are 0") { @@ -1147,28 +1151,13 @@ TEST_CASE("Testing Command Line Parser") SUBCASE("Option: --periodic-concurrency-range") { - CHECK_RANGE_PASS( - "--periodic-concurrency-range", exp->using_periodic_concurrency_range, - exp->periodic_concurrency_range.start, - exp->periodic_concurrency_range.end, - exp->periodic_concurrency_range.step); - - expected_msg = CreateUsageMessage( - "--periodic-concurrency-range", - "The value does not match ."); - CHECK_RANGE_FAIL("--periodic-concurrency-range", expected_msg); + CheckValidRange( + app_name, model_name, "--periodic-concurrency-range", parser, act, + exp->using_periodic_concurrency_range, exp->periodic_concurrency_range); - expected_msg = CreateUsageMessage( - "--periodic-concurrency-range", "Invalid value provided: bad:400:10"); - CHECK_RANGE_START_VALUE("--periodic-concurrency-range", expected_msg); - - expected_msg = CreateUsageMessage( - "--periodic-concurrency-range", "Invalid value provided: 100:bad:10"); - CHECK_RANGE_END_VALUE("--periodic-concurrency-range", expected_msg); - - expected_msg = CreateUsageMessage( - "--periodic-concurrency-range", "Invalid value provided: 100:400:bad"); - CHECK_RANGE_STEP_VALUE("--periodic-concurrency-range", expected_msg); + CheckInvalidRange( + app_name, model_name, "--periodic-concurrency-range", parser, act, + check_params); } SUBCASE("Option : --request-period") From 1edad2f6d10d046d79379aca94a5522616b512bc Mon Sep 17 00:00:00 2001 From: Hyunjae Woo Date: Fri, 1 Sep 2023 11:25:11 -0700 Subject: [PATCH 07/24] Add more tests --- src/c++/perf_analyzer/command_line_parser.cc | 32 ++- .../perf_analyzer/test_command_line_parser.cc | 205 ++++++++++++++---- 2 files changed, 193 insertions(+), 44 deletions(-) diff --git a/src/c++/perf_analyzer/command_line_parser.cc b/src/c++/perf_analyzer/command_line_parser.cc index b3d08fad2..c403aef2a 100644 --- a/src/c++/perf_analyzer/command_line_parser.cc +++ b/src/c++/perf_analyzer/command_line_parser.cc @@ -1545,6 +1545,16 @@ CLParser::ParseCommandLine(int argc, char** argv) index++; } + Range range{params_->periodic_concurrency_range}; + if (range.step == 0) { + Usage( + "Failed to parse --periodic-concurrency-range. The " + "value must be > 0."); + } else if ((range.end - range.start) % range.step != 0) { + Usage( + "Failed to parse --periodic-concurrency-range. The " + "value must be a factor of the range size ( - )."); + } break; } case 60: { @@ -1713,16 +1723,24 @@ CLParser::VerifyOptions() Usage("Cannot use concurrency options with --request-rate-range."); } - if (params_->using_request_rate_range && params_->using_concurrency_range) { + std::vector load_modes{ + params_->using_periodic_concurrency_range, + params_->using_concurrency_range, params_->using_request_rate_range, + params_->using_custom_intervals}; + if (std::count(load_modes.begin(), load_modes.end(), true) > 1) { Usage( - "Cannot specify --concurrency-range and --request-rate-range " - "simultaneously."); + "Cannot specify more then one inference load mode. Please choose only " + "one of the following modes: --concurrency-range, " + "--periodic-concurrency-range, --request-rate-range, or " + "--request-intervals."); } - // TODO: - // - check if two or more mode is specified - // - check if --profile-export-file is specified - // - check if (end - start) % step == 0 + if (params_->using_periodic_concurrency_range && + (params_->profile_export_file == "")) { + Usage( + "Must provide --profile-export-file when using the " + "--periodic-concurrency-range option."); + } if (params_->using_request_rate_range && params_->mpi_driver->IsMPIRun() && (params_->request_rate_range[SEARCH_RANGE::kEND] != 1.0 || diff --git a/src/c++/perf_analyzer/test_command_line_parser.cc b/src/c++/perf_analyzer/test_command_line_parser.cc index 3ba99f1b7..4c424f0d6 100644 --- a/src/c++/perf_analyzer/test_command_line_parser.cc +++ b/src/c++/perf_analyzer/test_command_line_parser.cc @@ -344,13 +344,17 @@ class TestCLParser : public CLParser { void CheckValidRange( - char* app_name, char* model_name, char* option_name, TestCLParser& parser, + std::vector& args, char* option_name, TestCLParser& parser, PAParamsPtr& act, bool& using_range, Range& range) { - SUBCASE("only start provided") + SUBCASE("start provided") { - int argc = 5; - char* argv[argc] = {app_name, "-m", model_name, option_name, "100"}; + args.push_back(option_name); + args.push_back("100"); // start + + int argc = args.size(); + char* argv[argc]; + std::copy(args.begin(), args.end(), argv); REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); CHECK(!parser.UsageCalled()); @@ -359,10 +363,14 @@ CheckValidRange( range.start = 100; } - SUBCASE("start and end provided") + SUBCASE("start:end provided") { - int argc = 5; - char* argv[argc] = {app_name, "-m", model_name, option_name, "100:400"}; + args.push_back(option_name); + args.push_back("100:400"); // start:end + + int argc = args.size(); + char* argv[argc]; + std::copy(args.begin(), args.end(), argv); REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); CHECK(!parser.UsageCalled()); @@ -372,10 +380,14 @@ CheckValidRange( range.end = 400; } - SUBCASE("start, end, and step provided") + SUBCASE("start:end:step provided") { - int argc = 5; - char* argv[argc] = {app_name, "-m", model_name, option_name, "100:400:10"}; + args.push_back(option_name); + args.push_back("100:400:10"); // start:end:step + + int argc = args.size(); + char* argv[argc]; + std::copy(args.begin(), args.end(), argv); REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); CHECK(!parser.UsageCalled()); @@ -389,16 +401,24 @@ CheckValidRange( void CheckInvalidRange( - char* app_name, char* model_name, char* option_name, TestCLParser& parser, + std::vector& args, char* option_name, TestCLParser& parser, PAParamsPtr& act, bool& check_params) { std::string expected_msg; + // FIXME (TMA-1307): Uncomment the subcase below when the issue is resolved. + // Currently the expected error message does not match the actual error + // message since TestCLParser ignores the exit statement when the Usage() is + // called and proceeds executing the program when it should stop the program. + /* SUBCASE("too many input values") { - int argc = 5; - char* argv[argc] = { - app_name, "-m", model_name, option_name, "200:100:25:10"}; + args.push_back(option_name); + args.push_back("200:100:25:10"); + + int argc = args.size(); + char* argv[argc]; + std::copy(args.begin(), args.end(), argv); REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); CHECK(parser.UsageCalled()); @@ -409,11 +429,16 @@ CheckInvalidRange( check_params = false; } + */ SUBCASE("invalid start value") { - int argc = 5; - char* argv[argc] = {app_name, "-m", model_name, option_name, "bad:400:10"}; + args.push_back(option_name); + args.push_back("bad:400:10"); + + int argc = args.size(); + char* argv[argc]; + std::copy(args.begin(), args.end(), argv); REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); CHECK(parser.UsageCalled()); @@ -427,8 +452,12 @@ CheckInvalidRange( SUBCASE("invalid end value") { - int argc = 5; - char* argv[argc] = {app_name, "-m", model_name, option_name, "100:bad:10"}; + args.push_back(option_name); + args.push_back("100:bad:10"); + + int argc = args.size(); + char* argv[argc]; + std::copy(args.begin(), args.end(), argv); REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); CHECK(parser.UsageCalled()); @@ -442,8 +471,12 @@ CheckInvalidRange( SUBCASE("invalid step value") { - int argc = 5; - char* argv[argc] = {app_name, "-m", model_name, option_name, "100:400:bad"}; + args.push_back(option_name); + args.push_back("100:400:bad"); + + int argc = args.size(); + char* argv[argc]; + std::copy(args.begin(), args.end(), argv); REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); CHECK(parser.UsageCalled()); @@ -455,10 +488,13 @@ CheckInvalidRange( check_params = false; } - SUBCASE("no options") + SUBCASE("no input values") { - int argc = 4; - char* argv[argc] = {app_name, "-m", model_name, option_name}; + args.push_back(option_name); + + int argc = args.size(); + char* argv[argc]; + std::copy(args.begin(), args.end(), argv); opterr = 0; // Disable error output for GetOpt library for this case @@ -476,8 +512,12 @@ CheckInvalidRange( SUBCASE("wrong separator") { - int argc = 5; - char* argv[argc] = {app_name, "-m", model_name, option_name, "100,400,10"}; + args.push_back(option_name); + args.push_back("100,400,10"); + + int argc = args.size(); + char* argv[argc]; + std::copy(args.begin(), args.end(), argv); REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); CHECK(!parser.UsageCalled()); @@ -487,8 +527,6 @@ CheckInvalidRange( // correctly. // - // using_range = true; - // range.start = 100; check_params = false; } } @@ -1119,12 +1157,12 @@ TEST_CASE("Testing Command Line Parser") SUBCASE("Option : --concurrency-range") { + std::vector args{app_name, "-m", model_name}; CheckValidRange( - app_name, model_name, "--concurrency-range", parser, act, - exp->using_concurrency_range, exp->concurrency_range); + args, "--concurrency-range", parser, act, exp->using_concurrency_range, + exp->concurrency_range); - CheckInvalidRange( - app_name, model_name, "--concurrency-range", parser, act, check_params); + CheckInvalidRange(args, "--concurrency-range", parser, act, check_params); SUBCASE("invalid condition - end and latency threshold are 0") { @@ -1149,15 +1187,102 @@ TEST_CASE("Testing Command Line Parser") } } - SUBCASE("Option: --periodic-concurrency-range") + SUBCASE("Option : --periodic-concurrency-range") { + char* option_name = "--periodic-concurrency-range"; + std::vector args{ + app_name, "-m", model_name, "--profile-export-file", "profile.json"}; + CheckValidRange( - app_name, model_name, "--periodic-concurrency-range", parser, act, - exp->using_periodic_concurrency_range, exp->periodic_concurrency_range); + args, option_name, parser, act, exp->using_periodic_concurrency_range, + exp->periodic_concurrency_range); + + CheckInvalidRange(args, option_name, parser, act, check_params); + + SUBCASE("more than one load mode") + { + args.push_back(option_name); + args.push_back("100:400"); + args.push_back("--concurrency-range"); + args.push_back("10:40"); + + int argc = args.size(); + char* argv[argc]; + std::copy(args.begin(), args.end(), argv); + + REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); + CHECK(parser.UsageCalled()); + + expected_msg = + "Cannot specify more then one inference load mode. Please choose " + "only one of the following modes: --concurrency-range, " + "--periodic-concurrency-range, --request-rate-range, or " + "--request-intervals."; + CHECK_STRING("Usage Message", parser.GetUsageMessage(), expected_msg); + + check_params = false; + } + + SUBCASE("no export file specified") + { + std::vector args_{app_name, "-m", model_name}; + args_.push_back(option_name); + args_.push_back("100:400"); - CheckInvalidRange( - app_name, model_name, "--periodic-concurrency-range", parser, act, - check_params); + int argc = args_.size(); + char* argv[argc]; + std::copy(args_.begin(), args_.end(), argv); + + REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); + CHECK(parser.UsageCalled()); + + expected_msg = + "Must provide --profile-export-file when using the " + "--periodic-concurrency-range option."; + CHECK_STRING("Usage Message", parser.GetUsageMessage(), expected_msg); + + check_params = false; + } + + SUBCASE("step is not factor of range size") + { + args.push_back(option_name); + args.push_back("100:400:7"); + + int argc = args.size(); + char* argv[argc]; + std::copy(args.begin(), args.end(), argv); + + REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); + CHECK(parser.UsageCalled()); + + expected_msg = CreateUsageMessage( + option_name, + "The value must be a factor of the range size ( - " + ")."); + CHECK_STRING("Usage Message", parser.GetUsageMessage(), expected_msg); + + check_params = false; + } + + SUBCASE("step is zero") + { + args.push_back(option_name); + args.push_back("10:400:0"); + + int argc = args.size(); + char* argv[argc]; + std::copy(args.begin(), args.end(), argv); + + REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); + CHECK(parser.UsageCalled()); + + expected_msg = + CreateUsageMessage(option_name, "The value must be > 0."); + CHECK_STRING("Usage Message", parser.GetUsageMessage(), expected_msg); + + check_params = false; + } } SUBCASE("Option : --request-period") @@ -1477,6 +1602,12 @@ TEST_CASE("Testing Command Line Parser") } if (check_params) { + if (act == nullptr) { + std::cerr << "Error: Attempting to access a null pointer `act` will lead " + "to undefined behavior. Terminating the test." + << std::endl; + exit(1); + } CHECK_PARAMS(act, exp); } optind = 1; // Reset GotOpt index, needed to parse the next command line From e68f37818d8b3a6c93e714a7d8fdd0ad210283f9 Mon Sep 17 00:00:00 2001 From: Hyunjae Woo Date: Tue, 5 Sep 2023 09:05:01 -0700 Subject: [PATCH 08/24] Small refactor --- .../perf_analyzer/test_command_line_parser.cc | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/src/c++/perf_analyzer/test_command_line_parser.cc b/src/c++/perf_analyzer/test_command_line_parser.cc index 4c424f0d6..e6e976e28 100644 --- a/src/c++/perf_analyzer/test_command_line_parser.cc +++ b/src/c++/perf_analyzer/test_command_line_parser.cc @@ -536,7 +536,9 @@ TEST_CASE("Testing Command Line Parser") { char* model_name = "my_model"; char* app_name = "test_perf_analyzer"; + std::string expected_msg; + std::vector args{app_name, "-m", model_name}; opterr = 1; // Enable error output for GetOpt library bool check_params = true; @@ -1157,7 +1159,6 @@ TEST_CASE("Testing Command Line Parser") SUBCASE("Option : --concurrency-range") { - std::vector args{app_name, "-m", model_name}; CheckValidRange( args, "--concurrency-range", parser, act, exp->using_concurrency_range, exp->concurrency_range); @@ -1190,8 +1191,10 @@ TEST_CASE("Testing Command Line Parser") SUBCASE("Option : --periodic-concurrency-range") { char* option_name = "--periodic-concurrency-range"; - std::vector args{ - app_name, "-m", model_name, "--profile-export-file", "profile.json"}; + + // Add required args that specifies where to dump profiled data + args.push_back("--profile-export-file"); + args.push_back("profile.json"); CheckValidRange( args, option_name, parser, act, exp->using_periodic_concurrency_range, @@ -1225,13 +1228,16 @@ TEST_CASE("Testing Command Line Parser") SUBCASE("no export file specified") { - std::vector args_{app_name, "-m", model_name}; - args_.push_back(option_name); - args_.push_back("100:400"); + // Remove the required args + args.pop_back(); + args.pop_back(); + + args.push_back(option_name); + args.push_back("100:400"); - int argc = args_.size(); + int argc = args.size(); char* argv[argc]; - std::copy(args_.begin(), args_.end(), argv); + std::copy(args.begin(), args.end(), argv); REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); CHECK(parser.UsageCalled()); @@ -1293,8 +1299,12 @@ TEST_CASE("Testing Command Line Parser") SUBCASE("set to 0") { - int argc = 5; - char* argv[argc] = {app_name, "-m", model_name, "--request-period", "0"}; + args.push_back("--request-period"); + args.push_back("0"); + + int argc = args.size(); + char* argv[argc]; + std::copy(args.begin(), args.end(), argv); REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); CHECK(parser.UsageCalled()); From 5fc5064ab7b4d501bba51ae82e44d1db81761de8 Mon Sep 17 00:00:00 2001 From: Hyunjae Woo Date: Tue, 5 Sep 2023 09:12:03 -0700 Subject: [PATCH 09/24] Refactor a subcase --- .../perf_analyzer/test_command_line_parser.cc | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/c++/perf_analyzer/test_command_line_parser.cc b/src/c++/perf_analyzer/test_command_line_parser.cc index e6e976e28..899cbda7b 100644 --- a/src/c++/perf_analyzer/test_command_line_parser.cc +++ b/src/c++/perf_analyzer/test_command_line_parser.cc @@ -1159,19 +1159,24 @@ TEST_CASE("Testing Command Line Parser") SUBCASE("Option : --concurrency-range") { + char* option_name = "--concurrency-range"; + CheckValidRange( - args, "--concurrency-range", parser, act, exp->using_concurrency_range, + args, option_name, parser, act, exp->using_concurrency_range, exp->concurrency_range); - CheckInvalidRange(args, "--concurrency-range", parser, act, check_params); + CheckInvalidRange(args, option_name, parser, act, check_params); SUBCASE("invalid condition - end and latency threshold are 0") { - int argc = 7; - char* argv[argc] = {app_name, "-m", - model_name, "--concurrency-range", - "100:0:25", "--latency-threshold", - "0"}; + args.push_back(option_name); + args.push_back("100:0:25"); + args.push_back("--latency-threshold"); + args.push_back("0"); + + int argc = args.size(); + char* argv[argc]; + std::copy(args.begin(), args.end(), argv); REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); CHECK(parser.UsageCalled()); @@ -1180,11 +1185,7 @@ TEST_CASE("Testing Command Line Parser") "The end of the search range and the latency limit can not be both 0 " "(or 0.0) simultaneously"); - exp->using_concurrency_range = true; - exp->concurrency_range.start = 100; - exp->concurrency_range.end = 0; - exp->concurrency_range.step = 25; - exp->latency_threshold_ms = 0; + check_params = false; } } From b060669423e0320b2b140db8a10f857c212b8ac6 Mon Sep 17 00:00:00 2001 From: Hyunjae Woo Date: Tue, 5 Sep 2023 09:48:20 -0700 Subject: [PATCH 10/24] Require bi-directional gRPC streaming for periodic concurrency mode --- src/c++/perf_analyzer/command_line_parser.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/c++/perf_analyzer/command_line_parser.cc b/src/c++/perf_analyzer/command_line_parser.cc index c403aef2a..5ff885de8 100644 --- a/src/c++/perf_analyzer/command_line_parser.cc +++ b/src/c++/perf_analyzer/command_line_parser.cc @@ -1735,6 +1735,12 @@ CLParser::VerifyOptions() "--request-intervals."); } + if (params_->using_periodic_concurrency_range && !params_->streaming) { + Usage( + "The --periodic-concurrency-range option requires bi-directional gRPC " + "streaming."); + } + if (params_->using_periodic_concurrency_range && (params_->profile_export_file == "")) { Usage( From be7a4417c1278db893ae11cd65e3b1fb86bfee15 Mon Sep 17 00:00:00 2001 From: Hyunjae Woo Date: Tue, 5 Sep 2023 14:32:48 -0700 Subject: [PATCH 11/24] Address feedback --- src/c++/perf_analyzer/command_line_parser.cc | 14 +++++++------- src/c++/perf_analyzer/docs/cli.md | 12 ++++++------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/c++/perf_analyzer/command_line_parser.cc b/src/c++/perf_analyzer/command_line_parser.cc index 5ff885de8..d4090ca6b 100644 --- a/src/c++/perf_analyzer/command_line_parser.cc +++ b/src/c++/perf_analyzer/command_line_parser.cc @@ -279,15 +279,15 @@ CLParser::Usage(const std::string& msg) std::cerr << FormatMessage( "--periodic-concurrency-range : Determines the " - "range of concurrency levels in the similar manner as the " - "--concurrency-range. The perf_analyzer will start from the " + "range of concurrency levels in the similar manner as " + "--concurrency-range. Perf Analyzer will start from the " "concurrency level of 'start' and go until it reaches 'end' with " "a stride of 'step'. Unlike --concurrency-range, the user can " "specify *when* to periodically increase the concurrency level " "using the --request-period option. The concurrency level will " "periodically increase for every n-th response specified by " "--request-period. Since this disables stability check in " - "perf_analyzer and reports response timestamps only, the user " + "Perf Analyzer and reports response timestamps only, the user " "must provide --profile-export-file to specify where to dump all " "the measured timestamps. The default values of 'start', 'end', " "and 'step' are 1.", @@ -295,10 +295,10 @@ CLParser::Usage(const std::string& msg) << std::endl; std::cerr << FormatMessage( - "--request-period: Indicates the number of responses that each " - "request will wait until it launches a new, concurrent " - "request when --periodic-concurrency-range is specified. " - "Default value is 10.", + "--request-period : Indicates the number of responses that " + "each request must receive before new, concurrent requests are " + "sent when --periodic-concurrency-range is specified. Default " + "value is 10.", 18) << std::endl; std::cerr diff --git a/src/c++/perf_analyzer/docs/cli.md b/src/c++/perf_analyzer/docs/cli.md index 602a453a0..406fc5b0a 100644 --- a/src/c++/perf_analyzer/docs/cli.md +++ b/src/c++/perf_analyzer/docs/cli.md @@ -182,23 +182,23 @@ mode. #### `--periodic-concurrency-range=` -Determines the range of concurrency levels in the similar manner as the -`--concurrency-range`. The perf_analyzer will start from the concurrency level +Specifies the range of concurrency levels in the similar manner as +`--concurrency-range`. Perf Analyzer will start from the concurrency level of 'start' and go until it reaches 'end' with a stride of 'step'. Unlike `--concurrency-range`, the user can specify *when* to periodically increase the concurrency level using the `--request-period` option. The concurrency level will periodically increase for every `n`-th response specified by `--request-period`. -Since this disables stability check in perf_analyzer and reports response +Since this disables stability check in Perf Analyzer and reports response timestamps only, the user must provide `--profile-export-file` to specify where to dump all the measured response timestamps. The default values of 'start', 'end', and 'step' are `1`. -#### `--request-period ` +#### `--request-period=` -Indicates the number of responses that each request will wait until it launches -a new, concurrent request when `--periodic-concurrency-range` is specified. +Specifies the number of responses that each request must receive before new, +concurrent requests are sent when `--periodic-concurrency-range` is specified. Default value is `10`. From 8b458a994a7cbe6004c1deeb0e4fedba78eb7475 Mon Sep 17 00:00:00 2001 From: Hyunjae Woo Date: Tue, 5 Sep 2023 14:52:17 -0700 Subject: [PATCH 12/24] Refine the error message --- src/c++/perf_analyzer/test_command_line_parser.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/c++/perf_analyzer/test_command_line_parser.cc b/src/c++/perf_analyzer/test_command_line_parser.cc index 899cbda7b..9592472c4 100644 --- a/src/c++/perf_analyzer/test_command_line_parser.cc +++ b/src/c++/perf_analyzer/test_command_line_parser.cc @@ -1614,9 +1614,10 @@ TEST_CASE("Testing Command Line Parser") if (check_params) { if (act == nullptr) { - std::cerr << "Error: Attempting to access a null pointer `act` will lead " - "to undefined behavior. Terminating the test." - << std::endl; + std::cerr + << "Error: Attempting to access `act` but was not initialized. Check " + "if the test cases are missing `check_params = false` statement." + << std::endl; exit(1); } CHECK_PARAMS(act, exp); From 53b653ad4abb385cf24c4d1116ed7130b27cb0c4 Mon Sep 17 00:00:00 2001 From: Hyunjae Woo Date: Tue, 5 Sep 2023 17:21:06 -0700 Subject: [PATCH 13/24] Add bi-directional gRPC streaming options for periodic concurrency mode --- src/c++/perf_analyzer/test_command_line_parser.cc | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/c++/perf_analyzer/test_command_line_parser.cc b/src/c++/perf_analyzer/test_command_line_parser.cc index 9592472c4..172e9a5a1 100644 --- a/src/c++/perf_analyzer/test_command_line_parser.cc +++ b/src/c++/perf_analyzer/test_command_line_parser.cc @@ -1194,8 +1194,13 @@ TEST_CASE("Testing Command Line Parser") char* option_name = "--periodic-concurrency-range"; // Add required args that specifies where to dump profiled data - args.push_back("--profile-export-file"); - args.push_back("profile.json"); + args.insert( + args.end(), {"-i", "grpc", "--async", "--streaming", + "--profile-export-file", "profile.json"}); + exp->protocol = cb::ProtocolType::GRPC; + exp->async = true; + exp->streaming = true; + exp->url = "localhost:8001"; // gRPC url CheckValidRange( args, option_name, parser, act, exp->using_periodic_concurrency_range, @@ -1229,7 +1234,7 @@ TEST_CASE("Testing Command Line Parser") SUBCASE("no export file specified") { - // Remove the required args + // Remove the export file args args.pop_back(); args.pop_back(); From 96758ece923c539da45ad8f28b64afd4a2af78f6 Mon Sep 17 00:00:00 2001 From: Hyunjae Woo Date: Thu, 21 Sep 2023 12:14:43 -0700 Subject: [PATCH 14/24] Add request-parameter option and refactor --- src/c++/perf_analyzer/command_line_parser.cc | 139 +++++++++++++----- src/c++/perf_analyzer/command_line_parser.h | 1 + src/c++/perf_analyzer/perf_utils.h | 7 + .../perf_analyzer/test_command_line_parser.cc | 102 ++++++++++--- 4 files changed, 195 insertions(+), 54 deletions(-) diff --git a/src/c++/perf_analyzer/command_line_parser.cc b/src/c++/perf_analyzer/command_line_parser.cc index d4090ca6b..356033794 100644 --- a/src/c++/perf_analyzer/command_line_parser.cc +++ b/src/c++/perf_analyzer/command_line_parser.cc @@ -47,6 +47,31 @@ CLParser::Parse(int argc, char** argv) return params_; } +std::vector +SplitString(const std::string& str, const std::string& delimiter = ":") +{ + std::vector substrs; + size_t pos = 0; + while (pos != std::string::npos) { + size_t colon_pos = str.find(":", pos); + substrs.push_back(str.substr(pos, colon_pos - pos)); + if (colon_pos == std::string::npos) { + pos = colon_pos; + } else { + pos = colon_pos + 1; + } + } + return substrs; +} + +void +ToLowerCase(std::string& s) +{ + std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { + return std::tolower(c); + }); +} + // Used to format the usage message std::string CLParser::FormatMessage(std::string str, int offset) const @@ -279,18 +304,21 @@ CLParser::Usage(const std::string& msg) std::cerr << FormatMessage( "--periodic-concurrency-range : Determines the " - "range of concurrency levels in the similar manner as " - "--concurrency-range. Perf Analyzer will start from the " - "concurrency level of 'start' and go until it reaches 'end' with " - "a stride of 'step'. Unlike --concurrency-range, the user can " + "range of concurrency levels in the similar but slightly " + "different manner as the --concurrency-range. Perf Analyzer will " + "start from the concurrency level of 'start' and increase by " + "'step' each time. Unlike --concurrency-range, the 'end' " + "indicates the *total* number of concurrency since the 'start' " + "(including) and will stop increasing once the cumulative number " + "of concurrent requests has reached the 'end'. The user can " "specify *when* to periodically increase the concurrency level " "using the --request-period option. The concurrency level will " "periodically increase for every n-th response specified by " - "--request-period. Since this disables stability check in " - "Perf Analyzer and reports response timestamps only, the user " - "must provide --profile-export-file to specify where to dump all " - "the measured timestamps. The default values of 'start', 'end', " - "and 'step' are 1.", + "--request-period. Since this disables stability check in Perf " + "Analyzer and reports response timestamps only, the user must " + "provide --profile-export-file to specify where to dump all the " + "measured timestamps. The default values of 'start', 'end', and " + "'step' are 1.", 18) << std::endl; std::cerr @@ -301,6 +329,17 @@ CLParser::Usage(const std::string& msg) "value is 10.", 18) << std::endl; + std::cerr + << FormatMessage( + "--request-parameter : Specifies a custom " + "parameter that can be sent to a Triton backend as part of the " + "request. For example, providing '--request-parameter " + "max_tokens:256:uint' to the command line will set an additional " + "parameter 'max_tokens' of type 'uint' to 256 as part of the " + "request. The --request-parameter may be specified multiple times " + "for different custom parameters.", + 18) + << std::endl; std::cerr << FormatMessage( " --request-rate-range : Determines the range of " @@ -835,6 +874,7 @@ CLParser::ParseCommandLine(int argc, char** argv) {"profile-export-file", required_argument, 0, 58}, {"periodic-concurrency-range", required_argument, 0, 59}, {"request-period", required_argument, 0, 60}, + {"request-parameter", required_argument, 0, 61}, {0, 0, 0, 0}}; // Parse commandline... @@ -1514,35 +1554,26 @@ CLParser::ParseCommandLine(int argc, char** argv) case 59: { params_->using_periodic_concurrency_range = true; std::string arg = optarg; - size_t pos = 0; - int index = 0; - while (pos != std::string::npos) { - size_t colon_pos = arg.find(":", pos); - if (index > 2) { - Usage( - "Failed to parse --periodic-concurrency-range. The value " - "does not match ."); - } - int64_t val; - if (colon_pos == std::string::npos) { - val = std::stoull(arg.substr(pos, colon_pos)); - pos = colon_pos; - } else { - val = std::stoull(arg.substr(pos, colon_pos - pos)); - pos = colon_pos + 1; - } - switch (index) { - case 0: - params_->periodic_concurrency_range.start = val; - break; - case 1: - params_->periodic_concurrency_range.end = val; - break; - case 2: - params_->periodic_concurrency_range.step = val; - break; + std::vector values{SplitString(arg)}; + if (values.size() < 2) { + Usage( + "Failed to parse --periodic-concurrency-range. Both " + "and values must be provided."); + } else if (values.size() > 3) { + Usage( + "Failed to parse --periodic-concurrency-range. The value does " + "not match ."); + } + + for (size_t i = 0; i < values.size(); ++i) { + uint64_t val = std::stoull(values[i]); + if (i == 0) { + params_->periodic_concurrency_range.start = val; + } else if (i == 1) { + params_->periodic_concurrency_range.end = val; + } else if (i == 2) { + params_->periodic_concurrency_range.step = val; } - index++; } Range range{params_->periodic_concurrency_range}; @@ -1550,6 +1581,10 @@ CLParser::ParseCommandLine(int argc, char** argv) Usage( "Failed to parse --periodic-concurrency-range. The " "value must be > 0."); + } else if (range.start > range.end) { + Usage( + "Failed to parse --periodic-concurrency-range. The " + "must be <= ."); } else if ((range.end - range.start) % range.step != 0) { Usage( "Failed to parse --periodic-concurrency-range. The " @@ -1566,6 +1601,36 @@ CLParser::ParseCommandLine(int argc, char** argv) } break; } + case 61: { + std::string arg = optarg; + std::vector values{SplitString(arg)}; + if (values.size() != 3) { + Usage( + "Failed to parse --request-parameter. The value does not match " + "."); + } + + std::for_each(values.begin(), values.end(), ToLowerCase); + std::string name{values[0]}; + std::string value{values[1]}; + std::string type{values[2]}; + + RequestParameter param; + if (type == "bool") { + param.bool_value = value == "true" ? true : false; + } else if (type == "uint") { + param.uint_value = std::stoull(value); + } else if (type == "int") { + param.int_value = std::stoll(value); + } else if (type == "string") { + param.str_value = value; + } else { + Usage( + "Failed to parse --request-parameter. Unsupported type: '" + + type + "'."); + } + break; + } case 'v': params_->extra_verbose = params_->verbose; params_->verbose = true; diff --git a/src/c++/perf_analyzer/command_line_parser.h b/src/c++/perf_analyzer/command_line_parser.h index e92f638ab..2dfcf8d43 100644 --- a/src/c++/perf_analyzer/command_line_parser.h +++ b/src/c++/perf_analyzer/command_line_parser.h @@ -61,6 +61,7 @@ struct PerfAnalyzerParameters { bool using_periodic_concurrency_range = false; Range periodic_concurrency_range{1, 1, 1}; uint64_t request_period = 10; + std::unordered_map request_parameters; uint64_t latency_threshold_ms = NO_LIMIT; double stability_threshold = 0.1; size_t max_trials = 10; diff --git a/src/c++/perf_analyzer/perf_utils.h b/src/c++/perf_analyzer/perf_utils.h index 7166936a9..92c391f60 100644 --- a/src/c++/perf_analyzer/perf_utils.h +++ b/src/c++/perf_analyzer/perf_utils.h @@ -83,6 +83,13 @@ class Range { T step; }; +struct RequestParameter { + std::string str_value; + int64_t int_value; + uint64_t uint_value; + bool bool_value; +}; + // Converts the datatype from tensorflow to perf analyzer space // \param tf_dtype The data type string returned from the model metadata. // \param datatype Returns the datatype in perf_analyzer space. diff --git a/src/c++/perf_analyzer/test_command_line_parser.cc b/src/c++/perf_analyzer/test_command_line_parser.cc index 172e9a5a1..794f775c8 100644 --- a/src/c++/perf_analyzer/test_command_line_parser.cc +++ b/src/c++/perf_analyzer/test_command_line_parser.cc @@ -347,22 +347,6 @@ CheckValidRange( std::vector& args, char* option_name, TestCLParser& parser, PAParamsPtr& act, bool& using_range, Range& range) { - SUBCASE("start provided") - { - args.push_back(option_name); - args.push_back("100"); // start - - int argc = args.size(); - char* argv[argc]; - std::copy(args.begin(), args.end(), argv); - - REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); - CHECK(!parser.UsageCalled()); - - using_range = true; - range.start = 100; - } - SUBCASE("start:end provided") { args.push_back(option_name); @@ -520,7 +504,7 @@ CheckInvalidRange( std::copy(args.begin(), args.end(), argv); REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); - CHECK(!parser.UsageCalled()); + CHECK(parser.UsageCalled()); // BUG (TMA-1307): Should detect this and through an error. User will // enter this and have no clue why the end and step sizes are not used @@ -1161,6 +1145,22 @@ TEST_CASE("Testing Command Line Parser") { char* option_name = "--concurrency-range"; + SUBCASE("start provided") + { + args.push_back(option_name); + args.push_back("100"); // start + + int argc = args.size(); + char* argv[argc]; + std::copy(args.begin(), args.end(), argv); + + REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); + CHECK(!parser.UsageCalled()); + + exp->using_concurrency_range = true; + exp->concurrency_range.start = 100; + } + CheckValidRange( args, option_name, parser, act, exp->using_concurrency_range, exp->concurrency_range); @@ -1202,6 +1202,26 @@ TEST_CASE("Testing Command Line Parser") exp->streaming = true; exp->url = "localhost:8001"; // gRPC url + SUBCASE("start provided") + { + args.push_back(option_name); + args.push_back("100"); // start + + int argc = args.size(); + char* argv[argc]; + std::copy(args.begin(), args.end(), argv); + + REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); + CHECK(parser.UsageCalled()); + + // FIXME (TMA-1307): Currently the expected error message does not match + // the actual error message since TestCLParser ignores the exit statement + // when the Usage() is called and proceeds executing the program when it + // should stop the program. + + check_params = false; + } + CheckValidRange( args, option_name, parser, act, exp->using_periodic_concurrency_range, exp->periodic_concurrency_range); @@ -1319,6 +1339,54 @@ TEST_CASE("Testing Command Line Parser") } } + SUBCASE("Option : --request-parameter") + { + char* option_name = "--request-parameter"; + + SUBCASE("missing type") + { + args.push_back(option_name); + args.push_back("max_tokens:256"); + + int argc = args.size(); + char* argv[argc]; + std::copy(args.begin(), args.end(), argv); + + REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); + CHECK(parser.UsageCalled()); + + // FIXME (TMA-1307): Currently the expected error message does not match + // the actual error message since TestCLParser ignores the exit statement + // when the Usage() is called and proceeds executing the program when it + // should stop the program. + // + // expected_msg = CreateUsageMessage( + // option_name, "The value does not match ."); + // CHECK_STRING("Usage Message", parser.GetUsageMessage(), expected_msg); + + check_params = false; + } + + SUBCASE("unsupported type") + { + args.push_back(option_name); + args.push_back("max_tokens:256:hello"); + + int argc = args.size(); + char* argv[argc]; + std::copy(args.begin(), args.end(), argv); + + REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); + CHECK(parser.UsageCalled()); + + expected_msg = + CreateUsageMessage(option_name, "Unsupported type: 'hello'."); + CHECK_STRING("Usage Message", parser.GetUsageMessage(), expected_msg); + + check_params = false; + } + } + SUBCASE("Option : --latency-threshold") { expected_msg = CreateUsageMessage( From fed756661c909b5e37d0edba56487e9e4f4bae59 Mon Sep 17 00:00:00 2001 From: Hyunjae Woo Date: Thu, 21 Sep 2023 13:23:23 -0700 Subject: [PATCH 15/24] Refactor --- src/c++/perf_analyzer/command_line_parser.cc | 44 +++++++------------ .../perf_analyzer/test_command_line_parser.cc | 40 ++++++++--------- 2 files changed, 35 insertions(+), 49 deletions(-) diff --git a/src/c++/perf_analyzer/command_line_parser.cc b/src/c++/perf_analyzer/command_line_parser.cc index 356033794..d02a96ed5 100644 --- a/src/c++/perf_analyzer/command_line_parser.cc +++ b/src/c++/perf_analyzer/command_line_parser.cc @@ -964,37 +964,23 @@ CLParser::ParseCommandLine(int argc, char** argv) case 7: { params_->using_concurrency_range = true; std::string arg = optarg; - size_t pos = 0; - int index = 0; - while (pos != std::string::npos) { - size_t colon_pos = arg.find(":", pos); - if (index > 2) { - Usage( - "Failed to parse --concurrency-range. The value does not " - "match ."); - } - int64_t val; - if (colon_pos == std::string::npos) { - val = std::stoull(arg.substr(pos, colon_pos)); - pos = colon_pos; - } else { - val = std::stoull(arg.substr(pos, colon_pos - pos)); - pos = colon_pos + 1; - } - switch (index) { - case 0: - params_->concurrency_range.start = val; - break; - case 1: - params_->concurrency_range.end = val; - break; - case 2: - params_->concurrency_range.step = val; - break; - } - index++; + std::vector values{SplitString(arg)}; + if (values.size() > 3) { + Usage( + "Failed to parse --concurrency-range. The value does not match " + "."); } + for (size_t i = 0; i < values.size(); ++i) { + uint64_t val = std::stoull(values[i]); + if (i == 0) { + params_->concurrency_range.start = val; + } else if (i == 1) { + params_->concurrency_range.end = val; + } else if (i == 2) { + params_->concurrency_range.step = val; + } + } break; } case 8: diff --git a/src/c++/perf_analyzer/test_command_line_parser.cc b/src/c++/perf_analyzer/test_command_line_parser.cc index 794f775c8..ef0375de8 100644 --- a/src/c++/perf_analyzer/test_command_line_parser.cc +++ b/src/c++/perf_analyzer/test_command_line_parser.cc @@ -493,26 +493,6 @@ CheckInvalidRange( check_params = false; } - - SUBCASE("wrong separator") - { - args.push_back(option_name); - args.push_back("100,400,10"); - - int argc = args.size(); - char* argv[argc]; - std::copy(args.begin(), args.end(), argv); - - REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); - CHECK(parser.UsageCalled()); - - // BUG (TMA-1307): Should detect this and through an error. User will - // enter this and have no clue why the end and step sizes are not used - // correctly. - // - - check_params = false; - } } @@ -1167,6 +1147,26 @@ TEST_CASE("Testing Command Line Parser") CheckInvalidRange(args, option_name, parser, act, check_params); + SUBCASE("wrong separator") + { + args.push_back(option_name); + args.push_back("100,400,10"); + + int argc = args.size(); + char* argv[argc]; + std::copy(args.begin(), args.end(), argv); + + REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); + CHECK(!parser.UsageCalled()); + + // BUG (TMA-1307): Should detect this and through an error. User will + // enter this and have no clue why the end and step sizes are not used + // correctly. + // + + check_params = false; + } + SUBCASE("invalid condition - end and latency threshold are 0") { args.push_back(option_name); From 336c8bf17924a398e0b6fbd5b3a770c7d6b1df60 Mon Sep 17 00:00:00 2001 From: Hyunjae Woo Date: Thu, 21 Sep 2023 13:27:50 -0700 Subject: [PATCH 16/24] Add valid case for request-parameter option --- .../perf_analyzer/test_command_line_parser.cc | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/c++/perf_analyzer/test_command_line_parser.cc b/src/c++/perf_analyzer/test_command_line_parser.cc index ef0375de8..8b6864b74 100644 --- a/src/c++/perf_analyzer/test_command_line_parser.cc +++ b/src/c++/perf_analyzer/test_command_line_parser.cc @@ -1367,6 +1367,23 @@ TEST_CASE("Testing Command Line Parser") check_params = false; } + SUBCASE("valid parameter") + { + args.push_back(option_name); + args.push_back("max_tokens:256:uint"); + + int argc = args.size(); + char* argv[argc]; + std::copy(args.begin(), args.end(), argv); + + REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); + CHECK(!parser.UsageCalled()); + + RequestParameter param; + param.uint_value = 256; + exp->request_parameters["max_tokens"] = param; + } + SUBCASE("unsupported type") { args.push_back(option_name); From 99e6366a7adf76d0e838071fe0713a5808c3f6a2 Mon Sep 17 00:00:00 2001 From: Hyunjae Woo Date: Thu, 21 Sep 2023 13:40:49 -0700 Subject: [PATCH 17/24] Add --request-parameter doc and edit periodic concurrency description --- src/c++/perf_analyzer/docs/cli.md | 36 ++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/src/c++/perf_analyzer/docs/cli.md b/src/c++/perf_analyzer/docs/cli.md index 406fc5b0a..6e5dd953e 100644 --- a/src/c++/perf_analyzer/docs/cli.md +++ b/src/c++/perf_analyzer/docs/cli.md @@ -182,16 +182,18 @@ mode. #### `--periodic-concurrency-range=` -Specifies the range of concurrency levels in the similar manner as -`--concurrency-range`. Perf Analyzer will start from the concurrency level -of 'start' and go until it reaches 'end' with a stride of 'step'. -Unlike `--concurrency-range`, the user can specify *when* to periodically -increase the concurrency level using the `--request-period` option. -The concurrency level will periodically increase for every `n`-th response -specified by `--request-period`. -Since this disables stability check in Perf Analyzer and reports response -timestamps only, the user must provide `--profile-export-file` to specify where -to dump all the measured response timestamps. +Specifies the range of concurrency levels in the similar but slightly different +manner as the `--concurrency-range`. Perf Analyzer will start from the +concurrency level of 'start' and increase by 'step' each time. Unlike +`--concurrency-range`, the 'end' indicates the *total* number of concurrency +since the 'start' (including) and will stop increasing once the cumulative +number of concurrent requests has reached the 'end'. The user can specify +*when* to periodically increase the concurrency level using the +`--request-period` option. The concurrency level will periodically increase for +every `n`-th response specified by `--request-period`. Since this disables +stability check in Perf Analyzer and reports response timestamps only, the user +must provide `--profile-export-file` to specify where to dump all the measured +timestamps. The default values of 'start', 'end', and 'step' are `1`. @@ -202,6 +204,20 @@ concurrent requests are sent when `--periodic-concurrency-range` is specified. Default value is `10`. +#### `--request-parameter=` + +Specifies a custom parameter that can be sent to a Triton backend as part of +the request. For example, providing '--request-parameter max_tokens:256:uint' +to the command line will set an additional parameter 'max_tokens' of type +'uint' to 256 as part of the request. The --request-parameter may be specified +multiple times for different custom parameters. + +Valid `type` values are: `bool`, `int`, `uint`, and `string`. + +> **NOTE** +> +> The `--request-parameter` is currently only supported by gRPC protocol. + #### `--request-rate-range=` Specifies the range of request rates for load generated by Perf Analyzer. This From 44bfa61719865beb20b4f8e315327b89da9476a4 Mon Sep 17 00:00:00 2001 From: Hyunjae Woo Date: Thu, 21 Sep 2023 13:45:03 -0700 Subject: [PATCH 18/24] Custom request parameter is currently only supported by gRPC --- src/c++/perf_analyzer/command_line_parser.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/c++/perf_analyzer/command_line_parser.cc b/src/c++/perf_analyzer/command_line_parser.cc index d02a96ed5..71d5ef549 100644 --- a/src/c++/perf_analyzer/command_line_parser.cc +++ b/src/c++/perf_analyzer/command_line_parser.cc @@ -1799,6 +1799,13 @@ CLParser::VerifyOptions() "--periodic-concurrency-range option."); } + if (params_->request_parameters.size() > 0 && + params_->protocol != cb::ProtocolType::GRPC) { + Usage( + "The --request-parameter option is currently only supported by gRPC " + "protocol."); + } + if (params_->using_request_rate_range && params_->mpi_driver->IsMPIRun() && (params_->request_rate_range[SEARCH_RANGE::kEND] != 1.0 || params_->request_rate_range[SEARCH_RANGE::kSTEP] != 1.0)) { From 67a3e64f74d9d39047d477830274f0b4f96df86a Mon Sep 17 00:00:00 2001 From: Hyunjae Woo Date: Mon, 25 Sep 2023 16:20:31 -0700 Subject: [PATCH 19/24] Parse and store the type of request parameter --- src/c++/perf_analyzer/command_line_parser.cc | 5 +++++ src/c++/perf_analyzer/perf_utils.h | 3 +++ 2 files changed, 8 insertions(+) diff --git a/src/c++/perf_analyzer/command_line_parser.cc b/src/c++/perf_analyzer/command_line_parser.cc index 71d5ef549..25b5f9af5 100644 --- a/src/c++/perf_analyzer/command_line_parser.cc +++ b/src/c++/perf_analyzer/command_line_parser.cc @@ -1603,18 +1603,23 @@ CLParser::ParseCommandLine(int argc, char** argv) RequestParameter param; if (type == "bool") { + param.type = RequestParameterType::BOOL; param.bool_value = value == "true" ? true : false; } else if (type == "uint") { + param.type = RequestParameterType::UINT; param.uint_value = std::stoull(value); } else if (type == "int") { + param.type = RequestParameterType::INT; param.int_value = std::stoll(value); } else if (type == "string") { + param.type = RequestParameterType::STRING; param.str_value = value; } else { Usage( "Failed to parse --request-parameter. Unsupported type: '" + type + "'."); } + params_->request_parameters[name] = param; break; } case 'v': diff --git a/src/c++/perf_analyzer/perf_utils.h b/src/c++/perf_analyzer/perf_utils.h index 92c391f60..887d38077 100644 --- a/src/c++/perf_analyzer/perf_utils.h +++ b/src/c++/perf_analyzer/perf_utils.h @@ -83,11 +83,14 @@ class Range { T step; }; +enum RequestParameterType { STRING = 0, INT = 1, UINT = 2, BOOL = 3 }; + struct RequestParameter { std::string str_value; int64_t int_value; uint64_t uint_value; bool bool_value; + RequestParameterType type; }; // Converts the datatype from tensorflow to perf analyzer space From f9bd696ba40c20b45011c74989b3b21d79e371ea Mon Sep 17 00:00:00 2001 From: Hyunjae Woo Date: Mon, 25 Sep 2023 16:21:00 -0700 Subject: [PATCH 20/24] Add checks between act vs. exp --- .../perf_analyzer/test_command_line_parser.cc | 78 ++++++++++++++----- 1 file changed, 58 insertions(+), 20 deletions(-) diff --git a/src/c++/perf_analyzer/test_command_line_parser.cc b/src/c++/perf_analyzer/test_command_line_parser.cc index 8b6864b74..2e666e9a2 100644 --- a/src/c++/perf_analyzer/test_command_line_parser.cc +++ b/src/c++/perf_analyzer/test_command_line_parser.cc @@ -175,6 +175,37 @@ CHECK_PARAMS(PAParamsPtr act, PAParamsPtr exp) CHECK_STRING(act->filename, act->filename); CHECK(act->mpi_driver != nullptr); CHECK_STRING(act->memory_type, exp->memory_type); + CHECK( + act->using_periodic_concurrency_range == + exp->using_periodic_concurrency_range); + CHECK( + act->periodic_concurrency_range.start == + exp->periodic_concurrency_range.start); + CHECK( + act->periodic_concurrency_range.end == + exp->periodic_concurrency_range.end); + CHECK( + act->periodic_concurrency_range.step == + exp->periodic_concurrency_range.step); + CHECK(act->request_period == exp->request_period); + CHECK(act->request_parameters.size() == exp->request_parameters.size()); + for (auto act_param : act->request_parameters) { + auto exp_param = exp->request_parameters.find(act_param.first); + REQUIRE_MESSAGE( + exp_param != exp->request_parameters.end(), + "Unexpected parameter: ", act_param.first); + + CHECK(act_param.second.type == exp_param->second.type); + if (act_param.second.type == RequestParameterType::STRING) { + CHECK(act_param.second.str_value == exp_param->second.str_value); + } else if (act_param.second.type == RequestParameterType::INT) { + CHECK(act_param.second.int_value == exp_param->second.int_value); + } else if (act_param.second.type == RequestParameterType::UINT) { + CHECK(act_param.second.uint_value == exp_param->second.uint_value); + } else if (act_param.second.type == RequestParameterType::BOOL) { + CHECK(act_param.second.bool_value == exp_param->second.bool_value); + } + } } @@ -1343,45 +1374,52 @@ TEST_CASE("Testing Command Line Parser") { char* option_name = "--request-parameter"; - SUBCASE("missing type") + // Add required args that specifies where to dump profiled data + args.insert(args.end(), {"-i", "grpc", "--async", "--streaming"}); + exp->protocol = cb::ProtocolType::GRPC; + exp->async = true; + exp->streaming = true; + exp->url = "localhost:8001"; // gRPC url + + SUBCASE("valid parameter") { args.push_back(option_name); - args.push_back("max_tokens:256"); + args.push_back("max_tokens:256:uint"); int argc = args.size(); char* argv[argc]; std::copy(args.begin(), args.end(), argv); REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); - CHECK(parser.UsageCalled()); - - // FIXME (TMA-1307): Currently the expected error message does not match - // the actual error message since TestCLParser ignores the exit statement - // when the Usage() is called and proceeds executing the program when it - // should stop the program. - // - // expected_msg = CreateUsageMessage( - // option_name, "The value does not match ."); - // CHECK_STRING("Usage Message", parser.GetUsageMessage(), expected_msg); + CHECK(!parser.UsageCalled()); - check_params = false; + RequestParameter param; + param.uint_value = 256; + param.type = RequestParameterType::UINT; + exp->request_parameters["max_tokens"] = param; } - SUBCASE("valid parameter") + SUBCASE("missing type") { args.push_back(option_name); - args.push_back("max_tokens:256:uint"); + args.push_back("max_tokens:256"); int argc = args.size(); char* argv[argc]; std::copy(args.begin(), args.end(), argv); - REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); - CHECK(!parser.UsageCalled()); + // FIXME (TMA-1307): Currently the expected error message does not match + // the actual error message since TestCLParser ignores the exit statement + // when the Usage() is called and proceeds executing the program when it + // should stop the program. + // + // REQUIRE_NOTHROW(act = parser.Parse(argc, argv)); + // CHECK(parser.UsageCalled()); + // expected_msg = CreateUsageMessage( + // option_name, "The value does not match ."); + // CHECK_STRING("Usage Message", parser.GetUsageMessage(), expected_msg); - RequestParameter param; - param.uint_value = 256; - exp->request_parameters["max_tokens"] = param; + check_params = false; } SUBCASE("unsupported type") From 2064297a36fc4a2fb14449d527151d62ad00faac Mon Sep 17 00:00:00 2001 From: Hyunjae Woo Date: Tue, 26 Sep 2023 17:35:30 -0700 Subject: [PATCH 21/24] Remove uint type and rebase --- src/c++/perf_analyzer/command_line_parser.cc | 15 ++++++--------- src/c++/perf_analyzer/command_line_parser.h | 6 +----- src/c++/perf_analyzer/docs/cli.md | 6 +++--- src/c++/perf_analyzer/perf_analyzer.cc | 10 +--------- src/c++/perf_analyzer/perf_utils.h | 3 +-- src/c++/perf_analyzer/test_command_line_parser.cc | 15 +++++++-------- 6 files changed, 19 insertions(+), 36 deletions(-) diff --git a/src/c++/perf_analyzer/command_line_parser.cc b/src/c++/perf_analyzer/command_line_parser.cc index 25b5f9af5..5a73d5927 100644 --- a/src/c++/perf_analyzer/command_line_parser.cc +++ b/src/c++/perf_analyzer/command_line_parser.cc @@ -334,8 +334,8 @@ CLParser::Usage(const std::string& msg) "--request-parameter : Specifies a custom " "parameter that can be sent to a Triton backend as part of the " "request. For example, providing '--request-parameter " - "max_tokens:256:uint' to the command line will set an additional " - "parameter 'max_tokens' of type 'uint' to 256 as part of the " + "max_tokens:256:int' to the command line will set an additional " + "parameter 'max_tokens' of type 'int' to 256 as part of the " "request. The --request-parameter may be specified multiple times " "for different custom parameters.", 18) @@ -1538,7 +1538,7 @@ CLParser::ParseCommandLine(int argc, char** argv) break; } case 59: { - params_->using_periodic_concurrency_range = true; + params_->is_using_periodic_concurrency_mode = true; std::string arg = optarg; std::vector values{SplitString(arg)}; if (values.size() < 2) { @@ -1605,9 +1605,6 @@ CLParser::ParseCommandLine(int argc, char** argv) if (type == "bool") { param.type = RequestParameterType::BOOL; param.bool_value = value == "true" ? true : false; - } else if (type == "uint") { - param.type = RequestParameterType::UINT; - param.uint_value = std::stoull(value); } else if (type == "int") { param.type = RequestParameterType::INT; param.int_value = std::stoll(value); @@ -1780,7 +1777,7 @@ CLParser::VerifyOptions() } std::vector load_modes{ - params_->using_periodic_concurrency_range, + params_->is_using_periodic_concurrency_mode, params_->using_concurrency_range, params_->using_request_rate_range, params_->using_custom_intervals}; if (std::count(load_modes.begin(), load_modes.end(), true) > 1) { @@ -1791,13 +1788,13 @@ CLParser::VerifyOptions() "--request-intervals."); } - if (params_->using_periodic_concurrency_range && !params_->streaming) { + if (params_->is_using_periodic_concurrency_mode && !params_->streaming) { Usage( "The --periodic-concurrency-range option requires bi-directional gRPC " "streaming."); } - if (params_->using_periodic_concurrency_range && + if (params_->is_using_periodic_concurrency_mode && (params_->profile_export_file == "")) { Usage( "Must provide --profile-export-file when using the " diff --git a/src/c++/perf_analyzer/command_line_parser.h b/src/c++/perf_analyzer/command_line_parser.h index 2dfcf8d43..518e7b2cf 100644 --- a/src/c++/perf_analyzer/command_line_parser.h +++ b/src/c++/perf_analyzer/command_line_parser.h @@ -58,9 +58,6 @@ struct PerfAnalyzerParameters { uint64_t measurement_window_ms = 5000; bool using_concurrency_range = false; Range concurrency_range{1, 1, 1}; - bool using_periodic_concurrency_range = false; - Range periodic_concurrency_range{1, 1, 1}; - uint64_t request_period = 10; std::unordered_map request_parameters; uint64_t latency_threshold_ms = NO_LIMIT; double stability_threshold = 0.1; @@ -155,9 +152,8 @@ struct PerfAnalyzerParameters { std::string profile_export_file{""}; bool is_using_periodic_concurrency_mode{false}; - Range periodic_concurrency_range{1, 1, 1}; - uint64_t periodic_concurrency_request_period{10}; + uint64_t request_period{10}; }; using PAParamsPtr = std::shared_ptr; diff --git a/src/c++/perf_analyzer/docs/cli.md b/src/c++/perf_analyzer/docs/cli.md index 6e5dd953e..5961224c8 100644 --- a/src/c++/perf_analyzer/docs/cli.md +++ b/src/c++/perf_analyzer/docs/cli.md @@ -207,12 +207,12 @@ Default value is `10`. #### `--request-parameter=` Specifies a custom parameter that can be sent to a Triton backend as part of -the request. For example, providing '--request-parameter max_tokens:256:uint' +the request. For example, providing '--request-parameter max_tokens:256:int' to the command line will set an additional parameter 'max_tokens' of type -'uint' to 256 as part of the request. The --request-parameter may be specified +'int' to 256 as part of the request. The --request-parameter may be specified multiple times for different custom parameters. -Valid `type` values are: `bool`, `int`, `uint`, and `string`. +Valid `type` values are: `bool`, `int`, and `string`. > **NOTE** > diff --git a/src/c++/perf_analyzer/perf_analyzer.cc b/src/c++/perf_analyzer/perf_analyzer.cc index 44ec520f2..c3e5e5f90 100644 --- a/src/c++/perf_analyzer/perf_analyzer.cc +++ b/src/c++/perf_analyzer/perf_analyzer.cc @@ -160,13 +160,6 @@ PerfAnalyzer::CreateAnalyzerObjects() } std::unique_ptr manager; - params_->is_using_periodic_concurrency_mode = true; - params_->periodic_concurrency_range = { - std::stoi(std::getenv("MY_START")), std::stoi(std::getenv("MY_END")), - std::stoi(std::getenv("MY_STEP"))}; - params_->periodic_concurrency_request_period = - std::stoi(std::getenv("MY_REQUEST_PERIOD")); - if (params_->targeting_concurrency()) { if ((parser_->SchedulerType() == pa::ModelParser::SEQUENCE) || (parser_->SchedulerType() == pa::ModelParser::ENSEMBLE_SEQUENCE)) { @@ -221,8 +214,7 @@ PerfAnalyzer::CreateAnalyzerObjects() params_->async, params_->streaming, params_->batch_size, params_->max_threads, params_->max_concurrency, params_->shared_memory_type, params_->output_shm_size, parser_, factory, - params_->periodic_concurrency_range, - params_->periodic_concurrency_request_period); + params_->periodic_concurrency_range, params_->request_period); } else if (params_->using_request_rate_range) { if ((params_->sequence_id_range != 0) && (params_->sequence_id_range < params_->num_of_sequences)) { diff --git a/src/c++/perf_analyzer/perf_utils.h b/src/c++/perf_analyzer/perf_utils.h index 887d38077..7be051a68 100644 --- a/src/c++/perf_analyzer/perf_utils.h +++ b/src/c++/perf_analyzer/perf_utils.h @@ -83,12 +83,11 @@ class Range { T step; }; -enum RequestParameterType { STRING = 0, INT = 1, UINT = 2, BOOL = 3 }; +enum RequestParameterType { STRING = 0, INT = 1, BOOL = 3 }; struct RequestParameter { std::string str_value; int64_t int_value; - uint64_t uint_value; bool bool_value; RequestParameterType type; }; diff --git a/src/c++/perf_analyzer/test_command_line_parser.cc b/src/c++/perf_analyzer/test_command_line_parser.cc index 2e666e9a2..bdf819ff8 100644 --- a/src/c++/perf_analyzer/test_command_line_parser.cc +++ b/src/c++/perf_analyzer/test_command_line_parser.cc @@ -176,8 +176,8 @@ CHECK_PARAMS(PAParamsPtr act, PAParamsPtr exp) CHECK(act->mpi_driver != nullptr); CHECK_STRING(act->memory_type, exp->memory_type); CHECK( - act->using_periodic_concurrency_range == - exp->using_periodic_concurrency_range); + act->is_using_periodic_concurrency_mode == + exp->is_using_periodic_concurrency_mode); CHECK( act->periodic_concurrency_range.start == exp->periodic_concurrency_range.start); @@ -200,8 +200,6 @@ CHECK_PARAMS(PAParamsPtr act, PAParamsPtr exp) CHECK(act_param.second.str_value == exp_param->second.str_value); } else if (act_param.second.type == RequestParameterType::INT) { CHECK(act_param.second.int_value == exp_param->second.int_value); - } else if (act_param.second.type == RequestParameterType::UINT) { - CHECK(act_param.second.uint_value == exp_param->second.uint_value); } else if (act_param.second.type == RequestParameterType::BOOL) { CHECK(act_param.second.bool_value == exp_param->second.bool_value); } @@ -1232,6 +1230,7 @@ TEST_CASE("Testing Command Line Parser") exp->async = true; exp->streaming = true; exp->url = "localhost:8001"; // gRPC url + exp->max_threads = 4; // not targeting concurrency SUBCASE("start provided") { @@ -1254,7 +1253,7 @@ TEST_CASE("Testing Command Line Parser") } CheckValidRange( - args, option_name, parser, act, exp->using_periodic_concurrency_range, + args, option_name, parser, act, exp->is_using_periodic_concurrency_mode, exp->periodic_concurrency_range); CheckInvalidRange(args, option_name, parser, act, check_params); @@ -1384,7 +1383,7 @@ TEST_CASE("Testing Command Line Parser") SUBCASE("valid parameter") { args.push_back(option_name); - args.push_back("max_tokens:256:uint"); + args.push_back("max_tokens:256:int"); int argc = args.size(); char* argv[argc]; @@ -1394,8 +1393,8 @@ TEST_CASE("Testing Command Line Parser") CHECK(!parser.UsageCalled()); RequestParameter param; - param.uint_value = 256; - param.type = RequestParameterType::UINT; + param.int_value = 256; + param.type = RequestParameterType::INT; exp->request_parameters["max_tokens"] = param; } From 040e0adc3ec5b137a75ad9c0c929bea36c7c3d1f Mon Sep 17 00:00:00 2001 From: Hyunjae Woo Date: Tue, 26 Sep 2023 17:39:11 -0700 Subject: [PATCH 22/24] Change doc --- src/c++/perf_analyzer/docs/cli.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/c++/perf_analyzer/docs/cli.md b/src/c++/perf_analyzer/docs/cli.md index 5961224c8..bb50c6a71 100644 --- a/src/c++/perf_analyzer/docs/cli.md +++ b/src/c++/perf_analyzer/docs/cli.md @@ -209,7 +209,7 @@ Default value is `10`. Specifies a custom parameter that can be sent to a Triton backend as part of the request. For example, providing '--request-parameter max_tokens:256:int' to the command line will set an additional parameter 'max_tokens' of type -'int' to 256 as part of the request. The --request-parameter may be specified +'int64' to 256 as part of the request. The --request-parameter may be specified multiple times for different custom parameters. Valid `type` values are: `bool`, `int`, and `string`. From 67e1b02f2ff98b233a90dabef92ad9b641466968 Mon Sep 17 00:00:00 2001 From: Hyunjae Woo Date: Tue, 26 Sep 2023 17:40:15 -0700 Subject: [PATCH 23/24] Minor fix --- src/c++/perf_analyzer/perf_utils.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/c++/perf_analyzer/perf_utils.h b/src/c++/perf_analyzer/perf_utils.h index 7be051a68..0871de42c 100644 --- a/src/c++/perf_analyzer/perf_utils.h +++ b/src/c++/perf_analyzer/perf_utils.h @@ -83,7 +83,7 @@ class Range { T step; }; -enum RequestParameterType { STRING = 0, INT = 1, BOOL = 3 }; +enum RequestParameterType { STRING = 0, INT = 1, BOOL = 2 }; struct RequestParameter { std::string str_value; From d5b0c94432c2be032d489d6d849b675bd6a28d72 Mon Sep 17 00:00:00 2001 From: Hyunjae Woo Date: Wed, 27 Sep 2023 12:03:25 -0700 Subject: [PATCH 24/24] Address feedback --- src/c++/perf_analyzer/docs/cli.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/c++/perf_analyzer/docs/cli.md b/src/c++/perf_analyzer/docs/cli.md index bb50c6a71..5961224c8 100644 --- a/src/c++/perf_analyzer/docs/cli.md +++ b/src/c++/perf_analyzer/docs/cli.md @@ -209,7 +209,7 @@ Default value is `10`. Specifies a custom parameter that can be sent to a Triton backend as part of the request. For example, providing '--request-parameter max_tokens:256:int' to the command line will set an additional parameter 'max_tokens' of type -'int64' to 256 as part of the request. The --request-parameter may be specified +'int' to 256 as part of the request. The --request-parameter may be specified multiple times for different custom parameters. Valid `type` values are: `bool`, `int`, and `string`.