Periodic concurrency mode

src/c++/library/common.h

@@ -38,6 +38,7 @@
 #include <mutex>
 #include <string>
 #include <thread>
+#include <unordered_map>
 #include <vector>
 
 #ifdef TRITON_INFERENCE_SERVER_CLIENT_CLASS
@@ -153,6 +154,12 @@ class InferenceServerClient {
   InferStat infer_stat_;
 };
 
+struct RequestParameter {
+  std::string name;
+  std::string value;
+  std::string type;
+};
+
 //==============================================================================
 /// Structure to hold options for Inference Request.
 ///
@@ -221,6 +228,8 @@ struct InferOptions {
   uint64_t client_timeout_;
   /// Whether to tell Triton to enable an empty final response.
   bool triton_enable_empty_final_response_;
+  /// Additional parameters to pass to the model
+  std::unordered_map<std::string, RequestParameter> request_parameters;
 };
 
 //==============================================================================

src/c++/library/grpc_client.cc

@@ -35,6 +35,7 @@
 #include <iostream>
 #include <mutex>
 #include <sstream>
+#include <string>
 
 #include "common.h"
 
@@ -1408,6 +1409,23 @@ InferenceServerGrpcClient::PreRunProcessing(
         options.server_timeout_);
   }
 
+
+  for (auto& param : options.request_parameters) {
+    if (param.second.type == "string") {
+      (*infer_request_.mutable_parameters())[param.first].set_string_param(
+          param.second.value);
+    } else if (param.second.type == "int") {
+      (*infer_request_.mutable_parameters())[param.first].set_int64_param(
+          std::stoi(param.second.value));
+    } else if (param.second.type == "bool") {
+      bool val = false;
+      if (param.second.value == "true") {
+        val = true;
+      }
+      (*infer_request_.mutable_parameters())[param.first].set_bool_param(val);
+    }
+  }
+
   int index = 0;
   infer_request_.mutable_raw_input_contents()->Clear();
   for (const auto input : inputs) {

src/c++/perf_analyzer/CMakeLists.txt

@@ -69,6 +69,8 @@ set(
   sequence_manager.cc
   profile_data_collector.cc
   profile_data_exporter.cc
+  periodic_concurrency_manager.cc
+  periodic_concurrency_worker.cc
 )
 
 set(
@@ -109,6 +111,8 @@ set(
   request_record.h
   profile_data_collector.h
   profile_data_exporter.h
+  periodic_concurrency_manager.h
+  periodic_concurrency_worker.h
 )
 
 add_executable(

src/c++/perf_analyzer/client_backend/client_backend.h

@@ -192,6 +192,15 @@ struct ModelStatistics {
   uint64_t cache_miss_time_ns_;
 };
 
+///
+/// Structure to hold Request parameter data for Inference Request.
+///
+struct RequestParameter {
+  std::string name;
+  std::string value;
+  std::string type;
+};
+
 //==============================================================================
 /// Structure to hold options for Inference Request.
 ///
@@ -230,6 +239,9 @@ struct InferOptions {
   bool sequence_end_;
   /// Whether to tell Triton to enable an empty final response.
   bool triton_enable_empty_final_response_;
+
+  /// Additional parameters to pass to the model
+  std::unordered_map<std::string, RequestParameter> request_parameters_;
 };
 
 struct SslOptionsBase {

src/c++/perf_analyzer/client_backend/triton/triton_client_backend.cc

@@ -590,6 +590,14 @@ TritonClientBackend::ParseInferOptionsToTriton(
   }
   triton_options->triton_enable_empty_final_response_ =
       options.triton_enable_empty_final_response_;
+
+  for (auto& map_entry : options.request_parameters_) {
+    auto rp = tc::RequestParameter();
+    rp.name = map_entry.second.name;
+    rp.value = map_entry.second.value;
+    rp.type = map_entry.second.type;
+    triton_options->request_parameters[map_entry.first] = rp;
+  }
 }