Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[TensorRT EP] Update deprecated TRT api #18834

Merged
merged 4 commits into from
Dec 18, 2023
Merged

[TensorRT EP] Update deprecated TRT api #18834

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6b55002
update api
yf711 Dec 15, 2023
7c42222
adjustment for upcoming trt 10
yf711 Dec 15, 2023
b36c62c
fix
yf711 Dec 15, 2023
fb922dc
lint
yf711 Dec 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 18 additions & 7 deletions onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2506,7 +2506,7 @@
auto trt_config = std::unique_ptr<nvinfer1::IBuilderConfig>(trt_builder->createBuilderConfig());
auto trt_parser = tensorrt_ptr::unique_pointer<nvonnxparser::IParser>(nvonnxparser::createParser(*trt_network, trt_logger));
trt_parser->parse(string_buf.data(), string_buf.size(), model_path_);
trt_config->setMaxWorkspaceSize(max_workspace_size_);
trt_config->setMemoryPoolLimit(nvinfer1::MemoryPoolType::kWORKSPACE, max_workspace_size_);

// Force Pow + Reduce ops in layer norm to run in FP32 to avoid overflow
if (fp16_enable_ && layer_norm_fp32_fallback_) {
Expand Down Expand Up @@ -2723,13 +2723,24 @@
trt_config->setFlag(nvinfer1::BuilderFlag::kSPARSE_WEIGHTS);
LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] Sparse weights are allowed";
}

// enable builder heuristics
#if NV_TENSORRT_MAJOR == 8 && NV_TENSORRT_MINOR == 5
if (build_heuristics_enable_) {
trt_config->setFlag(nvinfer1::BuilderFlag::kENABLE_TACTIC_HEURISTIC);
LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] Builder heuristics are enabled";
LOGS_DEFAULT(WARNING) << "[TensorRT EP] Builder heuristics are enabled."
<< " For TRT > 8.5, trt_build_heuristics_enable is deprecated, please set builder optimization level as 2 to enable builder heuristics.";

Check warning on line 2730 in onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc

View workflow job for this annotation

GitHub Actions / cpplint

[cpplint] onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc#L2730 

 Lines should be <= 120 characters long  [whitespace/line_length] [2]
Raw output 
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc:2730:  Lines should be <= 120 characters long  [whitespace/line_length] [2]
}
#if NV_TENSORRT_MINOR > 5 && NV_TENSORRT_MAJOR >= 8
#elif NV_TENSORRT_MAJOR == 8 && NV_TENSORRT_MINOR > 5 || NV_TENSORRT_MAJOR > 8
// for TRT 8.6 onwards, heuristic-based tactic option is automatically enabled by setting builder optimization level 2

Check warning on line 2733 in onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc

View workflow job for this annotation

GitHub Actions / cpplint

[cpplint] onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc#L2733 

 Lines should be <= 120 characters long  [whitespace/line_length] [2]
Raw output 
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc:2733:  Lines should be <= 120 characters long  [whitespace/line_length] [2]
if (build_heuristics_enable_) {
if (builder_optimization_level_ == 2) {
LOGS_DEFAULT(WARNING) << "[TensorRT EP] Builder heuristics are automatically enabled by builder optimization level 2. trt_build_heuristics_enable is deprecated on TRT 8.6 onwards.";

Check warning on line 2736 in onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc

View workflow job for this annotation

GitHub Actions / cpplint

[cpplint] onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc#L2736 

 Lines should be <= 120 characters long  [whitespace/line_length] [2]
Raw output 
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc:2736:  Lines should be <= 120 characters long  [whitespace/line_length] [2]
} else {
LOGS_DEFAULT(WARNING) << "[TensorRT EP] trt_build_heuristics_enable is deprecated on TRT 8.6 onwards. Please set builder optimization level as 2 to enable builder heuristics.";

Check warning on line 2738 in onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc

View workflow job for this annotation

GitHub Actions / cpplint

[cpplint] onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc#L2738 

 Lines should be <= 120 characters long  [whitespace/line_length] [2]
Raw output 
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc:2738:  Lines should be <= 120 characters long  [whitespace/line_length] [2]
}
}
#endif

#if NV_TENSORRT_MAJOR == 8 && NV_TENSORRT_MINOR > 5 || NV_TENSORRT_MAJOR > 8
// switch optimizaion level
if (builder_optimization_level_ != 3) {
trt_config->setBuilderOptimizationLevel(builder_optimization_level_);
Expand Down Expand Up @@ -3125,7 +3136,7 @@
trt_state->context->reset();
trt_state->engine->reset();
auto trt_config = std::unique_ptr<nvinfer1::IBuilderConfig>(trt_builder->createBuilderConfig());
trt_config->setMaxWorkspaceSize(*(trt_state->max_workspace_size_ptr));
trt_config->setMemoryPoolLimit(nvinfer1::MemoryPoolType::kWORKSPACE, *(trt_state->max_workspace_size_ptr));
for (auto trt_profile : trt_profiles) {
trt_config->addOptimizationProfile(trt_profile);
}
Expand Down Expand Up @@ -3166,7 +3177,7 @@
trt_config->setFlag(nvinfer1::BuilderFlag::kENABLE_TACTIC_HEURISTIC);
LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] Builder heuristics are enabled";
}
#if NV_TENSORRT_MINOR > 5 && NV_TENSORRT_MAJOR >= 8
#if NV_TENSORRT_MAJOR == 8 && NV_TENSORRT_MINOR > 5 || NV_TENSORRT_MAJOR > 8
// switch optimizaion level
if (trt_state->builder_optimization_level != 3) {
trt_config->setBuilderOptimizationLevel(trt_state->builder_optimization_level);
Expand Down
Loading