From 0f6efdcae7de08f09e8a78ea2799af65b636d3bc Mon Sep 17 00:00:00 2001 From: yf711 Date: Tue, 30 Jul 2024 16:40:28 -0700 Subject: [PATCH 1/5] check if trt_engine invalid --- .../core/providers/tensorrt/tensorrt_execution_provider.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc index cdbb7bb2a8094..7178963fb5acd 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc @@ -3672,6 +3672,10 @@ Status TensorrtExecutionProvider::CreateNodeComputeInfoFromGraph(const GraphView return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, "TensorRT EP Failed to Build Engine."); } trt_engine = trt_state->engine->get(); + // Check before using trt_engine + if (trt_engine == nullptr) { + return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, "Invalid TensorRT engine, please rebuild."); + } if (trt_state->engine_cache_enable) { // Serialize engine profile SerializeProfileV2(profile_cache_path, shape_ranges); From cfd44a7c17a02bb2c21bdd025b69900c53b99a98 Mon Sep 17 00:00:00 2001 From: yf711 Date: Tue, 30 Jul 2024 16:56:36 -0700 Subject: [PATCH 2/5] update --- .../core/providers/tensorrt/tensorrt_execution_provider.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc index 7178963fb5acd..2df461174338e 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc @@ -3756,6 +3756,11 @@ Status TensorrtExecutionProvider::CreateNodeComputeInfoFromGraph(const GraphView trt_context = trt_state->context->get(); } + // Check before using trt_engine + if (trt_engine == nullptr) { + return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, "Invalid TensorRT engine, please rebuild."); + } + // Get input and output binding names int total_bindings = trt_engine->getNbIOTensors(); std::vector input_binding_names, output_binding_names; From ad3dd4681ff0729dd66619c5dea1eaaf99776d32 Mon Sep 17 00:00:00 2001 From: yf711 Date: Mon, 5 Aug 2024 14:26:05 -0700 Subject: [PATCH 3/5] remove redundant --- .../core/providers/tensorrt/tensorrt_execution_provider.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc index 2df461174338e..5f52604548b18 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc @@ -3672,10 +3672,6 @@ Status TensorrtExecutionProvider::CreateNodeComputeInfoFromGraph(const GraphView return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, "TensorRT EP Failed to Build Engine."); } trt_engine = trt_state->engine->get(); - // Check before using trt_engine - if (trt_engine == nullptr) { - return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, "Invalid TensorRT engine, please rebuild."); - } if (trt_state->engine_cache_enable) { // Serialize engine profile SerializeProfileV2(profile_cache_path, shape_ranges); From 785d350f6c4285dd221be2624e8bfe02628d5f2f Mon Sep 17 00:00:00 2001 From: yf711 Date: Mon, 5 Aug 2024 15:35:08 -0700 Subject: [PATCH 4/5] lint --- .../core/providers/tensorrt/tensorrt_execution_provider.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc index 5f52604548b18..0fb527499be36 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc @@ -3756,7 +3756,7 @@ Status TensorrtExecutionProvider::CreateNodeComputeInfoFromGraph(const GraphView if (trt_engine == nullptr) { return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, "Invalid TensorRT engine, please rebuild."); } - + // Get input and output binding names int total_bindings = trt_engine->getNbIOTensors(); std::vector input_binding_names, output_binding_names; From ac19d4efd3d18f088d3c18815d282fe368b61da6 Mon Sep 17 00:00:00 2001 From: yf711 Date: Tue, 6 Aug 2024 15:16:59 -0700 Subject: [PATCH 5/5] comments applied --- .../core/providers/tensorrt/tensorrt_execution_provider.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc index 0fb527499be36..0f32b58314466 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc @@ -3754,7 +3754,7 @@ Status TensorrtExecutionProvider::CreateNodeComputeInfoFromGraph(const GraphView // Check before using trt_engine if (trt_engine == nullptr) { - return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, "Invalid TensorRT engine, please rebuild."); + return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, "No engine is found."); } // Get input and output binding names @@ -4080,6 +4080,11 @@ Status TensorrtExecutionProvider::CreateNodeComputeInfoFromPrecompiledEngine(con Ort::ThrowOnError(api->KernelContext_GetGPUComputeStream(context, &cuda_stream)); cudaStream_t stream = static_cast(cuda_stream); + // Check before using trt_engine + if (trt_engine == nullptr) { + return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, "No engine is found."); + } + // Get input and output binding names int total_bindings = trt_engine->getNbIOTensors(); std::vector input_binding_names, output_binding_names;