From 8e6019af2ebcca0bbb1da425bc882e74a3843015 Mon Sep 17 00:00:00 2001 From: Adrian Lizarraga Date: Tue, 3 Oct 2023 16:24:33 -0700 Subject: [PATCH] [QNN EP] Enable QNN Saver for debugging issues (#17747) ### Description - Enables option to use the QNN Saver backend for dumping QNN API calls to file. - Adds logic to read environment variable `ORT_UNIT_TEST_ENABLE_QNN_SAVER` from QNN EP unit tests. If enabled, unit tests will use the QNN Saver backend and dump files to `./saver_output/`. ### Motivation and Context QNN Saver makes it easier to debug issues when unit tests fail. The output files generated by QNN Saver can be used to replay the exact QNN API calls that lead to a specific error condition. QNN Saver dumps QNN API calls (and weights) to disk. - saver_output/saver_output.c: C file containing all QNN API calls. - saver_output/params.bin: binary file containing all input/output/parameter tensor data provided during tensor creation, op config validation, and graph execution. Enabling the QNN Saver backend has 2 note-worthy effects: 1. All QNN API calls will succeed. 2. Inference output returns dummy data. Because the output files from QNN Saver are always overwritten, it is recommended to run individual unit tests via the `--gtest_filter` command-line option. Example (linux): ```shell $ ORT_UNIT_TEST_ENABLE_QNN_SAVER=1 ./onnxruntime_test_all --gtest_filter=QnnHTPBackendTests.Resize_DownSample_Linear_AlignCorners ``` --- .../core/session/onnxruntime_c_api.h | 3 + .../qnn/builder/qnn_backend_manager.cc | 200 ++++++++++++------ .../qnn/builder/qnn_backend_manager.h | 21 +- .../providers/qnn/qnn_execution_provider.cc | 24 ++- .../providers/qnn/qnn_execution_provider.h | 1 - onnxruntime/test/onnx/main.cc | 3 + .../test/providers/qnn/qnn_basic_test.cc | 45 +++- .../test/providers/qnn/qnn_test_utils.cc | 19 +- .../test/providers/qnn/qnn_test_utils.h | 24 ++- 9 files changed, 259 insertions(+), 81 deletions(-) diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index 4b911e3482e90..486e2ff2b90a2 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -3597,6 +3597,9 @@ struct OrtApi { * "rpc_control_latency": QNN RPC control latency. * "htp_performance_mode": QNN performance mode, options: "burst", "balanced", "default", "high_performance", * "high_power_saver", "low_balanced", "low_power_saver", "power_saver", "sustained_high_performance". Default to "default". + * "qnn_saver_path": File path to the QNN Saver backend library. If specified, QNN Saver will be enabled and will + * dump QNN API calls to disk for replay/debugging. QNN Saver produces incorrect model inference results and + * may alter model/EP partitioning. Use only for debugging. * * SNPE supported keys: * "runtime": SNPE runtime engine, options: "CPU", "CPU_FLOAT32", "GPU", "GPU_FLOAT32_16_HYBRID", "GPU_FLOAT16", diff --git a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc index 8e31124ce4c85..e2083371acca4 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc +++ b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc @@ -30,12 +30,20 @@ typedef Qnn_ErrorHandle_t (*QnnSystemInterfaceGetProvidersFn_t)(const QnnSystemI constexpr const char* QNN_PROVIDER = "ORTQNNEP"; +static Qnn_Version_t GetQnnInterfaceApiVersion(const QnnInterface_t* qnn_interface) { + return qnn_interface->apiVersion.coreApiVersion; +} + +static Qnn_Version_t GetQnnInterfaceApiVersion(const QnnSystemInterface_t* qnn_interface) { + return qnn_interface->systemApiVersion; +} + template -Status QnnBackendManager::GetQnnInterfaceProviders(const char* lib_path, - const char* interface_provider_name, - void** backend_lib_handle, - T*** interface_providers, - uint32_t& num_providers) { +Status QnnBackendManager::GetQnnInterfaceProvider(const char* lib_path, + const char* interface_provider_name, + void** backend_lib_handle, + Qnn_Version_t req_version, + T** interface_provider) { std::string error_msg; *backend_lib_handle = LoadLib(lib_path, static_cast(DlOpenFlag::DL_NOW) | static_cast(DlOpenFlag::DL_LOCAL), @@ -47,10 +55,36 @@ Status QnnBackendManager::GetQnnInterfaceProviders(const char* lib_path, GetInterfaceProviders = ResolveSymbol(*backend_lib_handle, interface_provider_name, *logger_); ORT_RETURN_IF(nullptr == GetInterfaceProviders, "Failed to get QNN providers!"); - auto result = GetInterfaceProviders((const T***)interface_providers, &num_providers); + T** interface_providers{nullptr}; + uint32_t num_providers{0}; + + auto result = GetInterfaceProviders((const T***)&interface_providers, &num_providers); ORT_RETURN_IF((QNN_SUCCESS != result || nullptr == *interface_providers || 0 == num_providers), "Failed to get QNN providers."); + bool found_valid_interface{false}; + for (size_t pIdx = 0; pIdx < num_providers; pIdx++) { + Qnn_Version_t interface_version = GetQnnInterfaceApiVersion(interface_providers[pIdx]); + + LOGS_DEFAULT(VERBOSE) << lib_path << " interface version: " << interface_version.major << "." + << interface_version.minor << "." << interface_version.patch; + + // Check the interface's API version against the required version. + // Major versions must match. The interface's minor version must be greater OR equal with a suitable patch version. + if (interface_version.major == req_version.major) { + bool minor_and_patch_version_ok = (interface_version.minor > req_version.minor) || + (interface_version.minor == req_version.minor && + interface_version.patch >= req_version.patch); + if (minor_and_patch_version_ok) { + found_valid_interface = true; + *interface_provider = interface_providers[pIdx]; + break; + } + } + } + + ORT_RETURN_IF_NOT(found_valid_interface, "Unable to find a valid interface for ", lib_path); + return Status::OK(); } @@ -76,38 +110,89 @@ void QnnBackendManager::SetQnnBackendType(uint32_t backend_id) { } Status QnnBackendManager::LoadBackend() { - QnnInterface_t** interface_providers{nullptr}; - uint32_t num_providers{0}; - auto rt = GetQnnInterfaceProviders(backend_path_.c_str(), - "QnnInterface_getProviders", - &backend_lib_handle_, - &interface_providers, - num_providers); + QnnInterface_t* backend_interface_provider{nullptr}; + auto rt = GetQnnInterfaceProvider(backend_path_.c_str(), + "QnnInterface_getProviders", + &backend_lib_handle_, + {QNN_API_VERSION_MAJOR, + QNN_API_VERSION_MINOR, + QNN_API_VERSION_PATCH}, + &backend_interface_provider); ORT_RETURN_IF_ERROR(rt); + qnn_interface_ = backend_interface_provider->QNN_INTERFACE_VER_NAME; + auto backend_id = backend_interface_provider->backendId; + SetQnnBackendType(backend_id); - bool found_valid_interface{false}; - LOGS_DEFAULT(VERBOSE) << "QNN_API_VERSION_MAJOR: " << QNN_API_VERSION_MAJOR - << " QNN_API_VERSION_MINOR: " << QNN_API_VERSION_MINOR; - for (size_t pIdx = 0; pIdx < num_providers; pIdx++) { - LOGS_DEFAULT(VERBOSE) << "interface_providers major: " << interface_providers[pIdx]->apiVersion.coreApiVersion.major - << " interface_providers minor: " << interface_providers[pIdx]->apiVersion.coreApiVersion.minor; - if (QNN_API_VERSION_MAJOR == interface_providers[pIdx]->apiVersion.coreApiVersion.major && - QNN_API_VERSION_MINOR <= interface_providers[pIdx]->apiVersion.coreApiVersion.minor) { - found_valid_interface = true; - qnn_interface_ = interface_providers[pIdx]->QNN_INTERFACE_VER_NAME; - auto backend_id = interface_providers[pIdx]->backendId; - SetQnnBackendType(backend_id); - - LOGS_DEFAULT(INFO) << "Found valid interface, version: " << QNN_API_VERSION_MAJOR - << "." << QNN_API_VERSION_MINOR - << " backend provider name: " << interface_providers[pIdx]->providerName - << " backend id: " << backend_id; - break; + Qnn_Version_t backend_interface_version = GetQnnInterfaceApiVersion(backend_interface_provider); + LOGS_DEFAULT(INFO) << "Found valid interface, version: " << backend_interface_version.major + << "." << backend_interface_version.minor << "." << backend_interface_version.patch + << " backend provider name: " << backend_interface_provider->providerName + << " backend id: " << backend_id; + + return Status::OK(); +} + +// Loads the intended backend (e.g., HTP, CPU, etc) to get its type, and then +// sets QNN Saver as the active backend. QNN op builders will still see the intended backend (e.g., HTP) +// as the backend type to ensure they emit the expected QNN API calls. +// +// QNN Saver is a "debugging" backend that serializes all QNN API calls (and weights) into local files. +// This information can be used to debug issues by replaying QNN API calls with another backend. +Status QnnBackendManager::LoadQnnSaverBackend() { + void* backend_lib_handle = nullptr; + + // Helper that unloads the intended backend library handle when the `unload_backend_lib` variable + // goes out of scope. Similar to `defer` in other languages. + auto unload_backend_lib = gsl::finally([&] { + if (backend_lib_handle != nullptr) { + auto result = UnloadLib(backend_lib_handle); + if (Status::OK() != result) { + ORT_THROW("Failed to unload backend library."); + } } - } + }); + + // Load the intended backend (e.g., HTP, CPU) to ensure it is valid and to get its type. + QnnInterface_t* backend_interface_provider{nullptr}; + auto rt = GetQnnInterfaceProvider(backend_path_.c_str(), + "QnnInterface_getProviders", + &backend_lib_handle, + {QNN_API_VERSION_MAJOR, + QNN_API_VERSION_MINOR, + QNN_API_VERSION_PATCH}, + &backend_interface_provider); + ORT_RETURN_IF_ERROR(rt); - ORT_RETURN_IF_NOT(found_valid_interface, "Unable to find a valid interface."); + // Set the "intended" backend type so that QNN builders still make the expected QNN API calls. + auto backend_id = backend_interface_provider->backendId; + SetQnnBackendType(backend_id); + + // Load the QNN Saver backend and set it as the activate backend. + QnnInterface_t* saver_interface_provider{nullptr}; + auto saver_rt = GetQnnInterfaceProvider(qnn_saver_path_.c_str(), + "QnnInterface_getProviders", + &backend_lib_handle_, // NOTE: QNN Saver library handle is set + {QNN_API_VERSION_MAJOR, + QNN_API_VERSION_MINOR, + QNN_API_VERSION_PATCH}, + &saver_interface_provider); + ORT_RETURN_IF_ERROR(saver_rt); + qnn_interface_ = saver_interface_provider->QNN_INTERFACE_VER_NAME; // NOTE: QNN Saver will provide the interfaces + + Qnn_Version_t backend_interface_version = GetQnnInterfaceApiVersion(backend_interface_provider); + Qnn_Version_t saver_interface_version = GetQnnInterfaceApiVersion(saver_interface_provider); + + LOGS_DEFAULT(INFO) << "Using QNN Saver version: " << saver_interface_version.major << "." + << saver_interface_version.minor << "." << saver_interface_version.patch + << " provider name : " << saver_interface_provider->providerName; + + LOGS_DEFAULT(INFO) << "Intended backend provider name: " << backend_interface_provider->providerName + << " backend id: " << backend_id + << " interface version: " << backend_interface_version.major + << "." << backend_interface_version.minor << "." << backend_interface_version.patch; return Status::OK(); } @@ -120,34 +205,22 @@ Status QnnBackendManager::LoadQnnSystemLib() { #endif // #ifdef _WIN32 std::filesystem::path lib_file_path(backend_path_.c_str()); std::string sys_file_path(lib_file_path.remove_filename().string() + system_lib_file); - QnnSystemInterface_t** system_interface_providers{nullptr}; - uint32_t num_providers = 0; - auto rt = GetQnnInterfaceProviders(sys_file_path.c_str(), - "QnnSystemInterface_getProviders", - &system_lib_handle_, - &system_interface_providers, - num_providers); + QnnSystemInterface_t* system_interface_provider{nullptr}; + auto rt = GetQnnInterfaceProvider(sys_file_path.c_str(), + "QnnSystemInterface_getProviders", + &system_lib_handle_, + {QNN_SYSTEM_API_VERSION_MAJOR, + QNN_SYSTEM_API_VERSION_MINOR, + QNN_SYSTEM_API_VERSION_PATCH}, + &system_interface_provider); ORT_RETURN_IF_ERROR(rt); + Qnn_Version_t system_interface_version = GetQnnInterfaceApiVersion(system_interface_provider); + qnn_sys_interface_ = system_interface_provider->QNN_SYSTEM_INTERFACE_VER_NAME; - bool found_valid_interface{false}; - for (size_t pIdx = 0; pIdx < num_providers; pIdx++) { - LOGS_DEFAULT(VERBOSE) << "system_interface_providers major: " << system_interface_providers[pIdx]->systemApiVersion.major - << " system_interface_providers minor: " << system_interface_providers[pIdx]->systemApiVersion.minor; - int64_t systems_version_major = static_cast(system_interface_providers[pIdx]->systemApiVersion.major); - int64_t systems_version_minor = static_cast(system_interface_providers[pIdx]->systemApiVersion.minor); - if (systems_version_major == QNN_SYSTEM_API_VERSION_MAJOR && - systems_version_minor >= QNN_SYSTEM_API_VERSION_MINOR) { - found_valid_interface = true; - qnn_sys_interface_ = system_interface_providers[pIdx]->QNN_SYSTEM_INTERFACE_VER_NAME; - LOGS_DEFAULT(INFO) << "Found valid system interface, version: " << QNN_API_VERSION_MAJOR - << "." << QNN_API_VERSION_MINOR - << " backend provider name: " << system_interface_providers[pIdx]->providerName; - break; - } - } - - ORT_RETURN_IF_NOT(found_valid_interface, "Unable to find a valid system interface."); + LOGS_DEFAULT(INFO) << "Found valid system interface, version: " << system_interface_version.major + << "." << system_interface_version.minor + << " backend provider name: " << system_interface_provider->providerName; return Status::OK(); } @@ -643,7 +716,12 @@ Status QnnBackendManager::SetupBackend(const logging::Logger& logger, bool load_ return Status::OK(); } - ORT_RETURN_IF_ERROR(LoadBackend()); + if (qnn_saver_path_.empty()) { + ORT_RETURN_IF_ERROR(LoadBackend()); + } else { + ORT_RETURN_IF_ERROR(LoadQnnSaverBackend()); + } + LOGS(logger, VERBOSE) << "LoadBackend succeed."; if (load_from_cached_context) { diff --git a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h index 4ca63a042c103..402f842c7a4bf 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h +++ b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h @@ -25,14 +25,16 @@ class QnnModel; class QnnBackendManager { public: - QnnBackendManager(std::string backend_path, + QnnBackendManager(std::string&& backend_path, ProfilingLevel profiling_level, uint32_t rpc_control_latency, - HtpPerformanceMode htp_performance_mode) + HtpPerformanceMode htp_performance_mode, + std::string&& qnn_saver_path) : backend_path_(backend_path), profiling_level_(profiling_level), rpc_control_latency_(rpc_control_latency), - htp_performance_mode_(htp_performance_mode) { + htp_performance_mode_(htp_performance_mode), + qnn_saver_path_(qnn_saver_path) { } ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(QnnBackendManager); @@ -140,6 +142,8 @@ class QnnBackendManager { Status LoadQnnSystemLib(); + Status LoadQnnSaverBackend(); + Status UnloadLib(void* handle); void* LibFunction(void* handle, const char* symbol, std::string& error_msg); @@ -155,11 +159,11 @@ class QnnBackendManager { } template - Status GetQnnInterfaceProviders(const char* lib_path, - const char* interface_provider_name, - void** backend_lib_handle, - T*** interface_providers, - uint32_t& num_providers); + Status GetQnnInterfaceProvider(const char* lib_path, + const char* interface_provider_name, + void** backend_lib_handle, + Qnn_Version_t req_version, + T** interface_provider); bool IsDevicePropertySupported(); @@ -210,6 +214,7 @@ class QnnBackendManager { #ifdef _WIN32 std::set mod_handles_; #endif + const std::string qnn_saver_path_; }; } // namespace qnn diff --git a/onnxruntime/core/providers/qnn/qnn_execution_provider.cc b/onnxruntime/core/providers/qnn/qnn_execution_provider.cc index 7bbfe807da0f2..ec5316eb13ce1 100644 --- a/onnxruntime/core/providers/qnn/qnn_execution_provider.cc +++ b/onnxruntime/core/providers/qnn/qnn_execution_provider.cc @@ -104,9 +104,10 @@ QNNExecutionProvider::QNNExecutionProvider(const ProviderOptions& provider_optio static const std::string BACKEND_PATH = "backend_path"; auto backend_path_pos = runtime_options_.find(BACKEND_PATH); + std::string backend_path; if (backend_path_pos != runtime_options_.end()) { - backend_path_ = backend_path_pos->second; - LOGS_DEFAULT(VERBOSE) << "Backend path: " << backend_path_; + backend_path = backend_path_pos->second; + LOGS_DEFAULT(VERBOSE) << "Backend path: " << backend_path; } else { LOGS_DEFAULT(ERROR) << "No backend path provided."; } @@ -131,10 +132,21 @@ QNNExecutionProvider::QNNExecutionProvider(const ProviderOptions& provider_optio ParseHtpPerformanceMode(htp_performance_mode_pos->second); } - qnn_backend_manager_ = std::make_unique(backend_path_, - profiling_level_, - rpc_control_latency_, - htp_performance_mode_); + // Enable use of QNN Saver if the user provides a path the QNN Saver backend library. + static const std::string QNN_SAVER_PATH_KEY = "qnn_saver_path"; + std::string qnn_saver_path; + auto qnn_saver_path_pos = runtime_options_.find(QNN_SAVER_PATH_KEY); + if (qnn_saver_path_pos != runtime_options_.end()) { + qnn_saver_path = qnn_saver_path_pos->second; + LOGS_DEFAULT(VERBOSE) << "User specified QNN Saver path: " << qnn_saver_path; + } + + qnn_backend_manager_ = std::make_unique( + std::move(backend_path), + profiling_level_, + rpc_control_latency_, + htp_performance_mode_, + std::move(qnn_saver_path)); } bool QNNExecutionProvider::IsNodeSupported(qnn::QnnModelWrapper& qnn_model_wrapper, const NodeUnit& node_unit, diff --git a/onnxruntime/core/providers/qnn/qnn_execution_provider.h b/onnxruntime/core/providers/qnn/qnn_execution_provider.h index 2fe507b70a6ab..3827e2044e2b1 100644 --- a/onnxruntime/core/providers/qnn/qnn_execution_provider.h +++ b/onnxruntime/core/providers/qnn/qnn_execution_provider.h @@ -58,7 +58,6 @@ class QNNExecutionProvider : public IExecutionProvider { private: ProviderOptions runtime_options_; - std::string backend_path_; qnn::ProfilingLevel profiling_level_ = qnn::ProfilingLevel::OFF; qnn::HtpPerformanceMode htp_performance_mode_ = qnn::HtpPerformanceMode::kHtpDefault; std::unique_ptr qnn_backend_manager_; diff --git a/onnxruntime/test/onnx/main.cc b/onnxruntime/test/onnx/main.cc index 062ca4ece86bf..287d657a2ce28 100644 --- a/onnxruntime/test/onnx/main.cc +++ b/onnxruntime/test/onnx/main.cc @@ -56,6 +56,7 @@ void usage() { "\t [QNN only] [rpc_control_latency]: QNN rpc control latency. default to 10.\n" "\t [QNN only] [htp_performance_mode]: QNN performance mode, options: 'burst', 'balanced', 'default', 'high_performance', \n" "\t 'high_power_saver', 'low_balanced', 'low_power_saver', 'power_saver', 'sustained_high_performance'. Default to 'default'. \n" + "\t [QNN only] [qnn_saver_path]: QNN Saver backend path. e.g '/folderpath/libQnnSaver.so'.\n" "\t [Usage]: -e -i '| |' \n\n" "\t [Example] [For QNN EP] -e qnn -i \"profiling_level|detailed backend_path|/folderpath/libQnnCpu.so\" \n\n" "\t [SNPE only] [runtime]: SNPE runtime, options: 'CPU', 'GPU', 'GPU_FLOAT16', 'DSP', 'AIP_FIXED_TF'. \n" @@ -477,6 +478,8 @@ int real_main(int argc, char* argv[], Ort::Env& env) { std::string str = str_stream.str(); ORT_THROW("Wrong value for htp_performance_mode. select from: " + str); } + } else if (key == "qnn_saver_path") { + // no validation } else { ORT_THROW(R"(Wrong key type entered. Choose from options: ['backend_path', 'qnn_context_cache_enable', 'qnn_context_cache_path', 'profiling_level', 'rpc_control_latency', 'htp_performance_mode'])"); diff --git a/onnxruntime/test/providers/qnn/qnn_basic_test.cc b/onnxruntime/test/providers/qnn/qnn_basic_test.cc index a441e828c0cc6..5f63813d8d84e 100644 --- a/onnxruntime/test/providers/qnn/qnn_basic_test.cc +++ b/onnxruntime/test/providers/qnn/qnn_basic_test.cc @@ -2,6 +2,7 @@ // Licensed under the MIT License. #include +#include #include "core/session/onnxruntime_cxx_api.h" #include "core/session/onnxruntime_session_options_config_keys.h" @@ -172,7 +173,7 @@ TEST(QnnEP, TestDisableCPUFallback_ConflictingConfig) { // The models passed to this function are subgraphs extracted from a larger model that exhibited // shape inferencing issues on QNN. Thus, the models are expected to have a specific input/output // types and shapes. -static void RunNHWCResizeModel(const ORTCHAR_T* ort_model_path, bool use_htp) { +static void RunNHWCResizeModel(const ORTCHAR_T* ort_model_path, bool use_htp, bool enable_qnn_saver = false) { Ort::SessionOptions so; // Ensure all type/shape inference warnings result in errors! @@ -183,8 +184,14 @@ static void RunNHWCResizeModel(const ORTCHAR_T* ort_model_path, bool use_htp) { #if defined(_WIN32) options["backend_path"] = use_htp ? "QnnHtp.dll" : "QnnCpu.dll"; + if (enable_qnn_saver) { + options["qnn_saver_path"] = "QnnSaver.dll"; + } #else options["backend_path"] = use_htp ? "libQnnHtp.so" : "libQnnCpu.so"; + if (enable_qnn_saver) { + options["qnn_saver_path"] = "libQnnSaver.so"; + } #endif so.AppendExecutionProvider("QNN", options); @@ -226,7 +233,7 @@ static void RunNHWCResizeModel(const ORTCHAR_T* ort_model_path, bool use_htp) { auto typeshape = ort_output.GetTensorTypeAndShapeInfo(); std::vector output_shape = typeshape.GetShape(); - ASSERT_THAT(output_shape, ::testing::ElementsAre(1, 6, 7, 10)); + EXPECT_THAT(output_shape, ::testing::ElementsAre(1, 6, 7, 10)); } // Test shape inference of NHWC Resize operator (opset 11) that uses @@ -253,6 +260,23 @@ TEST_F(QnnCPUBackendTests, TestNHWCResizeShapeInference_sizes_opset18) { RunNHWCResizeModel(ORT_MODEL_FOLDER "nhwc_resize_sizes_opset18.onnx", false); } +// Test that QNN Saver generates the expected files for a model meant to run on the QNN CPU backend. +TEST_F(QnnCPUBackendTests, QnnSaver_OutputFiles) { + const std::filesystem::path qnn_saver_output_dir = "saver_output"; + + // Remove pre-existing QNN Saver output files. Note that fs::remove_all() can handle non-existing paths. + std::filesystem::remove_all(qnn_saver_output_dir); + ASSERT_FALSE(std::filesystem::exists(qnn_saver_output_dir)); + + RunNHWCResizeModel(ORT_MODEL_FOLDER "nhwc_resize_sizes_opset18.onnx", + false, // use_htp + true); // enable_qnn_saver + + // Check that QNN Saver output files exist. + EXPECT_TRUE(std::filesystem::exists(qnn_saver_output_dir / "saver_output.c")); + EXPECT_TRUE(std::filesystem::exists(qnn_saver_output_dir / "params.bin")); +} + #if defined(__aarch64__) || defined(_M_ARM64) || defined(__linux__) // Test shape inference of QDQ NHWC Resize operator (opset 18) that uses @@ -261,6 +285,23 @@ TEST_F(QnnHTPBackendTests, TestNHWCResizeShapeInference_qdq_sizes_opset18) { RunNHWCResizeModel(ORT_MODEL_FOLDER "nhwc_resize_sizes_opset18.quant.onnx", true); } +// Test that QNN Saver generates the expected files for a model meant to run on the QNN HTP backend. +TEST_F(QnnHTPBackendTests, QnnSaver_OutputFiles) { + const std::filesystem::path qnn_saver_output_dir = "saver_output"; + + // Remove pre-existing QNN Saver output files. Note that fs::remove_all() can handle non-existing paths. + std::filesystem::remove_all(qnn_saver_output_dir); + ASSERT_FALSE(std::filesystem::exists(qnn_saver_output_dir)); + + RunNHWCResizeModel(ORT_MODEL_FOLDER "nhwc_resize_sizes_opset18.onnx", + true, // use_htp + true); // enable_qnn_saver + + // Check that QNN Saver output files exist. + EXPECT_TRUE(std::filesystem::exists(qnn_saver_output_dir / "saver_output.c")); + EXPECT_TRUE(std::filesystem::exists(qnn_saver_output_dir / "params.bin")); +} + #endif // defined(__aarch64__) || defined(_M_ARM64) || defined(__linux__) #endif // !defined(ORT_MINIMAL_BUILD) diff --git a/onnxruntime/test/providers/qnn/qnn_test_utils.cc b/onnxruntime/test/providers/qnn/qnn_test_utils.cc index 51df93f8853ec..a067c9c53e57a 100644 --- a/onnxruntime/test/providers/qnn/qnn_test_utils.cc +++ b/onnxruntime/test/providers/qnn/qnn_test_utils.cc @@ -9,6 +9,7 @@ #include "test/util/include/default_providers.h" #include "test/util/include/test/test_environment.h" +#include "core/platform/env_var_utils.h" #include "core/common/span_utils.h" #include "core/framework/compute_capability.h" #include "core/graph/graph.h" @@ -41,7 +42,22 @@ std::vector GetFloatDataInRange(float min_val, float max_val, size_t num_ return data; } -void RunQnnModelTest(const GetTestModelFn& build_test_case, const ProviderOptions& provider_options, +void TryEnableQNNSaver(ProviderOptions& qnn_options) { + // Allow dumping QNN API calls to file by setting an environment variable that enables the QNN Saver backend. + constexpr auto kEnableQNNSaverEnvironmentVariableName = "ORT_UNIT_TEST_ENABLE_QNN_SAVER"; + static std::optional enable_qnn_saver = onnxruntime::ParseEnvironmentVariable( + kEnableQNNSaverEnvironmentVariableName); + + if (enable_qnn_saver.has_value() && *enable_qnn_saver != 0) { +#if defined(_WIN32) + qnn_options["qnn_saver_path"] = "QnnSaver.dll"; +#else + qnn_options["qnn_saver_path"] = "libQnnSaver.so"; +#endif // defined(_WIN32) + } +} + +void RunQnnModelTest(const GetTestModelFn& build_test_case, ProviderOptions provider_options, int opset_version, ExpectedEPNodeAssignment expected_ep_assignment, float fp32_abs_err, logging::Severity log_severity) { EPVerificationParams verification_params; @@ -65,6 +81,7 @@ void RunQnnModelTest(const GetTestModelFn& build_test_case, const ProviderOption // Serialize the model to a string. std::string model_data; model.ToProto().SerializeToString(&model_data); + TryEnableQNNSaver(provider_options); RunAndVerifyOutputsWithEP(AsByteSpan(model_data.data(), model_data.size()), "QNN_EP_TestLogID", QnnExecutionProviderWithOptions(provider_options), helper.feeds_, verification_params); diff --git a/onnxruntime/test/providers/qnn/qnn_test_utils.h b/onnxruntime/test/providers/qnn/qnn_test_utils.h index 14c62f98f6a3e..b4c84d893c828 100644 --- a/onnxruntime/test/providers/qnn/qnn_test_utils.h +++ b/onnxruntime/test/providers/qnn/qnn_test_utils.h @@ -220,6 +220,25 @@ void InferenceModel(const std::string& model_data, const char* log_id, ExpectedEPNodeAssignment expected_ep_assignment, const NameMLValMap& feeds, std::vector& output_vals); +/** + * If the ORT_UNIT_TEST_ENABLE_QNN_SAVER environment variable is enabled (set to 1), this function modifies + * the QNN EP provider options to enable the QNN Saver backend, which dumps QNN API calls (and weights) to disk. + * + * - saver_output/saver_output.c: C file containing all QNN API calls. + * - saver_output/params.bin: binary file containing all input/output/parameter tensor data provided during tensor + * creation, op config validation, and graph execution. + * + * Enabling the QNN Saver backend has 2 note-worthy effects: + * 1. All QNN API calls will succeed. + * 2. Inference output returns dummy data. + * + * Because output files from QNN Saver are always overwritten, it is recommended to run individual unit tests via the + * --gtest_filter command-line option. Ex: --gtest_filter=QnnHTPBackendTests.Resize_DownSample_Linear_AlignCorners + * + * \param qnn_options QNN EP provider options that may be modified to enable QNN Saver. + */ +void TryEnableQNNSaver(ProviderOptions& qnn_options); + /** * Tests the accuracy of a QDQ model on QNN EP by runnning 3 inferences: * @@ -240,7 +259,7 @@ void InferenceModel(const std::string& model_data, const char* log_id, */ template inline void TestQDQModelAccuracy(const GetTestModelFn& f32_model_fn, const GetTestQDQModelFn& qdq_model_fn, - const ProviderOptions& qnn_options, int opset_version, + ProviderOptions qnn_options, int opset_version, ExpectedEPNodeAssignment expected_ep_assignment, float fp32_abs_err = 1e-4f, logging::Severity log_severity = logging::Severity::kERROR) { // Add kMSDomain to cover contrib op like Gelu @@ -300,6 +319,7 @@ inline void TestQDQModelAccuracy(const GetTestModelFn& f32_model_fn, const GetTe qdq_model.ToProto().SerializeToString(&qdq_model_data); // Run QDQ model on QNN EP and collect outputs. + TryEnableQNNSaver(qnn_options); std::vector qnn_qdq_outputs; InferenceModel(qdq_model_data, "qdq_model_logger", QnnExecutionProviderWithOptions(qnn_options), expected_ep_assignment, qdq_helper.feeds_, qnn_qdq_outputs); @@ -538,7 +558,7 @@ inline GetTestQDQModelFn BuildQDQOpTestCase(const std::string& op_typ * \param fp32_abs_err The acceptable error between CPU EP and QNN EP. * \param log_severity The logger's minimum severity level. */ -void RunQnnModelTest(const GetTestModelFn& build_test_case, const ProviderOptions& provider_options, +void RunQnnModelTest(const GetTestModelFn& build_test_case, ProviderOptions provider_options, int opset_version, ExpectedEPNodeAssignment expected_ep_assignment, float fp32_abs_err = 1e-5f, logging::Severity log_severity = logging::Severity::kERROR);