-
Notifications
You must be signed in to change notification settings - Fork 3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[QNN EP] Session option for graph optimization (#18262)
### Description Adds the QNN session option `htp_graph_finalization_optimization_mode` to enable QNN graph optimizations at the expense of longer preparation time. ### Motivation and Context Allow enabling QNN graph optimizations per app/model.
- Loading branch information
1 parent
c8def0c
commit a0eeeaf
Showing
15 changed files
with
270 additions
and
37 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
43 changes: 43 additions & 0 deletions
43
onnxruntime/core/providers/qnn/builder/qnn_graph_configs_helper.cc
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT License. | ||
|
|
||
| #include "core/providers/qnn/builder/qnn_graph_configs_helper.h" | ||
|
|
||
| #include "HTP/QnnHtpGraph.h" | ||
|
|
||
| namespace onnxruntime { | ||
| namespace qnn { | ||
|
|
||
| const QnnGraph_Config_t** QnnGraphConfigsBuilder::GetQnnGraphConfigs() { | ||
| if (graph_config_ptrs_.empty()) { | ||
| return nullptr; | ||
| } | ||
|
|
||
| if (!IsNullTerminated()) { | ||
| graph_config_ptrs_.push_back(nullptr); | ||
| } | ||
|
|
||
| return graph_config_ptrs_.data(); | ||
| } | ||
|
|
||
| QnnHtpGraph_CustomConfig_t& QnnGraphConfigsBuilder::PushHtpGraphCustomConfig() { | ||
| htp_custom_graph_configs_.push_back(QNN_HTP_GRAPH_CUSTOM_CONFIG_INIT); | ||
| return htp_custom_graph_configs_.back(); | ||
| } | ||
|
|
||
| QnnGraph_Config_t& QnnGraphConfigsBuilder::PushGraphConfig() { | ||
| graph_configs_.push_back(QNN_GRAPH_CONFIG_INIT); | ||
| QnnGraph_Config_t& config = graph_configs_.back(); | ||
|
|
||
| // Add pointer to this new graph config to the list of graph config pointers. | ||
| if (IsNullTerminated()) { | ||
| graph_config_ptrs_.back() = &config; // Replace last nullptr entry. | ||
| } else { | ||
| graph_config_ptrs_.push_back(&config); | ||
| } | ||
|
|
||
| return config; | ||
| } | ||
|
|
||
| } // namespace qnn | ||
| } // namespace onnxruntime |
56 changes: 56 additions & 0 deletions
56
onnxruntime/core/providers/qnn/builder/qnn_graph_configs_helper.h
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,56 @@ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT License. | ||
|
|
||
| #pragma once | ||
|
|
||
| #include <core/common/inlined_containers_fwd.h> | ||
|
|
||
| #include "HTP/QnnHtpGraph.h" | ||
|
|
||
| namespace onnxruntime { | ||
| namespace qnn { | ||
|
|
||
| /** | ||
| * Helper class for building a null-terminated list of QNN Graph configurations. | ||
| * A QNN configuration consists of multiple objects with references to each other. This | ||
| * class ensures that all configuration objects have the same lifetime, so that they remain valid | ||
| * across the call to graphCreate(). | ||
| */ | ||
| class QnnGraphConfigsBuilder { | ||
| public: | ||
| /** | ||
| * Returns a pointer to the beginning of a null-terminated array of QNN Graph configurations. | ||
| * This result is passed QNN's graphCreate() API. | ||
| * | ||
| * \return Pointer to null-terminated QnnGraph_Config_t* array. | ||
| */ | ||
| const QnnGraph_Config_t** GetQnnGraphConfigs(); | ||
|
|
||
| /** | ||
| * Creates and returns a reference to a new HTP graph configuration object. The object is initialized to | ||
| * the QNN recommended default value. The caller is meant to override fields in this object. | ||
| * | ||
| * \return A reference to a default QnnHtpGraph_CustomConfig_t object. | ||
| */ | ||
| QnnHtpGraph_CustomConfig_t& PushHtpGraphCustomConfig(); | ||
|
|
||
| /** | ||
| * Creates and returns a reference to a new graph configuration object. The object is initialized to | ||
| * the QNN recommended default value. The caller is meant to override fields in this object. | ||
| * | ||
| * \return A reference to a default QnnGraph_Config_t object. | ||
| */ | ||
| QnnGraph_Config_t& PushGraphConfig(); | ||
|
|
||
| private: | ||
| bool IsNullTerminated() const { | ||
| return !graph_config_ptrs_.empty() && graph_config_ptrs_.back() == nullptr; | ||
| } | ||
|
|
||
| InlinedVector<QnnHtpGraph_CustomConfig_t> htp_custom_graph_configs_; | ||
| InlinedVector<QnnGraph_Config_t> graph_configs_; | ||
| InlinedVector<const QnnGraph_Config_t*> graph_config_ptrs_; | ||
| }; | ||
|
|
||
| } // namespace qnn | ||
| } // namespace onnxruntime |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.