From f42fee48b09b7889b3107fc00911aa4f8df1cd79 Mon Sep 17 00:00:00 2001 From: Lei Cao Date: Tue, 2 Jan 2024 12:34:50 -0800 Subject: [PATCH 01/14] make GenerateMetaDefId a standalone function, decouple it from EP --- .../core/framework/execution_provider.h | 54 +++++++++---------- .../core/framework/execution_provider.cc | 18 ++----- .../providers/cann/cann_execution_provider.cc | 2 +- .../coreml/coreml_execution_provider.cc | 2 +- .../providers/dnnl/dnnl_execution_provider.cc | 4 +- .../migraphx/migraphx_execution_provider.cc | 2 +- .../nnapi_builtin/nnapi_execution_provider.cc | 2 +- .../providers/qnn/qnn_execution_provider.cc | 2 +- .../provider_bridge_provider.cc | 4 -- .../shared_library/provider_interfaces.h | 2 - .../webnn/webnn_execution_provider.cc | 2 +- .../core/session/provider_bridge_ort.cc | 4 -- .../test/framework/execution_provider_test.cc | 4 +- onnxruntime/test/framework/tunable_op_test.cc | 2 +- .../internal_testing_execution_provider.cc | 4 +- 15 files changed, 42 insertions(+), 66 deletions(-) diff --git a/include/onnxruntime/core/framework/execution_provider.h b/include/onnxruntime/core/framework/execution_provider.h index ea4f52f99649d..999b15eb642cc 100644 --- a/include/onnxruntime/core/framework/execution_provider.h +++ b/include/onnxruntime/core/framework/execution_provider.h @@ -59,14 +59,11 @@ enum class DataLayout { class IExecutionProvider { protected: - IExecutionProvider(const std::string& type, bool use_metadef_id_creator = false) - : IExecutionProvider(type, OrtDevice(), use_metadef_id_creator) {} + IExecutionProvider(const std::string& type) + : IExecutionProvider(type, OrtDevice()) {} - IExecutionProvider(const std::string& type, OrtDevice device, bool use_metadef_id_creator = false) + IExecutionProvider(const std::string& type, OrtDevice device) : default_device_(device), type_{type} { - if (use_metadef_id_creator) { - metadef_id_generator_ = std::make_unique(); - } } /* @@ -274,19 +271,6 @@ class IExecutionProvider { return logger_; } - /** Generate a unique id that can be used in a MetaDef name. Values are unique for a model instance. - The model hash is also returned if you wish to include that in the MetaDef name to ensure uniqueness across models. - @param graph_viewer[in] Graph viewer that GetCapability was called with. Can be for the main graph or nested graph. - @param model_hash[out] Returns the hash for the main (i.e. top level) graph in the model. - This is created using the model path if available, - or the model input names and the output names from all nodes in the main graph. - @remarks e.g. the TensorRT Execution Provider is used in multiple sessions and the underlying infrastructure caches - compiled kernels, so the name must be unique and deterministic across models and sessions. - NOTE: Ideally this would be a protected method, but to work across the EP bridge it has to be public and - virtual, and ModelMetadefIdGenerator but be defined in the header as well. - */ - virtual int GenerateMetaDefId(const onnxruntime::GraphViewer& graph_viewer, HashValue& model_hash) const; - virtual std::unique_ptr GetProfiler() { return {}; } @@ -331,18 +315,28 @@ class IExecutionProvider { // It will be set when this object is registered to a session const logging::Logger* logger_ = nullptr; +}; - // helper to generate ids that are unique to model and deterministic, even if the execution provider is shared across - // multiple sessions. - class ModelMetadefIdGenerator { - public: - int GenerateId(const onnxruntime::GraphViewer& graph_viewer, HashValue& model_hash); - - private: - std::unordered_map main_graph_hash_; // map graph instance hash to model contents hash - std::unordered_map model_metadef_id_; // current unique id for model - }; +// helper to generate ids that are unique to model and deterministic, even if the execution provider is shared across +// multiple sessions. +class ModelMetadefIdGenerator { + public: + /** Generate a unique id that can be used in a MetaDef name. Values are unique for a model instance. + The model hash is also returned if you wish to include that in the MetaDef name to ensure uniqueness across models. + @param graph_viewer[in] Graph viewer that GetCapability was called with. Can be for the main graph or nested graph. + @param model_hash[out] Returns the hash for the main (i.e. top level) graph in the model. + This is created using the model path if available, + or the model input names and the output names from all nodes in the main graph. + @remarks e.g. the TensorRT Execution Provider is used in multiple sessions and the underlying infrastructure caches + compiled kernels, so the name must be unique and deterministic across models and sessions. + NOTE: Ideally this would be a protected method, but to work across the EP bridge it has to be public and + virtual, and ModelMetadefIdGenerator but be defined in the header as well. + */ + static int GenerateMetaDefId(const onnxruntime::GraphViewer& graph_viewer, HashValue& model_hash); - std::unique_ptr metadef_id_generator_; + private: + inline static std::unordered_map main_graph_hash_; // map graph instance hash to model contents hash + inline static std::unordered_map model_metadef_id_; // current unique id for model }; + } // namespace onnxruntime diff --git a/onnxruntime/core/framework/execution_provider.cc b/onnxruntime/core/framework/execution_provider.cc index 7f8009216ce3a..1d8666931368f 100644 --- a/onnxruntime/core/framework/execution_provider.cc +++ b/onnxruntime/core/framework/execution_provider.cc @@ -36,8 +36,11 @@ common::Status IExecutionProvider::Compile(const std::vector& #endif -int IExecutionProvider::ModelMetadefIdGenerator::GenerateId(const onnxruntime::GraphViewer& graph_viewer, - HashValue& model_hash) { +int ModelMetadefIdGenerator::GenerateMetaDefId(const onnxruntime::GraphViewer& graph_viewer, HashValue& model_hash) { + // if the EP is shared across multiple sessions there's a very small potential for concurrency issues. + // use a lock when generating an id to be paranoid + static OrtMutex mutex; + std::lock_guard lock(mutex); model_hash = 0; // find the top level graph @@ -97,15 +100,4 @@ int IExecutionProvider::ModelMetadefIdGenerator::GenerateId(const onnxruntime::G return model_metadef_id_[model_hash]++; } -int IExecutionProvider::GenerateMetaDefId(const onnxruntime::GraphViewer& graph_viewer, HashValue& model_hash) const { - ORT_ENFORCE(metadef_id_generator_, - "IExecutionProvider constructor must be called with true for use_metadef_id_creator"); - - // if the EP is shared across multiple sessions there's a very small potential for concurrency issues. - // use a lock when generating an id to be paranoid - static OrtMutex mutex; - std::lock_guard lock(mutex); - return metadef_id_generator_->GenerateId(graph_viewer, model_hash); -} - } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cann/cann_execution_provider.cc b/onnxruntime/core/providers/cann/cann_execution_provider.cc index 127c37bd84d0f..eba4e27186ee9 100644 --- a/onnxruntime/core/providers/cann/cann_execution_provider.cc +++ b/onnxruntime/core/providers/cann/cann_execution_provider.cc @@ -1197,7 +1197,7 @@ std::unique_ptr CANNExecutionProvider::GetSubGraph( // Generate unique kernel name for CANN subgraph HashValue model_hash = 0; - int id = GenerateMetaDefId(graph_viewer, model_hash); + int id = ModelMetadefIdGenerator::GenerateMetaDefId(graph_viewer, model_hash); auto meta_def = IndexedSubGraph_MetaDef::Create(); meta_def->name() = graph_viewer.Name() + "_" + std::to_string(model_hash) + "_" + std::to_string(id); diff --git a/onnxruntime/core/providers/coreml/coreml_execution_provider.cc b/onnxruntime/core/providers/coreml/coreml_execution_provider.cc index c9973671ffa28..16ee537d979e0 100644 --- a/onnxruntime/core/providers/coreml/coreml_execution_provider.cc +++ b/onnxruntime/core/providers/coreml/coreml_execution_provider.cc @@ -54,7 +54,7 @@ CoreMLExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_vie const auto gen_metadef_name = [&]() { HashValue model_hash; - int metadef_id = GenerateMetaDefId(graph_viewer, model_hash); + int metadef_id = ModelMetadefIdGenerator::GenerateMetaDefId(graph_viewer, model_hash); return MakeString(COREML, "_", model_hash, "_", metadef_id); }; diff --git a/onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc b/onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc index 05eb0091a8c83..d50a31d222830 100644 --- a/onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc +++ b/onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc @@ -229,7 +229,7 @@ std::vector> DnnlExecutionProvider::GetCapabi // Assign inputs and outputs to subgraph's meta_def HashValue model_hash; - int metadef_id = GenerateMetaDefId(graph_viewer, model_hash); + int metadef_id = ModelMetadefIdGenerator::GenerateMetaDefId(graph_viewer, model_hash); auto meta_def = ::onnxruntime::IndexedSubGraph_MetaDef::Create(); meta_def->name() = "DNNL_" + std::to_string(model_hash) + "_" + std::to_string(metadef_id); meta_def->domain() = kMSDomain; @@ -264,7 +264,7 @@ std::vector> DnnlExecutionProvider::GetCapabi graph_viewer.ToProto(*model_proto->mutable_graph(), false, true); model_proto->set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION); HashValue model_hash; - int metadef_id = GenerateMetaDefId(graph_viewer, model_hash); + int metadef_id = ModelMetadefIdGenerator::GenerateMetaDefId(graph_viewer, model_hash); std::fstream dump("DNNL_" + std::to_string(model_hash) + "_" + std::to_string(metadef_id) + ".onnx", std::ios::out | std::ios::trunc | std::ios::binary); model_proto->SerializeToOstream(dump); } diff --git a/onnxruntime/core/providers/migraphx/migraphx_execution_provider.cc b/onnxruntime/core/providers/migraphx/migraphx_execution_provider.cc index 8bfa66710e2fc..32055d526ab3a 100644 --- a/onnxruntime/core/providers/migraphx/migraphx_execution_provider.cc +++ b/onnxruntime/core/providers/migraphx/migraphx_execution_provider.cc @@ -757,7 +757,7 @@ std::unique_ptr MIGraphXExecutionProvider::GetSubGraph(const st // Generate unique kernel name for MIGraphX subgraph uint64_t model_hash = 0; - int id = GenerateMetaDefId(graph, model_hash); + int id = ModelMetadefIdGenerator::GenerateMetaDefId(graph, model_hash); std::string subgraph_id = std::to_string(model_hash) + "_" + std::to_string(id); auto meta_def = IndexedSubGraph_MetaDef::Create(); const std::string graph_type = graph.IsSubgraph() ? "subgraph" : "graph"; diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc index 727917ad9232e..aad4290d19057 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc @@ -176,7 +176,7 @@ NnapiExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_view const auto gen_metadef_name = [&]() { HashValue model_hash; - int metadef_id = GenerateMetaDefId(graph_viewer, model_hash); + int metadef_id = ModelMetadefIdGenerator::GenerateMetaDefId(graph_viewer, model_hash); return MakeString(NNAPI, "_", model_hash, "_", metadef_id); }; diff --git a/onnxruntime/core/providers/qnn/qnn_execution_provider.cc b/onnxruntime/core/providers/qnn/qnn_execution_provider.cc index c72012fd4a19b..bb8867f6fc9ff 100644 --- a/onnxruntime/core/providers/qnn/qnn_execution_provider.cc +++ b/onnxruntime/core/providers/qnn/qnn_execution_provider.cc @@ -396,7 +396,7 @@ QNNExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_viewer const auto gen_metadef_name = [&]() { uint64_t model_hash; - int metadef_id = GenerateMetaDefId(graph_viewer, model_hash); + int metadef_id = ModelMetadefIdGenerator::GenerateMetaDefId(graph_viewer, model_hash); return MakeString(QNN, "_", model_hash, "_", metadef_id); }; diff --git a/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc b/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc index a3155fe6b86cf..a16d2890b47c2 100644 --- a/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc +++ b/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc @@ -329,10 +329,6 @@ common::Status IExecutionProvider::Compile(const std::vector& return g_host->IExecutionProvider__Compile(this, fused_nodes_and_graphs, node_compute_funcs); } -int IExecutionProvider::GenerateMetaDefId(const onnxruntime::GraphViewer& graph_viewer, HashValue& model_hash) const { - return g_host->IExecutionProvider__GenerateMetaDefId(this, graph_viewer, model_hash); -} - #ifdef USE_TENSORRT std::unique_ptr CreateCUDAAllocator(int16_t device_id, const char* name) { return g_host->CreateCUDAAllocator(device_id, name); diff --git a/onnxruntime/core/providers/shared_library/provider_interfaces.h b/onnxruntime/core/providers/shared_library/provider_interfaces.h index 27226005a9c0b..528c1449b524d 100644 --- a/onnxruntime/core/providers/shared_library/provider_interfaces.h +++ b/onnxruntime/core/providers/shared_library/provider_interfaces.h @@ -227,8 +227,6 @@ struct ProviderHost { virtual common::Status IExecutionProvider__Compile(IExecutionProvider* p, const std::vector& fused_nodes_and_graphs, std::vector& node_compute_funcs) = 0; - virtual int IExecutionProvider__GenerateMetaDefId(const IExecutionProvider* p, const onnxruntime::GraphViewer& graph_viewer, HashValue& model_hash) = 0; - // Status virtual std::string Status__ToString(const Status* p) = 0; diff --git a/onnxruntime/core/providers/webnn/webnn_execution_provider.cc b/onnxruntime/core/providers/webnn/webnn_execution_provider.cc index 4da54aaad3a33..876df4786b075 100644 --- a/onnxruntime/core/providers/webnn/webnn_execution_provider.cc +++ b/onnxruntime/core/providers/webnn/webnn_execution_provider.cc @@ -158,7 +158,7 @@ WebNNExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_view // Assign inputs and outputs to subgraph's meta_def. uint64_t model_hash; - int metadef_id = GenerateMetaDefId(graph_viewer, model_hash); + int metadef_id = ModelMetadefIdGenerator::GenerateMetaDefId(graph_viewer, model_hash); auto meta_def = std::make_unique<::onnxruntime::IndexedSubGraph::MetaDef>(); meta_def->name = "WEBNN_" + std::to_string(model_hash) + "_" + std::to_string(metadef_id); meta_def->domain = kMSDomain; diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index e3b8dea90a898..df2396a78d7ec 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -312,10 +312,6 @@ struct ProviderHostImpl : ProviderHost { return p->IExecutionProvider::Compile(fused_nodes_and_graphs, node_compute_funcs); } - int IExecutionProvider__GenerateMetaDefId(const IExecutionProvider* p, const onnxruntime::GraphViewer& graph_viewer, HashValue& model_hash) override { - return p->IExecutionProvider::GenerateMetaDefId(graph_viewer, model_hash); - } - // Status (direct) std::string Status__ToString(const Status* p) override { return p->Status::ToString(); } diff --git a/onnxruntime/test/framework/execution_provider_test.cc b/onnxruntime/test/framework/execution_provider_test.cc index 5a7351a766fa3..5eb8288181761 100644 --- a/onnxruntime/test/framework/execution_provider_test.cc +++ b/onnxruntime/test/framework/execution_provider_test.cc @@ -18,10 +18,10 @@ class TestEP : public IExecutionProvider { static constexpr const char* kEPType = "TestEP"; public: - TestEP() : IExecutionProvider{kEPType, true} {} + TestEP() : IExecutionProvider{kEPType} {} int GetId(const GraphViewer& viewer, HashValue& model_hash) { - return GenerateMetaDefId(viewer, model_hash); + return ModelMetadefIdGenerator::GenerateMetaDefId(viewer, model_hash); } }; diff --git a/onnxruntime/test/framework/tunable_op_test.cc b/onnxruntime/test/framework/tunable_op_test.cc index 0d9e557ebc813..61b26e35e0ddc 100644 --- a/onnxruntime/test/framework/tunable_op_test.cc +++ b/onnxruntime/test/framework/tunable_op_test.cc @@ -82,7 +82,7 @@ class TestEP : public IExecutionProvider { TestTuningContext tuning_ctx_{this}; public: - TestEP() : IExecutionProvider{kEPType, true} {} + TestEP() : IExecutionProvider{kEPType} {} ITuningContext* GetTuningContext() const override { return const_cast(&tuning_ctx_); diff --git a/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.cc b/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.cc index 957443c23e7c3..a622157d62fbb 100644 --- a/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.cc +++ b/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.cc @@ -85,7 +85,7 @@ constexpr const char* INTERNAL_TESTING_EP = "InternalTestingEP"; InternalTestingExecutionProvider::InternalTestingExecutionProvider(const std::unordered_set& ops, const std::unordered_set& stop_ops, DataLayout preferred_layout) - : IExecutionProvider{utils::kInternalTestingExecutionProvider, true}, + : IExecutionProvider{utils::kInternalTestingExecutionProvider}, ep_name_{INTERNAL_TESTING_EP}, ops_{ops}, stop_ops_{stop_ops}, @@ -212,7 +212,7 @@ InternalTestingExecutionProvider::GetCapability(const onnxruntime::GraphViewer& // create functor to generate a guaranteed unique metadef id auto generate_metadef_name = [this, &graph_viewer]() { HashValue model_hash; - int metadef_id = GenerateMetaDefId(graph_viewer, model_hash); + int metadef_id = ModelMetadefIdGenerator::GenerateMetaDefId(graph_viewer, model_hash); auto meta_def = std::make_unique<::onnxruntime::IndexedSubGraph::MetaDef>(); return ep_name_ + "_" + std::to_string(model_hash) + "_" + std::to_string(metadef_id); }; From dfadf3716c74d28e999b0aeb66923b4491ebab62 Mon Sep 17 00:00:00 2001 From: Lei Cao Date: Wed, 3 Jan 2024 17:05:53 -0800 Subject: [PATCH 02/14] fix comments to move generator function to a standalone header --- .../core/framework/execution_provider.h | 23 ------ .../core/framework/execution_provider.cc | 65 ---------------- .../framework/model_metadef_id_generator.cc | 76 +++++++++++++++++++ .../framework/model_metadef_id_generator.h | 23 ++++++ .../providers/cann/cann_execution_provider.cc | 5 +- .../coreml/coreml_execution_provider.cc | 5 +- .../providers/dnnl/dnnl_execution_provider.cc | 7 +- .../providers/js/js_execution_provider.cc | 2 +- .../migraphx/migraphx_execution_provider.cc | 5 +- .../nnapi_builtin/nnapi_execution_provider.cc | 5 +- .../core/providers/partitioning_utils.h | 2 +- .../providers/qnn/qnn_execution_provider.cc | 5 +- .../tensorrt/tensorrt_execution_provider.cc | 2 +- .../webnn/webnn_execution_provider.cc | 5 +- .../xnnpack/xnnpack_execution_provider.cc | 2 +- .../test/framework/execution_provider_test.cc | 3 +- .../internal_testing_execution_provider.cc | 3 +- 17 files changed, 129 insertions(+), 109 deletions(-) create mode 100644 onnxruntime/core/framework/model_metadef_id_generator.cc create mode 100644 onnxruntime/core/framework/model_metadef_id_generator.h diff --git a/include/onnxruntime/core/framework/execution_provider.h b/include/onnxruntime/core/framework/execution_provider.h index 999b15eb642cc..6f6d375b91729 100644 --- a/include/onnxruntime/core/framework/execution_provider.h +++ b/include/onnxruntime/core/framework/execution_provider.h @@ -316,27 +316,4 @@ class IExecutionProvider { // It will be set when this object is registered to a session const logging::Logger* logger_ = nullptr; }; - -// helper to generate ids that are unique to model and deterministic, even if the execution provider is shared across -// multiple sessions. -class ModelMetadefIdGenerator { - public: - /** Generate a unique id that can be used in a MetaDef name. Values are unique for a model instance. - The model hash is also returned if you wish to include that in the MetaDef name to ensure uniqueness across models. - @param graph_viewer[in] Graph viewer that GetCapability was called with. Can be for the main graph or nested graph. - @param model_hash[out] Returns the hash for the main (i.e. top level) graph in the model. - This is created using the model path if available, - or the model input names and the output names from all nodes in the main graph. - @remarks e.g. the TensorRT Execution Provider is used in multiple sessions and the underlying infrastructure caches - compiled kernels, so the name must be unique and deterministic across models and sessions. - NOTE: Ideally this would be a protected method, but to work across the EP bridge it has to be public and - virtual, and ModelMetadefIdGenerator but be defined in the header as well. - */ - static int GenerateMetaDefId(const onnxruntime::GraphViewer& graph_viewer, HashValue& model_hash); - - private: - inline static std::unordered_map main_graph_hash_; // map graph instance hash to model contents hash - inline static std::unordered_map model_metadef_id_; // current unique id for model -}; - } // namespace onnxruntime diff --git a/onnxruntime/core/framework/execution_provider.cc b/onnxruntime/core/framework/execution_provider.cc index 1d8666931368f..b39924d4c3ff9 100644 --- a/onnxruntime/core/framework/execution_provider.cc +++ b/onnxruntime/core/framework/execution_provider.cc @@ -35,69 +35,4 @@ common::Status IExecutionProvider::Compile(const std::vector& } #endif - -int ModelMetadefIdGenerator::GenerateMetaDefId(const onnxruntime::GraphViewer& graph_viewer, HashValue& model_hash) { - // if the EP is shared across multiple sessions there's a very small potential for concurrency issues. - // use a lock when generating an id to be paranoid - static OrtMutex mutex; - std::lock_guard lock(mutex); - model_hash = 0; - - // find the top level graph - const Graph* cur_graph = &graph_viewer.GetGraph(); - while (cur_graph->IsSubgraph()) { - cur_graph = cur_graph->ParentGraph(); - } - - uint32_t instance_hash[4] = {0, 0, 0, 0}; - - const Graph& main_graph = *cur_graph; - - // hash the bytes in the Graph instance. we can't just use the address as a new Graph instance may use - // the same memory (unit tests prove this can occur). the raw bytes of the Graph instance should be a unique - // fingerprint for the instance that can use used as the key to the hash of the model path/contents. - MurmurHash3::x86_128(&main_graph, gsl::narrow_cast(sizeof(Graph)), instance_hash[0], &instance_hash); - HashValue graph_instance_hash = instance_hash[0] | (uint64_t(instance_hash[1]) << 32); - - // if we've already hashed this main graph instance use the cached value - auto entry = main_graph_hash_.find(graph_instance_hash); - if (entry != main_graph_hash_.cend()) { - model_hash = entry->second; - } else { - uint32_t hash[4] = {0, 0, 0, 0}; - - // prefer path the model was loaded from - // this may not be available if the model was loaded from a stream or in-memory bytes - const auto& model_path_str = main_graph.ModelPath().ToPathString(); - if (!model_path_str.empty()) { - MurmurHash3::x86_128(model_path_str.data(), gsl::narrow_cast(model_path_str.size()), hash[0], &hash); - } else { - auto hash_str = [&hash](const std::string& str) { - MurmurHash3::x86_128(str.data(), gsl::narrow_cast(str.size()), hash[0], &hash); - }; - - // fingerprint the main graph by hashing graph inputs and the ordered outputs from each node - for (const auto* node_arg : main_graph.GetInputsIncludingInitializers()) { - hash_str(node_arg->Name()); - } - - // note: process nodes in order defined in model to be deterministic - for (const auto& node : main_graph.Nodes()) { - for (const auto* node_arg : node.OutputDefs()) { - if (node_arg->Exists()) { - hash_str(node_arg->Name()); - } - } - } - } - - model_hash = hash[0] | (uint64_t(hash[1]) << 32); - - main_graph_hash_[graph_instance_hash] = model_hash; - } - - // return the current unique id, and increment to update - return model_metadef_id_[model_hash]++; -} - } // namespace onnxruntime diff --git a/onnxruntime/core/framework/model_metadef_id_generator.cc b/onnxruntime/core/framework/model_metadef_id_generator.cc new file mode 100644 index 0000000000000..d4b0c39b8ff6e --- /dev/null +++ b/onnxruntime/core/framework/model_metadef_id_generator.cc @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#include +#include "model_metadef_id_generator.h" +#include "core/platform/ort_mutex.h" +#include "core/graph/graph_viewer.h" +#include "core/framework/murmurhash3.h + +namespace onnxruntime { +int GenerateMetaDefId(const onnxruntime::GraphViewer& graph_viewer, HashValue& model_hash) { + static std::unordered_map main_graph_hash_; // map graph instance hash to model contents hash + static std::unordered_map model_metadef_id_; // current unique id for model + + // if the EP is shared across multiple sessions there's a very small potential for concurrency issues. + // use a lock when generating an id to be paranoid + static OrtMutex mutex; + std::lock_guard lock(mutex); + model_hash = 0; + + // find the top level graph + const Graph* cur_graph = &graph_viewer.GetGraph(); + while (cur_graph->IsSubgraph()) { + cur_graph = cur_graph->ParentGraph(); + } + + uint32_t instance_hash[4] = {0, 0, 0, 0}; + + const Graph& main_graph = *cur_graph; + + // hash the bytes in the Graph instance. we can't just use the address as a new Graph instance may use + // the same memory (unit tests prove this can occur). the raw bytes of the Graph instance should be a unique + // fingerprint for the instance that can use used as the key to the hash of the model path/contents. + MurmurHash3::x86_128(&main_graph, gsl::narrow_cast(sizeof(Graph)), instance_hash[0], &instance_hash); + HashValue graph_instance_hash = instance_hash[0] | (uint64_t(instance_hash[1]) << 32); + + // if we've already hashed this main graph instance use the cached value + auto entry = main_graph_hash_.find(graph_instance_hash); + if (entry != main_graph_hash_.cend()) { + model_hash = entry->second; + } else { + uint32_t hash[4] = {0, 0, 0, 0}; + + // prefer path the model was loaded from + // this may not be available if the model was loaded from a stream or in-memory bytes + const auto& model_path_str = main_graph.ModelPath().ToPathString(); + if (!model_path_str.empty()) { + MurmurHash3::x86_128(model_path_str.data(), gsl::narrow_cast(model_path_str.size()), hash[0], &hash); + } else { + auto hash_str = [&hash](const std::string& str) { + MurmurHash3::x86_128(str.data(), gsl::narrow_cast(str.size()), hash[0], &hash); + }; + + // fingerprint the main graph by hashing graph inputs and the ordered outputs from each node + for (const auto* node_arg : main_graph.GetInputsIncludingInitializers()) { + hash_str(node_arg->Name()); + } + + // note: process nodes in order defined in model to be deterministic + for (const auto& node : main_graph.Nodes()) { + for (const auto* node_arg : node.OutputDefs()) { + if (node_arg->Exists()) { + hash_str(node_arg->Name()); + } + } + } + } + + model_hash = hash[0] | (uint64_t(hash[1]) << 32); + + main_graph_hash_[graph_instance_hash] = model_hash; + } + + // return the current unique id, and increment to update + return model_metadef_id_[model_hash]++; +} +} diff --git a/onnxruntime/core/framework/model_metadef_id_generator.h b/onnxruntime/core/framework/model_metadef_id_generator.h new file mode 100644 index 0000000000000..7b343f3cdaf10 --- /dev/null +++ b/onnxruntime/core/framework/model_metadef_id_generator.h @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/common/basic_types.h" +namespace onnxruntime { +class GraphViewer; + +// helper to generate ids that are unique to model and deterministic, even if the execution provider is shared across +// multiple sessions. +/** Generate a unique id that can be used in a MetaDef name. Values are unique for a model instance. + The model hash is also returned if you wish to include that in the MetaDef name to ensure uniqueness across models. + @param graph_viewer[in] Graph viewer that GetCapability was called with. Can be for the main graph or nested graph. + @param model_hash[out] Returns the hash for the main (i.e. top level) graph in the model. + This is created using the model path if available, + or the model input names and the output names from all nodes in the main graph. + @remarks e.g. the TensorRT Execution Provider is used in multiple sessions and the underlying infrastructure caches + compiled kernels, so the name must be unique and deterministic across models and sessions. + NOTE: Ideally this would be a protected method, but to work across the EP bridge it has to be public and + virtual, and ModelMetadefIdGenerator but be defined in the header as well. + */ +int GenerateMetaDefId(const onnxruntime::GraphViewer& graph_viewer, HashValue& model_hash); +} diff --git a/onnxruntime/core/providers/cann/cann_execution_provider.cc b/onnxruntime/core/providers/cann/cann_execution_provider.cc index eba4e27186ee9..8aae5bf56cadd 100644 --- a/onnxruntime/core/providers/cann/cann_execution_provider.cc +++ b/onnxruntime/core/providers/cann/cann_execution_provider.cc @@ -19,6 +19,7 @@ #include "core/providers/cann/cann_fwd.h" #include "core/providers/cann/cann_stream_handle.h" #include "core/providers/cann/npu_data_transfer.h" +#include "core/framework/model_metadef_id_generator.h" using onnxruntime::cann::BuildONNXModel; using onnxruntime::cann::CannModelPreparation; @@ -1029,7 +1030,7 @@ Status RegisterCANNKernels(KernelRegistry& kernel_registry) { } // namespace cann CANNExecutionProvider::CANNExecutionProvider(const CANNExecutionProviderInfo& info) - : IExecutionProvider{onnxruntime::kCannExecutionProvider, OrtDevice(OrtDevice::NPU, OrtDevice::MemType::DEFAULT, info.device_id), true}, info_{info} { + : IExecutionProvider{onnxruntime::kCannExecutionProvider, OrtDevice(OrtDevice::NPU, OrtDevice::MemType::DEFAULT, info.device_id)}, info_{info} { InitProviderOrtApi(); CANN_CALL_THROW(aclrtSetDevice(info_.device_id)); @@ -1197,7 +1198,7 @@ std::unique_ptr CANNExecutionProvider::GetSubGraph( // Generate unique kernel name for CANN subgraph HashValue model_hash = 0; - int id = ModelMetadefIdGenerator::GenerateMetaDefId(graph_viewer, model_hash); + int id = GenerateMetaDefId(graph_viewer, model_hash); auto meta_def = IndexedSubGraph_MetaDef::Create(); meta_def->name() = graph_viewer.Name() + "_" + std::to_string(model_hash) + "_" + std::to_string(id); diff --git a/onnxruntime/core/providers/coreml/coreml_execution_provider.cc b/onnxruntime/core/providers/coreml/coreml_execution_provider.cc index 16ee537d979e0..aa9dc1eea0420 100644 --- a/onnxruntime/core/providers/coreml/coreml_execution_provider.cc +++ b/onnxruntime/core/providers/coreml/coreml_execution_provider.cc @@ -11,6 +11,7 @@ #include "core/providers/coreml/builders/helper.h" #include "core/providers/partitioning_utils.h" #include "core/session/onnxruntime_cxx_api.h" +#include "core/framework/model_metadef_id_generator.h" #ifdef __APPLE__ #include "core/providers/coreml/builders/model_builder.h" @@ -24,7 +25,7 @@ namespace onnxruntime { constexpr const char* COREML = "CoreML"; CoreMLExecutionProvider::CoreMLExecutionProvider(uint32_t coreml_flags) - : IExecutionProvider{onnxruntime::kCoreMLExecutionProvider, true}, + : IExecutionProvider{onnxruntime::kCoreMLExecutionProvider}, coreml_flags_(coreml_flags) { } @@ -54,7 +55,7 @@ CoreMLExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_vie const auto gen_metadef_name = [&]() { HashValue model_hash; - int metadef_id = ModelMetadefIdGenerator::GenerateMetaDefId(graph_viewer, model_hash); + int metadef_id = GenerateMetaDefId(graph_viewer, model_hash); return MakeString(COREML, "_", model_hash, "_", metadef_id); }; diff --git a/onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc b/onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc index d50a31d222830..b254253f3be2c 100644 --- a/onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc +++ b/onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc @@ -20,6 +20,7 @@ #include "core/providers/dnnl/dnnl_fwd.h" #include "core/providers/dnnl/dnnl_node_capability.h" #include "core/providers/dnnl/subgraph/dnnl_subgraph_transformer.h" +#include "core/framework/model_metadef_id_generator.h" #define ORT_API_MANUAL_INIT #include "core/session/onnxruntime_cxx_api.h" @@ -30,7 +31,7 @@ constexpr const char* DNNL = "Dnnl"; constexpr const char* DNNL_CPU = "DnnlCpu"; DnnlExecutionProvider::DnnlExecutionProvider(const DnnlExecutionProviderInfo& info) - : IExecutionProvider{onnxruntime::kDnnlExecutionProvider, true}, + : IExecutionProvider{onnxruntime::kDnnlExecutionProvider}, info_(info) { InitProviderOrtApi(); @@ -229,7 +230,7 @@ std::vector> DnnlExecutionProvider::GetCapabi // Assign inputs and outputs to subgraph's meta_def HashValue model_hash; - int metadef_id = ModelMetadefIdGenerator::GenerateMetaDefId(graph_viewer, model_hash); + int metadef_id = GenerateMetaDefId(graph_viewer, model_hash); auto meta_def = ::onnxruntime::IndexedSubGraph_MetaDef::Create(); meta_def->name() = "DNNL_" + std::to_string(model_hash) + "_" + std::to_string(metadef_id); meta_def->domain() = kMSDomain; @@ -264,7 +265,7 @@ std::vector> DnnlExecutionProvider::GetCapabi graph_viewer.ToProto(*model_proto->mutable_graph(), false, true); model_proto->set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION); HashValue model_hash; - int metadef_id = ModelMetadefIdGenerator::GenerateMetaDefId(graph_viewer, model_hash); + int metadef_id = GenerateMetaDefId(graph_viewer, model_hash); std::fstream dump("DNNL_" + std::to_string(model_hash) + "_" + std::to_string(metadef_id) + ".onnx", std::ios::out | std::ios::trunc | std::ios::binary); model_proto->SerializeToOstream(dump); } diff --git a/onnxruntime/core/providers/js/js_execution_provider.cc b/onnxruntime/core/providers/js/js_execution_provider.cc index c2ff2ebc39e13..21bf09b274aff 100644 --- a/onnxruntime/core/providers/js/js_execution_provider.cc +++ b/onnxruntime/core/providers/js/js_execution_provider.cc @@ -680,7 +680,7 @@ std::unique_ptr RegisterKernels() { using namespace js; JsExecutionProvider::JsExecutionProvider(const JsExecutionProviderInfo& info) - : IExecutionProvider{kJsExecutionProvider, OrtDevice(OrtDevice::GPU, OrtDevice::MemType::DEFAULT, 0), true}, + : IExecutionProvider{kJsExecutionProvider, OrtDevice(OrtDevice::GPU, OrtDevice::MemType::DEFAULT, 0)}, preferred_data_layout_{info.data_layout} { } diff --git a/onnxruntime/core/providers/migraphx/migraphx_execution_provider.cc b/onnxruntime/core/providers/migraphx/migraphx_execution_provider.cc index 32055d526ab3a..0a361bbcacfbe 100644 --- a/onnxruntime/core/providers/migraphx/migraphx_execution_provider.cc +++ b/onnxruntime/core/providers/migraphx/migraphx_execution_provider.cc @@ -19,6 +19,7 @@ // TODO: find a better way to share this #include "core/providers/rocm/rocm_stream_handle.h" +#include "core/framework/model_metadef_id_generator.h" #if defined(_MSC_VER) #pragma warning(disable : 4244 4245) @@ -102,7 +103,7 @@ std::shared_ptr MIGraphXExecutionProvider::GetKernelRegistry() c } MIGraphXExecutionProvider::MIGraphXExecutionProvider(const MIGraphXExecutionProviderInfo& info) - : IExecutionProvider{onnxruntime::kMIGraphXExecutionProvider, OrtDevice(OrtDevice::GPU, OrtDevice::MemType::DEFAULT, info.device_id), true}, device_id_(info.device_id) { + : IExecutionProvider{onnxruntime::kMIGraphXExecutionProvider, OrtDevice(OrtDevice::GPU, OrtDevice::MemType::DEFAULT, info.device_id)}, device_id_(info.device_id) { InitProviderOrtApi(); // Set GPU device to be used HIP_CALL_THROW(hipSetDevice(device_id_)); @@ -757,7 +758,7 @@ std::unique_ptr MIGraphXExecutionProvider::GetSubGraph(const st // Generate unique kernel name for MIGraphX subgraph uint64_t model_hash = 0; - int id = ModelMetadefIdGenerator::GenerateMetaDefId(graph, model_hash); + int id = GenerateMetaDefId(graph, model_hash); std::string subgraph_id = std::to_string(model_hash) + "_" + std::to_string(id); auto meta_def = IndexedSubGraph_MetaDef::Create(); const std::string graph_type = graph.IsSubgraph() ? "subgraph" : "graph"; diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc index aad4290d19057..03be7a0f2a1e3 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc @@ -19,6 +19,7 @@ #include "core/providers/partitioning_utils.h" #include "core/providers/shared/node_unit/node_unit.h" #include "core/session/onnxruntime_cxx_api.h" +#include "core/framework/model_metadef_id_generator.h" namespace onnxruntime { @@ -50,7 +51,7 @@ std::unordered_set GetPartitioningStopOps(const optional& partitioning_stop_ops_list) - : IExecutionProvider{onnxruntime::kNnapiExecutionProvider, true}, + : IExecutionProvider{onnxruntime::kNnapiExecutionProvider}, nnapi_flags_(nnapi_flags), partitioning_stop_ops_(GetPartitioningStopOps(partitioning_stop_ops_list)) { nnapi_handle_ = NnApiImplementation(); @@ -176,7 +177,7 @@ NnapiExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_view const auto gen_metadef_name = [&]() { HashValue model_hash; - int metadef_id = ModelMetadefIdGenerator::GenerateMetaDefId(graph_viewer, model_hash); + int metadef_id = GenerateMetaDefId(graph_viewer, model_hash); return MakeString(NNAPI, "_", model_hash, "_", metadef_id); }; diff --git a/onnxruntime/core/providers/partitioning_utils.h b/onnxruntime/core/providers/partitioning_utils.h index f9d5f7403f17b..8abbfde752a39 100644 --- a/onnxruntime/core/providers/partitioning_utils.h +++ b/onnxruntime/core/providers/partitioning_utils.h @@ -40,7 +40,7 @@ using OnGroupClosedFn = std::function& group /** Called to create a metadef name. -Most likely should call IExecutionProvider::GenerateMetaDefId. +Most likely should call GenerateMetaDefId. See onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.cc for example usage. @return The metadef name. diff --git a/onnxruntime/core/providers/qnn/qnn_execution_provider.cc b/onnxruntime/core/providers/qnn/qnn_execution_provider.cc index bb8867f6fc9ff..b1ddf04921578 100644 --- a/onnxruntime/core/providers/qnn/qnn_execution_provider.cc +++ b/onnxruntime/core/providers/qnn/qnn_execution_provider.cc @@ -17,6 +17,7 @@ #include "core/providers/qnn/builder/op_builder_factory.h" #include "core/providers/qnn/builder/qnn_def.h" #include "core/providers/qnn/builder/onnx_ctx_model_helper.h" +#include "core/framework/model_metadef_id_generator.h" namespace onnxruntime { @@ -110,7 +111,7 @@ void QNNExecutionProvider::ParseHtpGraphFinalizationOptimizationMode(const std:: QNNExecutionProvider::QNNExecutionProvider(const ProviderOptions& provider_options_map, const SessionOptions* session_options) - : IExecutionProvider{onnxruntime::kQnnExecutionProvider, true} { + : IExecutionProvider{onnxruntime::kQnnExecutionProvider} { if (session_options) { disable_cpu_ep_fallback_ = session_options->config_options.GetConfigOrDefault( kOrtSessionOptionsDisableCPUEPFallback, "0") == "1"; @@ -396,7 +397,7 @@ QNNExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_viewer const auto gen_metadef_name = [&]() { uint64_t model_hash; - int metadef_id = ModelMetadefIdGenerator::GenerateMetaDefId(graph_viewer, model_hash); + int metadef_id = GenerateMetaDefId(graph_viewer, model_hash); return MakeString(QNN, "_", model_hash, "_", metadef_id); }; diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc index 684303a8b6448..c1ce5462badc2 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc @@ -1311,7 +1311,7 @@ TensorrtExecutionProvider::PerThreadContext& TensorrtExecutionProvider::GetPerTh } TensorrtExecutionProvider::TensorrtExecutionProvider(const TensorrtExecutionProviderInfo& info) - : IExecutionProvider{onnxruntime::kTensorrtExecutionProvider, OrtDevice(OrtDevice::GPU, OrtDevice::MemType::DEFAULT, info.device_id), true}, info_(info), device_id_(info.device_id) { + : IExecutionProvider{onnxruntime::kTensorrtExecutionProvider, OrtDevice(OrtDevice::GPU, OrtDevice::MemType::DEFAULT, info.device_id)}, info_(info), device_id_(info.device_id) { InitProviderOrtApi(); CUDA_CALL_THROW(cudaSetDevice(device_id_)); diff --git a/onnxruntime/core/providers/webnn/webnn_execution_provider.cc b/onnxruntime/core/providers/webnn/webnn_execution_provider.cc index 876df4786b075..7526e977cb0fc 100644 --- a/onnxruntime/core/providers/webnn/webnn_execution_provider.cc +++ b/onnxruntime/core/providers/webnn/webnn_execution_provider.cc @@ -14,12 +14,13 @@ #include "builders/model.h" #include "builders/helper.h" #include "builders/model_builder.h" +#include "core/framework/model_metadef_id_generator.h" namespace onnxruntime { WebNNExecutionProvider::WebNNExecutionProvider(const std::string& webnn_device_flags, const std::string& webnn_threads_number, const std::string& webnn_power_flags) - : IExecutionProvider{onnxruntime::kWebNNExecutionProvider, true} { + : IExecutionProvider{onnxruntime::kWebNNExecutionProvider} { // Create WebNN context and graph builder. const emscripten::val ml = emscripten::val::global("navigator")["ml"]; if (!ml.as()) { @@ -158,7 +159,7 @@ WebNNExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_view // Assign inputs and outputs to subgraph's meta_def. uint64_t model_hash; - int metadef_id = ModelMetadefIdGenerator::GenerateMetaDefId(graph_viewer, model_hash); + int metadef_id = GenerateMetaDefId(graph_viewer, model_hash); auto meta_def = std::make_unique<::onnxruntime::IndexedSubGraph::MetaDef>(); meta_def->name = "WEBNN_" + std::to_string(model_hash) + "_" + std::to_string(metadef_id); meta_def->domain = kMSDomain; diff --git a/onnxruntime/core/providers/xnnpack/xnnpack_execution_provider.cc b/onnxruntime/core/providers/xnnpack/xnnpack_execution_provider.cc index a2a776df439e4..eafbfae6f01e1 100644 --- a/onnxruntime/core/providers/xnnpack/xnnpack_execution_provider.cc +++ b/onnxruntime/core/providers/xnnpack/xnnpack_execution_provider.cc @@ -155,7 +155,7 @@ std::unique_ptr RegisterKernels() { using namespace xnnpack; XnnpackExecutionProvider::XnnpackExecutionProvider(const XnnpackExecutionProviderInfo& info) - : IExecutionProvider{kXnnpackExecutionProvider, true} { + : IExecutionProvider{kXnnpackExecutionProvider} { int xnn_thread_pool_size = info.xnn_thread_pool_size; int ort_thread_pool_size = info.session_options ? info.session_options->intra_op_param.thread_pool_size : 1; bool allow_intra_op_spinning = (info.session_options == nullptr) || diff --git a/onnxruntime/test/framework/execution_provider_test.cc b/onnxruntime/test/framework/execution_provider_test.cc index 5eb8288181761..8f8abb7ef5d0c 100644 --- a/onnxruntime/test/framework/execution_provider_test.cc +++ b/onnxruntime/test/framework/execution_provider_test.cc @@ -6,6 +6,7 @@ #include "test_utils.h" #include "test/test_environment.h" #include "test/util/include/asserts.h" +#include "core/framework/model_metadef_id_generator.h" #include "gtest/gtest.h" @@ -21,7 +22,7 @@ class TestEP : public IExecutionProvider { TestEP() : IExecutionProvider{kEPType} {} int GetId(const GraphViewer& viewer, HashValue& model_hash) { - return ModelMetadefIdGenerator::GenerateMetaDefId(viewer, model_hash); + return GenerateMetaDefId(viewer, model_hash); } }; diff --git a/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.cc b/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.cc index a622157d62fbb..7b3b95d96cff0 100644 --- a/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.cc +++ b/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.cc @@ -20,6 +20,7 @@ #include "core/framework/op_kernel.h" #include "core/framework/kernel_registry.h" #include "internal_testing_ep_static_kernels.h" // for BuildKernelCreateInfo declaration +#include "core/framework/model_metadef_id_generator.h" namespace onnxruntime { namespace internal_testing_ep { @@ -212,7 +213,7 @@ InternalTestingExecutionProvider::GetCapability(const onnxruntime::GraphViewer& // create functor to generate a guaranteed unique metadef id auto generate_metadef_name = [this, &graph_viewer]() { HashValue model_hash; - int metadef_id = ModelMetadefIdGenerator::GenerateMetaDefId(graph_viewer, model_hash); + int metadef_id = GenerateMetaDefId(graph_viewer, model_hash); auto meta_def = std::make_unique<::onnxruntime::IndexedSubGraph::MetaDef>(); return ep_name_ + "_" + std::to_string(model_hash) + "_" + std::to_string(metadef_id); }; From 0654786b344a7b8cba6c500c7ed930515daf6afc Mon Sep 17 00:00:00 2001 From: Lei Cao Date: Wed, 3 Jan 2024 17:46:04 -0800 Subject: [PATCH 03/14] typo --- onnxruntime/core/framework/model_metadef_id_generator.cc | 5 +++-- onnxruntime/core/framework/model_metadef_id_generator.h | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/onnxruntime/core/framework/model_metadef_id_generator.cc b/onnxruntime/core/framework/model_metadef_id_generator.cc index d4b0c39b8ff6e..6dcce1e71dceb 100644 --- a/onnxruntime/core/framework/model_metadef_id_generator.cc +++ b/onnxruntime/core/framework/model_metadef_id_generator.cc @@ -4,7 +4,7 @@ #include "model_metadef_id_generator.h" #include "core/platform/ort_mutex.h" #include "core/graph/graph_viewer.h" -#include "core/framework/murmurhash3.h +#include "core/framework/murmurhash3.h" namespace onnxruntime { int GenerateMetaDefId(const onnxruntime::GraphViewer& graph_viewer, HashValue& model_hash) { @@ -73,4 +73,5 @@ int GenerateMetaDefId(const onnxruntime::GraphViewer& graph_viewer, HashValue& m // return the current unique id, and increment to update return model_metadef_id_[model_hash]++; } -} + +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/model_metadef_id_generator.h b/onnxruntime/core/framework/model_metadef_id_generator.h index 7b343f3cdaf10..0f9ca02bee994 100644 --- a/onnxruntime/core/framework/model_metadef_id_generator.h +++ b/onnxruntime/core/framework/model_metadef_id_generator.h @@ -20,4 +20,5 @@ class GraphViewer; virtual, and ModelMetadefIdGenerator but be defined in the header as well. */ int GenerateMetaDefId(const onnxruntime::GraphViewer& graph_viewer, HashValue& model_hash); -} + +} // namespace onnxruntime From b6a769e2b110fa929c9f99d47dafd18d809ae4f8 Mon Sep 17 00:00:00 2001 From: Lei Cao Date: Wed, 10 Jan 2024 16:38:23 -0800 Subject: [PATCH 04/14] fix comments from Scott --- .../framework/model_metadef_id_generator.cc | 6 ++---- .../framework/model_metadef_id_generator.h | 20 ++++++++----------- .../providers/cann/cann_execution_provider.cc | 4 ++-- .../providers/cann/cann_execution_provider.h | 2 ++ .../coreml/coreml_execution_provider.cc | 4 ++-- .../coreml/coreml_execution_provider.h | 2 ++ .../providers/dnnl/dnnl_execution_provider.cc | 9 ++++----- .../providers/dnnl/dnnl_execution_provider.h | 2 ++ .../migraphx/migraphx_execution_provider.cc | 5 +++-- .../migraphx/migraphx_execution_provider.h | 2 ++ .../nnapi_builtin/nnapi_execution_provider.cc | 5 +++-- .../nnapi_builtin/nnapi_execution_provider.h | 2 ++ .../core/providers/partitioning_utils.h | 2 +- .../providers/qnn/qnn_execution_provider.cc | 5 +++-- .../providers/qnn/qnn_execution_provider.h | 2 ++ .../webnn/webnn_execution_provider.cc | 4 ++-- .../webnn/webnn_execution_provider.h | 2 ++ .../test/framework/execution_provider_test.cc | 8 ++++++-- .../internal_testing_execution_provider.cc | 3 ++- 19 files changed, 52 insertions(+), 37 deletions(-) diff --git a/onnxruntime/core/framework/model_metadef_id_generator.cc b/onnxruntime/core/framework/model_metadef_id_generator.cc index 6dcce1e71dceb..757c5282ca49b 100644 --- a/onnxruntime/core/framework/model_metadef_id_generator.cc +++ b/onnxruntime/core/framework/model_metadef_id_generator.cc @@ -7,10 +7,8 @@ #include "core/framework/murmurhash3.h" namespace onnxruntime { -int GenerateMetaDefId(const onnxruntime::GraphViewer& graph_viewer, HashValue& model_hash) { - static std::unordered_map main_graph_hash_; // map graph instance hash to model contents hash - static std::unordered_map model_metadef_id_; // current unique id for model - +int ModelMetadefIdGenerator::GenerateId(const onnxruntime::GraphViewer& graph_viewer, + HashValue& model_hash) { // if the EP is shared across multiple sessions there's a very small potential for concurrency issues. // use a lock when generating an id to be paranoid static OrtMutex mutex; diff --git a/onnxruntime/core/framework/model_metadef_id_generator.h b/onnxruntime/core/framework/model_metadef_id_generator.h index 0f9ca02bee994..0d0fc6b26dda1 100644 --- a/onnxruntime/core/framework/model_metadef_id_generator.h +++ b/onnxruntime/core/framework/model_metadef_id_generator.h @@ -8,17 +8,13 @@ class GraphViewer; // helper to generate ids that are unique to model and deterministic, even if the execution provider is shared across // multiple sessions. -/** Generate a unique id that can be used in a MetaDef name. Values are unique for a model instance. - The model hash is also returned if you wish to include that in the MetaDef name to ensure uniqueness across models. - @param graph_viewer[in] Graph viewer that GetCapability was called with. Can be for the main graph or nested graph. - @param model_hash[out] Returns the hash for the main (i.e. top level) graph in the model. - This is created using the model path if available, - or the model input names and the output names from all nodes in the main graph. - @remarks e.g. the TensorRT Execution Provider is used in multiple sessions and the underlying infrastructure caches - compiled kernels, so the name must be unique and deterministic across models and sessions. - NOTE: Ideally this would be a protected method, but to work across the EP bridge it has to be public and - virtual, and ModelMetadefIdGenerator but be defined in the header as well. - */ -int GenerateMetaDefId(const onnxruntime::GraphViewer& graph_viewer, HashValue& model_hash); +class ModelMetadefIdGenerator { + public: + int GenerateId(const onnxruntime::GraphViewer& graph_viewer, HashValue& model_hash); + + private: + std::unordered_map main_graph_hash_; // map graph instance hash to model contents hash + std::unordered_map model_metadef_id_; // current unique id for model +}; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cann/cann_execution_provider.cc b/onnxruntime/core/providers/cann/cann_execution_provider.cc index 8aae5bf56cadd..e3398ac07709a 100644 --- a/onnxruntime/core/providers/cann/cann_execution_provider.cc +++ b/onnxruntime/core/providers/cann/cann_execution_provider.cc @@ -19,7 +19,6 @@ #include "core/providers/cann/cann_fwd.h" #include "core/providers/cann/cann_stream_handle.h" #include "core/providers/cann/npu_data_transfer.h" -#include "core/framework/model_metadef_id_generator.h" using onnxruntime::cann::BuildONNXModel; using onnxruntime::cann::CannModelPreparation; @@ -1037,6 +1036,7 @@ CANNExecutionProvider::CANNExecutionProvider(const CANNExecutionProviderInfo& in soc_name_ = aclrtGetSocName(); ORT_ENFORCE(soc_name_ != nullptr, "aclrtGetSocName return nullptr"); + metadef_id_generator_ = std::make_unique(); } CANNExecutionProvider::~CANNExecutionProvider() { @@ -1198,7 +1198,7 @@ std::unique_ptr CANNExecutionProvider::GetSubGraph( // Generate unique kernel name for CANN subgraph HashValue model_hash = 0; - int id = GenerateMetaDefId(graph_viewer, model_hash); + int id = metadef_id_generator_->GenerateId(graph_viewer, model_hash); auto meta_def = IndexedSubGraph_MetaDef::Create(); meta_def->name() = graph_viewer.Name() + "_" + std::to_string(model_hash) + "_" + std::to_string(id); diff --git a/onnxruntime/core/providers/cann/cann_execution_provider.h b/onnxruntime/core/providers/cann/cann_execution_provider.h index 76d3d9c331563..cba3cc23b1887 100644 --- a/onnxruntime/core/providers/cann/cann_execution_provider.h +++ b/onnxruntime/core/providers/cann/cann_execution_provider.h @@ -12,6 +12,7 @@ #include "core/providers/shared_library/provider_api.h" #include "core/framework/arena_extend_strategy.h" #include "core/framework/execution_provider.h" +#include "core/framework/model_metadef_id_generator.h" #include "core/platform/ort_mutex.h" #include "core/providers/cann/cann_execution_provider_info.h" #include "core/providers/cann/cann_inc.h" @@ -81,6 +82,7 @@ class CANNExecutionProvider : public IExecutionProvider { std::unordered_map modelIDs_; std::unordered_map models_; std::unordered_map> names_; + std::unique_ptr metadef_id_generator_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/coreml/coreml_execution_provider.cc b/onnxruntime/core/providers/coreml/coreml_execution_provider.cc index aa9dc1eea0420..40349d5b2a99b 100644 --- a/onnxruntime/core/providers/coreml/coreml_execution_provider.cc +++ b/onnxruntime/core/providers/coreml/coreml_execution_provider.cc @@ -11,7 +11,6 @@ #include "core/providers/coreml/builders/helper.h" #include "core/providers/partitioning_utils.h" #include "core/session/onnxruntime_cxx_api.h" -#include "core/framework/model_metadef_id_generator.h" #ifdef __APPLE__ #include "core/providers/coreml/builders/model_builder.h" @@ -27,6 +26,7 @@ constexpr const char* COREML = "CoreML"; CoreMLExecutionProvider::CoreMLExecutionProvider(uint32_t coreml_flags) : IExecutionProvider{onnxruntime::kCoreMLExecutionProvider}, coreml_flags_(coreml_flags) { + metadef_id_generator_ = std::make_unique(); } CoreMLExecutionProvider::~CoreMLExecutionProvider() {} @@ -55,7 +55,7 @@ CoreMLExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_vie const auto gen_metadef_name = [&]() { HashValue model_hash; - int metadef_id = GenerateMetaDefId(graph_viewer, model_hash); + int metadef_id = metadef_id_generator_->GenerateId(graph_viewer, model_hash); return MakeString(COREML, "_", model_hash, "_", metadef_id); }; diff --git a/onnxruntime/core/providers/coreml/coreml_execution_provider.h b/onnxruntime/core/providers/coreml/coreml_execution_provider.h index 67050e8079cf9..c0d46fc4c9c4e 100644 --- a/onnxruntime/core/providers/coreml/coreml_execution_provider.h +++ b/onnxruntime/core/providers/coreml/coreml_execution_provider.h @@ -4,6 +4,7 @@ #pragma once #include "core/framework/execution_provider.h" +#include "core/framework/model_metadef_id_generator.h" #include "core/providers/coreml/coreml_provider_factory.h" namespace onnxruntime { @@ -34,5 +35,6 @@ class CoreMLExecutionProvider : public IExecutionProvider { #ifdef __APPLE__ std::unordered_map> coreml_models_; #endif + std::unique_ptr metadef_id_generator_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc b/onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc index b254253f3be2c..fd2e01e0789b1 100644 --- a/onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc +++ b/onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc @@ -20,7 +20,6 @@ #include "core/providers/dnnl/dnnl_fwd.h" #include "core/providers/dnnl/dnnl_node_capability.h" #include "core/providers/dnnl/subgraph/dnnl_subgraph_transformer.h" -#include "core/framework/model_metadef_id_generator.h" #define ORT_API_MANUAL_INIT #include "core/session/onnxruntime_cxx_api.h" @@ -78,8 +77,8 @@ DnnlExecutionProvider::DnnlExecutionProvider(const DnnlExecutionProviderInfo& in // Log the number of threads used LOGS_DEFAULT(INFO) << "Allocated " << omp_get_max_threads() << " OpenMP threads for oneDNN ep\n"; #endif // defined(DNNL_OPENMP) - -} // namespace onnxruntime + metadef_id_generator_ = std::make_unique(); +} DnnlExecutionProvider::~DnnlExecutionProvider() { } @@ -230,7 +229,7 @@ std::vector> DnnlExecutionProvider::GetCapabi // Assign inputs and outputs to subgraph's meta_def HashValue model_hash; - int metadef_id = GenerateMetaDefId(graph_viewer, model_hash); + int metadef_id = metadef_id_generator_->GenerateId(graph_viewer, model_hash); auto meta_def = ::onnxruntime::IndexedSubGraph_MetaDef::Create(); meta_def->name() = "DNNL_" + std::to_string(model_hash) + "_" + std::to_string(metadef_id); meta_def->domain() = kMSDomain; @@ -265,7 +264,7 @@ std::vector> DnnlExecutionProvider::GetCapabi graph_viewer.ToProto(*model_proto->mutable_graph(), false, true); model_proto->set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION); HashValue model_hash; - int metadef_id = GenerateMetaDefId(graph_viewer, model_hash); + int metadef_id = metadef_id_generator_->GenerateId(graph_viewer, model_hash); std::fstream dump("DNNL_" + std::to_string(model_hash) + "_" + std::to_string(metadef_id) + ".onnx", std::ios::out | std::ios::trunc | std::ios::binary); model_proto->SerializeToOstream(dump); } diff --git a/onnxruntime/core/providers/dnnl/dnnl_execution_provider.h b/onnxruntime/core/providers/dnnl/dnnl_execution_provider.h index 41062ccb4bc1b..dd8d6586c7af9 100644 --- a/onnxruntime/core/providers/dnnl/dnnl_execution_provider.h +++ b/onnxruntime/core/providers/dnnl/dnnl_execution_provider.h @@ -8,6 +8,7 @@ #include #include +#include "core/framework/model_metadef_id_generator.h" #include "core/providers/dnnl/dnnl_execution_provider_info.h" #include "core/providers/dnnl/dnnl_threadpool.h" #include "core/providers/dnnl/dnnl_op_manager.h" @@ -41,6 +42,7 @@ class DnnlExecutionProvider : public IExecutionProvider { bool debug_log_ = false; // enable fusion by default bool enable_fusion_ = true; + std::unique_ptr metadef_id_generator_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/migraphx/migraphx_execution_provider.cc b/onnxruntime/core/providers/migraphx/migraphx_execution_provider.cc index 0a361bbcacfbe..02c2ab6c613dd 100644 --- a/onnxruntime/core/providers/migraphx/migraphx_execution_provider.cc +++ b/onnxruntime/core/providers/migraphx/migraphx_execution_provider.cc @@ -19,7 +19,6 @@ // TODO: find a better way to share this #include "core/providers/rocm/rocm_stream_handle.h" -#include "core/framework/model_metadef_id_generator.h" #if defined(_MSC_VER) #pragma warning(disable : 4244 4245) @@ -166,6 +165,8 @@ MIGraphXExecutionProvider::MIGraphXExecutionProvider(const MIGraphXExecutionProv MIOPEN_CALL_THROW(miopenCreate(&external_miopen_handle_)); MIOPEN_CALL_THROW(miopenSetStream(external_miopen_handle_, stream_)); + metadef_id_generator_ = std::make_unique(); + LOGS_DEFAULT(VERBOSE) << "[MIGraphX EP] MIGraphX provider options: " << "device_id: " << device_id_ << ", migraphx_fp16_enable: " << fp16_enable_ @@ -758,7 +759,7 @@ std::unique_ptr MIGraphXExecutionProvider::GetSubGraph(const st // Generate unique kernel name for MIGraphX subgraph uint64_t model_hash = 0; - int id = GenerateMetaDefId(graph, model_hash); + int id = metadef_id_generator_->GenerateId(graph, model_hash); std::string subgraph_id = std::to_string(model_hash) + "_" + std::to_string(id); auto meta_def = IndexedSubGraph_MetaDef::Create(); const std::string graph_type = graph.IsSubgraph() ? "subgraph" : "graph"; diff --git a/onnxruntime/core/providers/migraphx/migraphx_execution_provider.h b/onnxruntime/core/providers/migraphx/migraphx_execution_provider.h index c094be51012e4..83c42e85d2cec 100644 --- a/onnxruntime/core/providers/migraphx/migraphx_execution_provider.h +++ b/onnxruntime/core/providers/migraphx/migraphx_execution_provider.h @@ -8,6 +8,7 @@ #include "core/framework/arena_extend_strategy.h" #include "core/framework/execution_provider.h" +#include "core/framework/model_metadef_id_generator.h" #include "core/platform/ort_mutex.h" #include "core/providers/migraphx/migraphx_execution_provider_info.h" #include "core/providers/migraphx/migraphx_inc.h" @@ -98,6 +99,7 @@ class MIGraphXExecutionProvider : public IExecutionProvider { AllocatorPtr allocator_; miopenHandle_t external_miopen_handle_ = nullptr; rocblas_handle external_rocblas_handle_ = nullptr; + std::unique_ptr metadef_id_generator_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc index 03be7a0f2a1e3..06129ac0fb969 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc @@ -19,7 +19,6 @@ #include "core/providers/partitioning_utils.h" #include "core/providers/shared/node_unit/node_unit.h" #include "core/session/onnxruntime_cxx_api.h" -#include "core/framework/model_metadef_id_generator.h" namespace onnxruntime { @@ -72,6 +71,8 @@ NnapiExecutionProvider::NnapiExecutionProvider(uint32_t nnapi_flags, // May we could just mark this EP as unavailable instead of throwing an error ORT_THROW_IF_ERROR(GetTargetDevices(*nnapi_handle_, target_device_option_, nnapi_target_devices_)); + metadef_id_generator_ = std::make_unique(); + LOGS_DEFAULT(VERBOSE) << "Found devices [" << nnapi::GetDevicesDescription(nnapi_target_devices_) << "] in NNAPI"; } @@ -177,7 +178,7 @@ NnapiExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_view const auto gen_metadef_name = [&]() { HashValue model_hash; - int metadef_id = GenerateMetaDefId(graph_viewer, model_hash); + int metadef_id = metadef_id_generator_->GenerateId(graph_viewer, model_hash); return MakeString(NNAPI, "_", model_hash, "_", metadef_id); }; diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h index e4911511e6db0..dd8b43c32e91b 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h @@ -6,6 +6,7 @@ #include "core/common/inlined_containers_fwd.h" #include "core/common/optional.h" #include "core/framework/execution_provider.h" +#include "core/framework/model_metadef_id_generator.h" #include "core/providers/nnapi/nnapi_builtin/nnapi_api_helper.h" #include "core/providers/nnapi/nnapi_provider_factory.h" @@ -48,5 +49,6 @@ class NnapiExecutionProvider : public IExecutionProvider { const NnApi* nnapi_handle_ = nullptr; nnapi::DeviceWrapperVector nnapi_target_devices_; nnapi::TargetDeviceOption target_device_option_; + std::unique_ptr metadef_id_generator_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/partitioning_utils.h b/onnxruntime/core/providers/partitioning_utils.h index 8abbfde752a39..136725c2f7250 100644 --- a/onnxruntime/core/providers/partitioning_utils.h +++ b/onnxruntime/core/providers/partitioning_utils.h @@ -40,7 +40,7 @@ using OnGroupClosedFn = std::function& group /** Called to create a metadef name. -Most likely should call GenerateMetaDefId. +Most likely should call ModelMetadefIdGenerator.GenerateId. See onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.cc for example usage. @return The metadef name. diff --git a/onnxruntime/core/providers/qnn/qnn_execution_provider.cc b/onnxruntime/core/providers/qnn/qnn_execution_provider.cc index b1ddf04921578..0540f36337fe6 100644 --- a/onnxruntime/core/providers/qnn/qnn_execution_provider.cc +++ b/onnxruntime/core/providers/qnn/qnn_execution_provider.cc @@ -17,7 +17,6 @@ #include "core/providers/qnn/builder/op_builder_factory.h" #include "core/providers/qnn/builder/qnn_def.h" #include "core/providers/qnn/builder/onnx_ctx_model_helper.h" -#include "core/framework/model_metadef_id_generator.h" namespace onnxruntime { @@ -207,6 +206,8 @@ QNNExecutionProvider::QNNExecutionProvider(const ProviderOptions& provider_optio htp_performance_mode, context_priority, std::move(qnn_saver_path)); + + metadef_id_generator_ = std::make_unique(); } bool QNNExecutionProvider::IsNodeSupported(qnn::QnnModelWrapper& qnn_model_wrapper, const NodeUnit& node_unit, @@ -397,7 +398,7 @@ QNNExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_viewer const auto gen_metadef_name = [&]() { uint64_t model_hash; - int metadef_id = GenerateMetaDefId(graph_viewer, model_hash); + int metadef_id = metadef_id_generator_->GenerateId(graph_viewer, model_hash); return MakeString(QNN, "_", model_hash, "_", metadef_id); }; diff --git a/onnxruntime/core/providers/qnn/qnn_execution_provider.h b/onnxruntime/core/providers/qnn/qnn_execution_provider.h index 8b5d0929209ee..a5f76005bd824 100644 --- a/onnxruntime/core/providers/qnn/qnn_execution_provider.h +++ b/onnxruntime/core/providers/qnn/qnn_execution_provider.h @@ -5,6 +5,7 @@ #include "core/framework/execution_provider.h" #include "core/framework/session_options.h" +#include "core/framework/model_metadef_id_generator.h" #include #include "core/providers/qnn/builder/qnn_backend_manager.h" #include "core/providers/qnn/builder/qnn_model.h" @@ -66,6 +67,7 @@ class QNNExecutionProvider : public IExecutionProvider { bool disable_cpu_ep_fallback_ = false; // True if CPU EP fallback has been disabled for this session. bool qnn_context_embed_mode_ = true; int32_t vtcm_size_in_mb_ = 0; + std::unique_ptr metadef_id_generator_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/webnn/webnn_execution_provider.cc b/onnxruntime/core/providers/webnn/webnn_execution_provider.cc index 7526e977cb0fc..7c3a1edc4fb67 100644 --- a/onnxruntime/core/providers/webnn/webnn_execution_provider.cc +++ b/onnxruntime/core/providers/webnn/webnn_execution_provider.cc @@ -14,7 +14,6 @@ #include "builders/model.h" #include "builders/helper.h" #include "builders/model_builder.h" -#include "core/framework/model_metadef_id_generator.h" namespace onnxruntime { @@ -51,6 +50,7 @@ WebNNExecutionProvider::WebNNExecutionProvider(const std::string& webnn_device_f if (!wnn_builder_.as()) { ORT_THROW("Failed to create WebNN builder."); } + metadef_id_generator_ = std::make_unique(); } WebNNExecutionProvider::~WebNNExecutionProvider() {} @@ -159,7 +159,7 @@ WebNNExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_view // Assign inputs and outputs to subgraph's meta_def. uint64_t model_hash; - int metadef_id = GenerateMetaDefId(graph_viewer, model_hash); + int metadef_id = metadef_id_generator_->GenerateId(graph_viewer, model_hash); auto meta_def = std::make_unique<::onnxruntime::IndexedSubGraph::MetaDef>(); meta_def->name = "WEBNN_" + std::to_string(model_hash) + "_" + std::to_string(metadef_id); meta_def->domain = kMSDomain; diff --git a/onnxruntime/core/providers/webnn/webnn_execution_provider.h b/onnxruntime/core/providers/webnn/webnn_execution_provider.h index 13a475327dc0c..d7d037fb53e76 100644 --- a/onnxruntime/core/providers/webnn/webnn_execution_provider.h +++ b/onnxruntime/core/providers/webnn/webnn_execution_provider.h @@ -6,6 +6,7 @@ #include "core/common/inlined_containers.h" #include "core/framework/execution_provider.h" +#include "core/framework/model_metadef_id_generator.h" #include "core/providers/webnn/builders/helper.h" #include @@ -48,5 +49,6 @@ class WebNNExecutionProvider : public IExecutionProvider { DataLayout preferred_layout_; webnn::WebnnDeviceType wnn_device_type_; InlinedHashMap> models_; + std::unique_ptr metadef_id_generator_; }; } // namespace onnxruntime diff --git a/onnxruntime/test/framework/execution_provider_test.cc b/onnxruntime/test/framework/execution_provider_test.cc index 8f8abb7ef5d0c..1df591c83ce6c 100644 --- a/onnxruntime/test/framework/execution_provider_test.cc +++ b/onnxruntime/test/framework/execution_provider_test.cc @@ -19,11 +19,15 @@ class TestEP : public IExecutionProvider { static constexpr const char* kEPType = "TestEP"; public: - TestEP() : IExecutionProvider{kEPType} {} + TestEP() : IExecutionProvider{kEPType} { + metadef_id_generator_ = std::make_unique(); + } int GetId(const GraphViewer& viewer, HashValue& model_hash) { - return GenerateMetaDefId(viewer, model_hash); + return metadef_id_generator_->GenerateId(viewer, model_hash); } + private: + std::unique_ptr metadef_id_generator_; }; TEST(ExecutionProviderTest, MetadefIdGeneratorUsingModelPath) { diff --git a/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.cc b/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.cc index 7b3b95d96cff0..7ac3c3775a3b8 100644 --- a/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.cc +++ b/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.cc @@ -213,7 +213,8 @@ InternalTestingExecutionProvider::GetCapability(const onnxruntime::GraphViewer& // create functor to generate a guaranteed unique metadef id auto generate_metadef_name = [this, &graph_viewer]() { HashValue model_hash; - int metadef_id = GenerateMetaDefId(graph_viewer, model_hash); + ModelMetadefIdGenerator metadef_id_generator; + int metadef_id = metadef_id_generator.GenerateId(graph_viewer, model_hash); auto meta_def = std::make_unique<::onnxruntime::IndexedSubGraph::MetaDef>(); return ep_name_ + "_" + std::to_string(model_hash) + "_" + std::to_string(metadef_id); }; From b20be192fcb4db0a68d2a5a0c7f4e28060fc195c Mon Sep 17 00:00:00 2001 From: Lei Cao Date: Thu, 18 Jan 2024 16:22:57 -0800 Subject: [PATCH 05/14] fix test and lint --- .../core/framework/model_metadef_id_generator.cc | 2 +- .../core/framework/model_metadef_id_generator.h | 11 +++++++++++ onnxruntime/test/framework/execution_provider_test.cc | 1 + .../internal_testing_execution_provider.cc | 5 ++--- .../internal_testing_execution_provider.h | 2 ++ 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/onnxruntime/core/framework/model_metadef_id_generator.cc b/onnxruntime/core/framework/model_metadef_id_generator.cc index 757c5282ca49b..73b88c9e7232a 100644 --- a/onnxruntime/core/framework/model_metadef_id_generator.cc +++ b/onnxruntime/core/framework/model_metadef_id_generator.cc @@ -8,7 +8,7 @@ namespace onnxruntime { int ModelMetadefIdGenerator::GenerateId(const onnxruntime::GraphViewer& graph_viewer, - HashValue& model_hash) { + HashValue& model_hash) { // if the EP is shared across multiple sessions there's a very small potential for concurrency issues. // use a lock when generating an id to be paranoid static OrtMutex mutex; diff --git a/onnxruntime/core/framework/model_metadef_id_generator.h b/onnxruntime/core/framework/model_metadef_id_generator.h index 0d0fc6b26dda1..d65924fb0a180 100644 --- a/onnxruntime/core/framework/model_metadef_id_generator.h +++ b/onnxruntime/core/framework/model_metadef_id_generator.h @@ -10,6 +10,17 @@ class GraphViewer; // multiple sessions. class ModelMetadefIdGenerator { public: + /** Generate a unique id that can be used in a MetaDef name. Values are unique for a model instance. + The model hash is also returned if you wish to include that in the MetaDef name to ensure uniqueness across models. + @param graph_viewer[in] Graph viewer that GetCapability was called with. Can be for the main graph or nested graph. + @param model_hash[out] Returns the hash for the main (i.e. top level) graph in the model. + This is created using the model path if available, + or the model input names and the output names from all nodes in the main graph. + @remarks e.g. the TensorRT Execution Provider is used in multiple sessions and the underlying infrastructure caches + compiled kernels, so the name must be unique and deterministic across models and sessions. + NOTE: Ideally this would be a protected method, but to work across the EP bridge it has to be public and + virtual, and ModelMetadefIdGenerator but be defined in the header as well. + */ int GenerateId(const onnxruntime::GraphViewer& graph_viewer, HashValue& model_hash); private: diff --git a/onnxruntime/test/framework/execution_provider_test.cc b/onnxruntime/test/framework/execution_provider_test.cc index 1df591c83ce6c..782e665d6b96e 100644 --- a/onnxruntime/test/framework/execution_provider_test.cc +++ b/onnxruntime/test/framework/execution_provider_test.cc @@ -26,6 +26,7 @@ class TestEP : public IExecutionProvider { int GetId(const GraphViewer& viewer, HashValue& model_hash) { return metadef_id_generator_->GenerateId(viewer, model_hash); } + private: std::unique_ptr metadef_id_generator_; }; diff --git a/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.cc b/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.cc index 7ac3c3775a3b8..ab52ed8803bf7 100644 --- a/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.cc +++ b/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.cc @@ -20,7 +20,6 @@ #include "core/framework/op_kernel.h" #include "core/framework/kernel_registry.h" #include "internal_testing_ep_static_kernels.h" // for BuildKernelCreateInfo declaration -#include "core/framework/model_metadef_id_generator.h" namespace onnxruntime { namespace internal_testing_ep { @@ -92,6 +91,7 @@ InternalTestingExecutionProvider::InternalTestingExecutionProvider(const std::un stop_ops_{stop_ops}, preferred_layout_{preferred_layout}, kernel_registry_{RegisterKernels()} { + metadef_id_generator_ = std::make_unique(); } std::vector InternalTestingExecutionProvider::CreatePreferredAllocators() { @@ -213,8 +213,7 @@ InternalTestingExecutionProvider::GetCapability(const onnxruntime::GraphViewer& // create functor to generate a guaranteed unique metadef id auto generate_metadef_name = [this, &graph_viewer]() { HashValue model_hash; - ModelMetadefIdGenerator metadef_id_generator; - int metadef_id = metadef_id_generator.GenerateId(graph_viewer, model_hash); + int metadef_id = metadef_id_generator_->GenerateId(graph_viewer, model_hash); auto meta_def = std::make_unique<::onnxruntime::IndexedSubGraph::MetaDef>(); return ep_name_ + "_" + std::to_string(model_hash) + "_" + std::to_string(metadef_id); }; diff --git a/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.h b/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.h index 6103352627667..f3eabf209432b 100644 --- a/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.h +++ b/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.h @@ -4,6 +4,7 @@ #pragma once #include #include "core/framework/execution_provider.h" +#include "core/framework/model_metadef_id_generator.h" namespace onnxruntime { namespace internal_testing_ep { @@ -82,6 +83,7 @@ class InternalTestingExecutionProvider : public IExecutionProvider { // per-instance kernel registry so tests using static kernels don't clash. // shared_ptr as required by IExecutionProvider::GetKernelRegistry std::shared_ptr kernel_registry_; + std::unique_ptr metadef_id_generator_; }; } // namespace internal_testing_ep From d9edfa001b1bd89312cb1740bf807998f2e88c45 Mon Sep 17 00:00:00 2001 From: Lei Cao Date: Fri, 19 Jan 2024 16:24:16 -0800 Subject: [PATCH 06/14] fix comments --- .../framework/model_metadef_id_generator.cc | 2 +- .../core/framework/model_metadef_id_generator.h | 17 ++++++++--------- .../providers/cann/cann_execution_provider.cc | 3 +-- .../providers/cann/cann_execution_provider.h | 2 +- .../coreml/coreml_execution_provider.cc | 3 +-- .../coreml/coreml_execution_provider.h | 2 +- .../providers/dnnl/dnnl_execution_provider.cc | 5 ++--- .../providers/dnnl/dnnl_execution_provider.h | 2 +- .../migraphx/migraphx_execution_provider.cc | 4 +--- .../migraphx/migraphx_execution_provider.h | 2 +- .../nnapi_builtin/nnapi_execution_provider.cc | 4 +--- .../nnapi_builtin/nnapi_execution_provider.h | 2 +- .../providers/qnn/qnn_execution_provider.cc | 4 +--- .../core/providers/qnn/qnn_execution_provider.h | 2 +- .../tensorrt_execution_provider_utils.h | 10 +++++++++- .../providers/webnn/webnn_execution_provider.cc | 3 +-- .../providers/webnn/webnn_execution_provider.h | 2 +- .../test/framework/execution_provider_test.cc | 8 +++----- .../internal_testing_execution_provider.cc | 3 +-- .../internal_testing_execution_provider.h | 2 +- 20 files changed, 38 insertions(+), 44 deletions(-) diff --git a/onnxruntime/core/framework/model_metadef_id_generator.cc b/onnxruntime/core/framework/model_metadef_id_generator.cc index 73b88c9e7232a..e51c6ebc29975 100644 --- a/onnxruntime/core/framework/model_metadef_id_generator.cc +++ b/onnxruntime/core/framework/model_metadef_id_generator.cc @@ -8,7 +8,7 @@ namespace onnxruntime { int ModelMetadefIdGenerator::GenerateId(const onnxruntime::GraphViewer& graph_viewer, - HashValue& model_hash) { + HashValue& model_hash) const { // if the EP is shared across multiple sessions there's a very small potential for concurrency issues. // use a lock when generating an id to be paranoid static OrtMutex mutex; diff --git a/onnxruntime/core/framework/model_metadef_id_generator.h b/onnxruntime/core/framework/model_metadef_id_generator.h index d65924fb0a180..8445ee3c609a7 100644 --- a/onnxruntime/core/framework/model_metadef_id_generator.h +++ b/onnxruntime/core/framework/model_metadef_id_generator.h @@ -2,12 +2,15 @@ // Licensed under the MIT License. #pragma once +#include #include "core/common/basic_types.h" namespace onnxruntime { class GraphViewer; -// helper to generate ids that are unique to model and deterministic, even if the execution provider is shared across -// multiple sessions. +/// +/// helper to generate ids that are unique to model and deterministic, even if the execution provider is shared across +/// multiple sessions. +/// class ModelMetadefIdGenerator { public: /** Generate a unique id that can be used in a MetaDef name. Values are unique for a model instance. @@ -16,16 +19,12 @@ class ModelMetadefIdGenerator { @param model_hash[out] Returns the hash for the main (i.e. top level) graph in the model. This is created using the model path if available, or the model input names and the output names from all nodes in the main graph. - @remarks e.g. the TensorRT Execution Provider is used in multiple sessions and the underlying infrastructure caches - compiled kernels, so the name must be unique and deterministic across models and sessions. - NOTE: Ideally this would be a protected method, but to work across the EP bridge it has to be public and - virtual, and ModelMetadefIdGenerator but be defined in the header as well. */ - int GenerateId(const onnxruntime::GraphViewer& graph_viewer, HashValue& model_hash); + int GenerateId(const onnxruntime::GraphViewer& graph_viewer, HashValue& model_hash) const; private: - std::unordered_map main_graph_hash_; // map graph instance hash to model contents hash - std::unordered_map model_metadef_id_; // current unique id for model + mutable std::unordered_map main_graph_hash_; // map graph instance hash to model contents hash + mutable std::unordered_map model_metadef_id_; // current unique id for model }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cann/cann_execution_provider.cc b/onnxruntime/core/providers/cann/cann_execution_provider.cc index e3398ac07709a..e6afe8072bbac 100644 --- a/onnxruntime/core/providers/cann/cann_execution_provider.cc +++ b/onnxruntime/core/providers/cann/cann_execution_provider.cc @@ -1036,7 +1036,6 @@ CANNExecutionProvider::CANNExecutionProvider(const CANNExecutionProviderInfo& in soc_name_ = aclrtGetSocName(); ORT_ENFORCE(soc_name_ != nullptr, "aclrtGetSocName return nullptr"); - metadef_id_generator_ = std::make_unique(); } CANNExecutionProvider::~CANNExecutionProvider() { @@ -1198,7 +1197,7 @@ std::unique_ptr CANNExecutionProvider::GetSubGraph( // Generate unique kernel name for CANN subgraph HashValue model_hash = 0; - int id = metadef_id_generator_->GenerateId(graph_viewer, model_hash); + int id = metadef_id_generator_.GenerateId(graph_viewer, model_hash); auto meta_def = IndexedSubGraph_MetaDef::Create(); meta_def->name() = graph_viewer.Name() + "_" + std::to_string(model_hash) + "_" + std::to_string(id); diff --git a/onnxruntime/core/providers/cann/cann_execution_provider.h b/onnxruntime/core/providers/cann/cann_execution_provider.h index cba3cc23b1887..8fb231fc0c6b3 100644 --- a/onnxruntime/core/providers/cann/cann_execution_provider.h +++ b/onnxruntime/core/providers/cann/cann_execution_provider.h @@ -82,7 +82,7 @@ class CANNExecutionProvider : public IExecutionProvider { std::unordered_map modelIDs_; std::unordered_map models_; std::unordered_map> names_; - std::unique_ptr metadef_id_generator_; + ModelMetadefIdGenerator metadef_id_generator_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/coreml/coreml_execution_provider.cc b/onnxruntime/core/providers/coreml/coreml_execution_provider.cc index 40349d5b2a99b..c133f7b82aba4 100644 --- a/onnxruntime/core/providers/coreml/coreml_execution_provider.cc +++ b/onnxruntime/core/providers/coreml/coreml_execution_provider.cc @@ -26,7 +26,6 @@ constexpr const char* COREML = "CoreML"; CoreMLExecutionProvider::CoreMLExecutionProvider(uint32_t coreml_flags) : IExecutionProvider{onnxruntime::kCoreMLExecutionProvider}, coreml_flags_(coreml_flags) { - metadef_id_generator_ = std::make_unique(); } CoreMLExecutionProvider::~CoreMLExecutionProvider() {} @@ -55,7 +54,7 @@ CoreMLExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_vie const auto gen_metadef_name = [&]() { HashValue model_hash; - int metadef_id = metadef_id_generator_->GenerateId(graph_viewer, model_hash); + int metadef_id = metadef_id_generator_.GenerateId(graph_viewer, model_hash); return MakeString(COREML, "_", model_hash, "_", metadef_id); }; diff --git a/onnxruntime/core/providers/coreml/coreml_execution_provider.h b/onnxruntime/core/providers/coreml/coreml_execution_provider.h index c0d46fc4c9c4e..0201739547dd1 100644 --- a/onnxruntime/core/providers/coreml/coreml_execution_provider.h +++ b/onnxruntime/core/providers/coreml/coreml_execution_provider.h @@ -35,6 +35,6 @@ class CoreMLExecutionProvider : public IExecutionProvider { #ifdef __APPLE__ std::unordered_map> coreml_models_; #endif - std::unique_ptr metadef_id_generator_; + ModelMetadefIdGenerator metadef_id_generator_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc b/onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc index fd2e01e0789b1..4686f58070342 100644 --- a/onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc +++ b/onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc @@ -77,7 +77,6 @@ DnnlExecutionProvider::DnnlExecutionProvider(const DnnlExecutionProviderInfo& in // Log the number of threads used LOGS_DEFAULT(INFO) << "Allocated " << omp_get_max_threads() << " OpenMP threads for oneDNN ep\n"; #endif // defined(DNNL_OPENMP) - metadef_id_generator_ = std::make_unique(); } DnnlExecutionProvider::~DnnlExecutionProvider() { @@ -229,7 +228,7 @@ std::vector> DnnlExecutionProvider::GetCapabi // Assign inputs and outputs to subgraph's meta_def HashValue model_hash; - int metadef_id = metadef_id_generator_->GenerateId(graph_viewer, model_hash); + int metadef_id = metadef_id_generator_.GenerateId(graph_viewer, model_hash); auto meta_def = ::onnxruntime::IndexedSubGraph_MetaDef::Create(); meta_def->name() = "DNNL_" + std::to_string(model_hash) + "_" + std::to_string(metadef_id); meta_def->domain() = kMSDomain; @@ -264,7 +263,7 @@ std::vector> DnnlExecutionProvider::GetCapabi graph_viewer.ToProto(*model_proto->mutable_graph(), false, true); model_proto->set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION); HashValue model_hash; - int metadef_id = metadef_id_generator_->GenerateId(graph_viewer, model_hash); + int metadef_id = metadef_id_generator_.GenerateId(graph_viewer, model_hash); std::fstream dump("DNNL_" + std::to_string(model_hash) + "_" + std::to_string(metadef_id) + ".onnx", std::ios::out | std::ios::trunc | std::ios::binary); model_proto->SerializeToOstream(dump); } diff --git a/onnxruntime/core/providers/dnnl/dnnl_execution_provider.h b/onnxruntime/core/providers/dnnl/dnnl_execution_provider.h index dd8d6586c7af9..8cb18720eaf1c 100644 --- a/onnxruntime/core/providers/dnnl/dnnl_execution_provider.h +++ b/onnxruntime/core/providers/dnnl/dnnl_execution_provider.h @@ -42,7 +42,7 @@ class DnnlExecutionProvider : public IExecutionProvider { bool debug_log_ = false; // enable fusion by default bool enable_fusion_ = true; - std::unique_ptr metadef_id_generator_; + ModelMetadefIdGenerator metadef_id_generator_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/migraphx/migraphx_execution_provider.cc b/onnxruntime/core/providers/migraphx/migraphx_execution_provider.cc index 02c2ab6c613dd..163dbd928728e 100644 --- a/onnxruntime/core/providers/migraphx/migraphx_execution_provider.cc +++ b/onnxruntime/core/providers/migraphx/migraphx_execution_provider.cc @@ -165,8 +165,6 @@ MIGraphXExecutionProvider::MIGraphXExecutionProvider(const MIGraphXExecutionProv MIOPEN_CALL_THROW(miopenCreate(&external_miopen_handle_)); MIOPEN_CALL_THROW(miopenSetStream(external_miopen_handle_, stream_)); - metadef_id_generator_ = std::make_unique(); - LOGS_DEFAULT(VERBOSE) << "[MIGraphX EP] MIGraphX provider options: " << "device_id: " << device_id_ << ", migraphx_fp16_enable: " << fp16_enable_ @@ -759,7 +757,7 @@ std::unique_ptr MIGraphXExecutionProvider::GetSubGraph(const st // Generate unique kernel name for MIGraphX subgraph uint64_t model_hash = 0; - int id = metadef_id_generator_->GenerateId(graph, model_hash); + int id = metadef_id_generator_.GenerateId(graph, model_hash); std::string subgraph_id = std::to_string(model_hash) + "_" + std::to_string(id); auto meta_def = IndexedSubGraph_MetaDef::Create(); const std::string graph_type = graph.IsSubgraph() ? "subgraph" : "graph"; diff --git a/onnxruntime/core/providers/migraphx/migraphx_execution_provider.h b/onnxruntime/core/providers/migraphx/migraphx_execution_provider.h index 83c42e85d2cec..f7482b071cee8 100644 --- a/onnxruntime/core/providers/migraphx/migraphx_execution_provider.h +++ b/onnxruntime/core/providers/migraphx/migraphx_execution_provider.h @@ -99,7 +99,7 @@ class MIGraphXExecutionProvider : public IExecutionProvider { AllocatorPtr allocator_; miopenHandle_t external_miopen_handle_ = nullptr; rocblas_handle external_rocblas_handle_ = nullptr; - std::unique_ptr metadef_id_generator_; + ModelMetadefIdGenerator metadef_id_generator_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc index 06129ac0fb969..b04703d7611ee 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc @@ -71,8 +71,6 @@ NnapiExecutionProvider::NnapiExecutionProvider(uint32_t nnapi_flags, // May we could just mark this EP as unavailable instead of throwing an error ORT_THROW_IF_ERROR(GetTargetDevices(*nnapi_handle_, target_device_option_, nnapi_target_devices_)); - metadef_id_generator_ = std::make_unique(); - LOGS_DEFAULT(VERBOSE) << "Found devices [" << nnapi::GetDevicesDescription(nnapi_target_devices_) << "] in NNAPI"; } @@ -178,7 +176,7 @@ NnapiExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_view const auto gen_metadef_name = [&]() { HashValue model_hash; - int metadef_id = metadef_id_generator_->GenerateId(graph_viewer, model_hash); + int metadef_id = metadef_id_generator_.GenerateId(graph_viewer, model_hash); return MakeString(NNAPI, "_", model_hash, "_", metadef_id); }; diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h index dd8b43c32e91b..460616c41991f 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h @@ -49,6 +49,6 @@ class NnapiExecutionProvider : public IExecutionProvider { const NnApi* nnapi_handle_ = nullptr; nnapi::DeviceWrapperVector nnapi_target_devices_; nnapi::TargetDeviceOption target_device_option_; - std::unique_ptr metadef_id_generator_; + ModelMetadefIdGenerator metadef_id_generator_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/qnn/qnn_execution_provider.cc b/onnxruntime/core/providers/qnn/qnn_execution_provider.cc index 048c893799819..42ea2d304098a 100644 --- a/onnxruntime/core/providers/qnn/qnn_execution_provider.cc +++ b/onnxruntime/core/providers/qnn/qnn_execution_provider.cc @@ -230,8 +230,6 @@ QNNExecutionProvider::QNNExecutionProvider(const ProviderOptions& provider_optio htp_performance_mode, context_priority, std::move(qnn_saver_path)); - - metadef_id_generator_ = std::make_unique(); } bool QNNExecutionProvider::IsNodeSupported(qnn::QnnModelWrapper& qnn_model_wrapper, const NodeUnit& node_unit, @@ -422,7 +420,7 @@ QNNExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_viewer const auto gen_metadef_name = [&]() { uint64_t model_hash; - int metadef_id = metadef_id_generator_->GenerateId(graph_viewer, model_hash); + int metadef_id = metadef_id_generator_.GenerateId(graph_viewer, model_hash); return MakeString(QNN, "_", model_hash, "_", metadef_id); }; diff --git a/onnxruntime/core/providers/qnn/qnn_execution_provider.h b/onnxruntime/core/providers/qnn/qnn_execution_provider.h index a5f76005bd824..5c7a0d068a31a 100644 --- a/onnxruntime/core/providers/qnn/qnn_execution_provider.h +++ b/onnxruntime/core/providers/qnn/qnn_execution_provider.h @@ -67,7 +67,7 @@ class QNNExecutionProvider : public IExecutionProvider { bool disable_cpu_ep_fallback_ = false; // True if CPU EP fallback has been disabled for this session. bool qnn_context_embed_mode_ = true; int32_t vtcm_size_in_mb_ = 0; - std::unique_ptr metadef_id_generator_; + ModelMetadefIdGenerator metadef_id_generator_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_utils.h b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_utils.h index a8e3ae3ddf6ec..92cce0c203927 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_utils.h +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_utils.h @@ -497,7 +497,15 @@ void RemoveCachesByType(const std::string& root, std::string file_extension) { } } -// Helper class to generate engine id via model name/model content/env metadata +/** + * + * Helper class to generate engine id via model name/model content/env metadata + * + * + * The TensorRT Execution Provider is used in multiple sessions and the underlying infrastructure caches + * compiled kernels, so the name must be unique and deterministic across models and sessions. + * + */ HashValue TRTGenerateId(const GraphViewer& graph_viewer) { HashValue model_hash = 0; diff --git a/onnxruntime/core/providers/webnn/webnn_execution_provider.cc b/onnxruntime/core/providers/webnn/webnn_execution_provider.cc index f2d7c8edd18f3..257a8fa74d930 100644 --- a/onnxruntime/core/providers/webnn/webnn_execution_provider.cc +++ b/onnxruntime/core/providers/webnn/webnn_execution_provider.cc @@ -50,7 +50,6 @@ WebNNExecutionProvider::WebNNExecutionProvider(const std::string& webnn_device_f if (!wnn_builder_.as()) { ORT_THROW("Failed to create WebNN builder."); } - metadef_id_generator_ = std::make_unique(); } WebNNExecutionProvider::~WebNNExecutionProvider() {} @@ -169,7 +168,7 @@ WebNNExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_view // Assign inputs and outputs to subgraph's meta_def. uint64_t model_hash; - int metadef_id = metadef_id_generator_->GenerateId(graph_viewer, model_hash); + int metadef_id = metadef_id_generator_.GenerateId(graph_viewer, model_hash); auto meta_def = std::make_unique<::onnxruntime::IndexedSubGraph::MetaDef>(); meta_def->name = "WEBNN_" + std::to_string(model_hash) + "_" + std::to_string(metadef_id); meta_def->domain = kMSDomain; diff --git a/onnxruntime/core/providers/webnn/webnn_execution_provider.h b/onnxruntime/core/providers/webnn/webnn_execution_provider.h index d7d037fb53e76..d9cfa5f17c0d4 100644 --- a/onnxruntime/core/providers/webnn/webnn_execution_provider.h +++ b/onnxruntime/core/providers/webnn/webnn_execution_provider.h @@ -49,6 +49,6 @@ class WebNNExecutionProvider : public IExecutionProvider { DataLayout preferred_layout_; webnn::WebnnDeviceType wnn_device_type_; InlinedHashMap> models_; - std::unique_ptr metadef_id_generator_; + ModelMetadefIdGenerator metadef_id_generator_; }; } // namespace onnxruntime diff --git a/onnxruntime/test/framework/execution_provider_test.cc b/onnxruntime/test/framework/execution_provider_test.cc index 782e665d6b96e..390fda7bfc5ad 100644 --- a/onnxruntime/test/framework/execution_provider_test.cc +++ b/onnxruntime/test/framework/execution_provider_test.cc @@ -19,16 +19,14 @@ class TestEP : public IExecutionProvider { static constexpr const char* kEPType = "TestEP"; public: - TestEP() : IExecutionProvider{kEPType} { - metadef_id_generator_ = std::make_unique(); - } + TestEP() : IExecutionProvider{kEPType} {} int GetId(const GraphViewer& viewer, HashValue& model_hash) { - return metadef_id_generator_->GenerateId(viewer, model_hash); + return metadef_id_generator_.GenerateId(viewer, model_hash); } private: - std::unique_ptr metadef_id_generator_; + ModelMetadefIdGenerator metadef_id_generator_; }; TEST(ExecutionProviderTest, MetadefIdGeneratorUsingModelPath) { diff --git a/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.cc b/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.cc index ab52ed8803bf7..0167f7a7718b1 100644 --- a/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.cc +++ b/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.cc @@ -91,7 +91,6 @@ InternalTestingExecutionProvider::InternalTestingExecutionProvider(const std::un stop_ops_{stop_ops}, preferred_layout_{preferred_layout}, kernel_registry_{RegisterKernels()} { - metadef_id_generator_ = std::make_unique(); } std::vector InternalTestingExecutionProvider::CreatePreferredAllocators() { @@ -213,7 +212,7 @@ InternalTestingExecutionProvider::GetCapability(const onnxruntime::GraphViewer& // create functor to generate a guaranteed unique metadef id auto generate_metadef_name = [this, &graph_viewer]() { HashValue model_hash; - int metadef_id = metadef_id_generator_->GenerateId(graph_viewer, model_hash); + int metadef_id = metadef_id_generator_.GenerateId(graph_viewer, model_hash); auto meta_def = std::make_unique<::onnxruntime::IndexedSubGraph::MetaDef>(); return ep_name_ + "_" + std::to_string(model_hash) + "_" + std::to_string(metadef_id); }; diff --git a/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.h b/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.h index f3eabf209432b..6615eb82f2b05 100644 --- a/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.h +++ b/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.h @@ -83,7 +83,7 @@ class InternalTestingExecutionProvider : public IExecutionProvider { // per-instance kernel registry so tests using static kernels don't clash. // shared_ptr as required by IExecutionProvider::GetKernelRegistry std::shared_ptr kernel_registry_; - std::unique_ptr metadef_id_generator_; + ModelMetadefIdGenerator metadef_id_generator_; }; } // namespace internal_testing_ep From 793619ae0a4c5710c233718fc786ce7cfd5aa78c Mon Sep 17 00:00:00 2001 From: Lei Cao Date: Sat, 20 Jan 2024 22:19:27 -0800 Subject: [PATCH 07/14] fix pipeline errors (Linux DNNL CI, Win CPU CI x64 rel dnnl) --- onnxruntime/core/framework/model_metadef_id_generator.h | 3 ++- onnxruntime/core/providers/shared_library/provider_api.h | 1 + .../core/providers/shared_library/provider_interfaces.h | 3 +++ .../core/providers/shared_library/provider_wrappedtypes.h | 4 ++++ onnxruntime/core/session/provider_bridge_ort.cc | 4 ++++ 5 files changed, 14 insertions(+), 1 deletion(-) diff --git a/onnxruntime/core/framework/model_metadef_id_generator.h b/onnxruntime/core/framework/model_metadef_id_generator.h index 8445ee3c609a7..2ef056dd4d7ad 100644 --- a/onnxruntime/core/framework/model_metadef_id_generator.h +++ b/onnxruntime/core/framework/model_metadef_id_generator.h @@ -5,7 +5,7 @@ #include #include "core/common/basic_types.h" namespace onnxruntime { -class GraphViewer; +struct GraphViewer; /// /// helper to generate ids that are unique to model and deterministic, even if the execution provider is shared across @@ -23,6 +23,7 @@ class ModelMetadefIdGenerator { int GenerateId(const onnxruntime::GraphViewer& graph_viewer, HashValue& model_hash) const; private: + // mutable as these are caches so we can minimize the hashing required on each usage of GenerateId mutable std::unordered_map main_graph_hash_; // map graph instance hash to model contents hash mutable std::unordered_map model_metadef_id_; // current unique id for model }; diff --git a/onnxruntime/core/providers/shared_library/provider_api.h b/onnxruntime/core/providers/shared_library/provider_api.h index 53ba4874c643c..0208fbe3f0f50 100644 --- a/onnxruntime/core/providers/shared_library/provider_api.h +++ b/onnxruntime/core/providers/shared_library/provider_api.h @@ -157,6 +157,7 @@ struct Tensor; struct SparseTensor; class TensorSeq; class SessionState; +class ModelMetadefIdGenerator; class If; class Loop; diff --git a/onnxruntime/core/providers/shared_library/provider_interfaces.h b/onnxruntime/core/providers/shared_library/provider_interfaces.h index 5242eed254a82..c001d4078389f 100644 --- a/onnxruntime/core/providers/shared_library/provider_interfaces.h +++ b/onnxruntime/core/providers/shared_library/provider_interfaces.h @@ -970,6 +970,9 @@ struct ProviderHost { #if !defined(ORT_MINIMAL_BUILD) || defined(ORT_MINIMAL_BUILD_CUSTOM_OPS) virtual Status LoadDynamicLibrary(onnxruntime::PathString library_name) = 0; #endif + + // ModelMetadefIdGenerator + virtual int ModelMetadefIdGenerator__GenerateId(const ModelMetadefIdGenerator* p, const GraphViewer& graph_viewer, HashValue& model_hash) = 0; }; #if defined(_MSC_VER) && !defined(__clang__) diff --git a/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h b/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h index eaf8ef459cf00..d8327393f7f17 100644 --- a/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h +++ b/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h @@ -1152,6 +1152,10 @@ class TensorSeq final { void Reserve(size_t capacity) { g_host->TensorSeq__Reserve(this, capacity); } }; +class ModelMetadefIdGenerator { + int GenerateId(const GraphViewer& graph_viewer, HashValue& model_hash) const { return g_host->ModelMetadefIdGenerator__GenerateId(this, graph_viewer, model_hash); } +}; + template <> inline gsl::span Tensor::DataAsSpan() const { return g_host->Tensor__DataAsSpan_int64(this); } diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index ed5186bd32826..4bbaa6bf2d196 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -30,6 +30,7 @@ #include "core/framework/sparse_utils.h" #include "core/graph/graph_proto_serializer.h" #include "core/framework/murmurhash3.h" +#include "core/framework/model_metadef_id_generator.h" #include "core/session/onnxruntime_c_api.h" #include "core/common/string_helper.h" @@ -1075,6 +1076,9 @@ struct ProviderHostImpl : ProviderHost { void TensorSeq__Add(TensorSeq* p, Tensor&& tensor) override { p->Add(std::move(tensor)); } void TensorSeq__Reserve(TensorSeq* p, size_t capacity) override { p->Reserve(capacity); } + // ModelMetadefIdGenerator(wrapped) + int ModelMetadefIdGenerator__GenerateId(const ModelMetadefIdGenerator* p, const GraphViewer& graph_viewer, HashValue& model_hash) override { return p->GenerateId(graph_viewer, model_hash); } + #if defined(ENABLE_TRAINING) && defined(ORT_USE_NCCL) training::DistributedRunContext& GetDistributedRunContextInstance() override { return training::DistributedRunContext::GetInstance(); } #endif From fc7632750af6b469b6b1999c45fa884c8387fd5b Mon Sep 17 00:00:00 2001 From: Lei Cao Date: Sat, 20 Jan 2024 23:14:21 -0800 Subject: [PATCH 08/14] dnnl shouldn't include session's ModelMetadefIdGenerator --- onnxruntime/core/framework/model_metadef_id_generator.h | 2 +- onnxruntime/core/providers/dnnl/dnnl_execution_provider.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/onnxruntime/core/framework/model_metadef_id_generator.h b/onnxruntime/core/framework/model_metadef_id_generator.h index 2ef056dd4d7ad..82f68c42b5c35 100644 --- a/onnxruntime/core/framework/model_metadef_id_generator.h +++ b/onnxruntime/core/framework/model_metadef_id_generator.h @@ -5,7 +5,7 @@ #include #include "core/common/basic_types.h" namespace onnxruntime { -struct GraphViewer; +class GraphViewer; /// /// helper to generate ids that are unique to model and deterministic, even if the execution provider is shared across diff --git a/onnxruntime/core/providers/dnnl/dnnl_execution_provider.h b/onnxruntime/core/providers/dnnl/dnnl_execution_provider.h index 8cb18720eaf1c..cbf7bef5fb1ef 100644 --- a/onnxruntime/core/providers/dnnl/dnnl_execution_provider.h +++ b/onnxruntime/core/providers/dnnl/dnnl_execution_provider.h @@ -8,12 +8,12 @@ #include #include -#include "core/framework/model_metadef_id_generator.h" #include "core/providers/dnnl/dnnl_execution_provider_info.h" #include "core/providers/dnnl/dnnl_threadpool.h" #include "core/providers/dnnl/dnnl_op_manager.h" #include "core/providers/dnnl/subgraph/dnnl_subgraph.h" #include "core/providers/dnnl/subgraph/dnnl_subgraph_primitive.h" +#include "core/providers/shared_library/provider_wrappedtypes.h" namespace onnxruntime { From eef463c6a284747fbb4abf4b380e5c9c7305269e Mon Sep 17 00:00:00 2001 From: Lei Cao Date: Sun, 21 Jan 2024 09:09:05 -0800 Subject: [PATCH 09/14] add public decorator in the class ModelMetadefIdGenerator in provider_wrappedtypes --- .../core/providers/shared_library/provider_wrappedtypes.h | 1 + 1 file changed, 1 insertion(+) diff --git a/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h b/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h index d8327393f7f17..17f9245959d12 100644 --- a/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h +++ b/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h @@ -1153,6 +1153,7 @@ class TensorSeq final { }; class ModelMetadefIdGenerator { + public: int GenerateId(const GraphViewer& graph_viewer, HashValue& model_hash) const { return g_host->ModelMetadefIdGenerator__GenerateId(this, graph_viewer, model_hash); } }; From 00535592b09ddbf252e50395faaf5ea90a129db4 Mon Sep 17 00:00:00 2001 From: Lei Cao Date: Sun, 21 Jan 2024 20:28:48 -0800 Subject: [PATCH 10/14] EP refers ModelMetadefIdGenerator in provider_wrappedtypes.h instead of model_metadef_id_generator.h --- onnxruntime/core/providers/cann/cann_execution_provider.h | 2 +- onnxruntime/core/providers/coreml/coreml_execution_provider.h | 2 +- .../core/providers/migraphx/migraphx_execution_provider.h | 2 +- .../providers/nnapi/nnapi_builtin/nnapi_execution_provider.h | 2 +- onnxruntime/core/providers/qnn/qnn_execution_provider.h | 2 +- onnxruntime/core/providers/webnn/webnn_execution_provider.h | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/onnxruntime/core/providers/cann/cann_execution_provider.h b/onnxruntime/core/providers/cann/cann_execution_provider.h index 8fb231fc0c6b3..3e900c966fa74 100644 --- a/onnxruntime/core/providers/cann/cann_execution_provider.h +++ b/onnxruntime/core/providers/cann/cann_execution_provider.h @@ -12,12 +12,12 @@ #include "core/providers/shared_library/provider_api.h" #include "core/framework/arena_extend_strategy.h" #include "core/framework/execution_provider.h" -#include "core/framework/model_metadef_id_generator.h" #include "core/platform/ort_mutex.h" #include "core/providers/cann/cann_execution_provider_info.h" #include "core/providers/cann/cann_inc.h" #include "core/providers/cann/cann_utils.h" #include "core/providers/cann/cann_graph.h" +#include "core/providers/shared_library/provider_wrappedtypes.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/coreml/coreml_execution_provider.h b/onnxruntime/core/providers/coreml/coreml_execution_provider.h index 0201739547dd1..1a40b606b7772 100644 --- a/onnxruntime/core/providers/coreml/coreml_execution_provider.h +++ b/onnxruntime/core/providers/coreml/coreml_execution_provider.h @@ -4,8 +4,8 @@ #pragma once #include "core/framework/execution_provider.h" -#include "core/framework/model_metadef_id_generator.h" #include "core/providers/coreml/coreml_provider_factory.h" +#include "core/providers/shared_library/provider_wrappedtypes.h" namespace onnxruntime { namespace coreml { diff --git a/onnxruntime/core/providers/migraphx/migraphx_execution_provider.h b/onnxruntime/core/providers/migraphx/migraphx_execution_provider.h index f7482b071cee8..e17e1ef9c9c70 100644 --- a/onnxruntime/core/providers/migraphx/migraphx_execution_provider.h +++ b/onnxruntime/core/providers/migraphx/migraphx_execution_provider.h @@ -8,10 +8,10 @@ #include "core/framework/arena_extend_strategy.h" #include "core/framework/execution_provider.h" -#include "core/framework/model_metadef_id_generator.h" #include "core/platform/ort_mutex.h" #include "core/providers/migraphx/migraphx_execution_provider_info.h" #include "core/providers/migraphx/migraphx_inc.h" +#include "core/providers/shared_library/provider_wrappedtypes.h" #include #include diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h index 460616c41991f..7317b6eff2b19 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h @@ -6,9 +6,9 @@ #include "core/common/inlined_containers_fwd.h" #include "core/common/optional.h" #include "core/framework/execution_provider.h" -#include "core/framework/model_metadef_id_generator.h" #include "core/providers/nnapi/nnapi_builtin/nnapi_api_helper.h" #include "core/providers/nnapi/nnapi_provider_factory.h" +#include "core/providers/shared_library/provider_wrappedtypes.h" struct NnApi; namespace onnxruntime { diff --git a/onnxruntime/core/providers/qnn/qnn_execution_provider.h b/onnxruntime/core/providers/qnn/qnn_execution_provider.h index e858e48a0cf2f..ad86881b16905 100644 --- a/onnxruntime/core/providers/qnn/qnn_execution_provider.h +++ b/onnxruntime/core/providers/qnn/qnn_execution_provider.h @@ -5,11 +5,11 @@ #include "core/framework/execution_provider.h" #include "core/framework/session_options.h" -#include "core/framework/model_metadef_id_generator.h" #include #include "core/providers/qnn/builder/qnn_backend_manager.h" #include "core/providers/qnn/builder/qnn_model.h" #include "core/providers/qnn/builder/qnn_graph_configs_helper.h" +#include "core/providers/shared_library/provider_wrappedtypes.h" #include "core/graph/model.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/webnn/webnn_execution_provider.h b/onnxruntime/core/providers/webnn/webnn_execution_provider.h index d9cfa5f17c0d4..c923985325ffd 100644 --- a/onnxruntime/core/providers/webnn/webnn_execution_provider.h +++ b/onnxruntime/core/providers/webnn/webnn_execution_provider.h @@ -6,8 +6,8 @@ #include "core/common/inlined_containers.h" #include "core/framework/execution_provider.h" -#include "core/framework/model_metadef_id_generator.h" #include "core/providers/webnn/builders/helper.h" +#include "core/providers/shared_library/provider_wrappedtypes.h" #include #include From 5a0607fa1399edc2e22d02dd6c66dc48fa2b9484 Mon Sep 17 00:00:00 2001 From: Lei Cao Date: Mon, 22 Jan 2024 10:25:40 -0800 Subject: [PATCH 11/14] include provider_api.h instead of provider_wrappedtypes.h in specific EP's header --- onnxruntime/core/providers/cann/cann_execution_provider.h | 1 - onnxruntime/core/providers/coreml/coreml_execution_provider.h | 2 +- onnxruntime/core/providers/dnnl/dnnl_execution_provider.h | 2 +- .../core/providers/migraphx/migraphx_execution_provider.h | 2 +- .../providers/nnapi/nnapi_builtin/nnapi_execution_provider.h | 2 +- onnxruntime/core/providers/qnn/qnn_execution_provider.h | 2 +- onnxruntime/core/providers/webnn/webnn_execution_provider.h | 2 +- 7 files changed, 6 insertions(+), 7 deletions(-) diff --git a/onnxruntime/core/providers/cann/cann_execution_provider.h b/onnxruntime/core/providers/cann/cann_execution_provider.h index 3e900c966fa74..129c44a5a41de 100644 --- a/onnxruntime/core/providers/cann/cann_execution_provider.h +++ b/onnxruntime/core/providers/cann/cann_execution_provider.h @@ -17,7 +17,6 @@ #include "core/providers/cann/cann_inc.h" #include "core/providers/cann/cann_utils.h" #include "core/providers/cann/cann_graph.h" -#include "core/providers/shared_library/provider_wrappedtypes.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/coreml/coreml_execution_provider.h b/onnxruntime/core/providers/coreml/coreml_execution_provider.h index 1a40b606b7772..a3a0fb23b1ba1 100644 --- a/onnxruntime/core/providers/coreml/coreml_execution_provider.h +++ b/onnxruntime/core/providers/coreml/coreml_execution_provider.h @@ -5,7 +5,7 @@ #include "core/framework/execution_provider.h" #include "core/providers/coreml/coreml_provider_factory.h" -#include "core/providers/shared_library/provider_wrappedtypes.h" +#include "core/providers/shared_library/provider_api.h" namespace onnxruntime { namespace coreml { diff --git a/onnxruntime/core/providers/dnnl/dnnl_execution_provider.h b/onnxruntime/core/providers/dnnl/dnnl_execution_provider.h index cbf7bef5fb1ef..496927d1b7317 100644 --- a/onnxruntime/core/providers/dnnl/dnnl_execution_provider.h +++ b/onnxruntime/core/providers/dnnl/dnnl_execution_provider.h @@ -13,7 +13,7 @@ #include "core/providers/dnnl/dnnl_op_manager.h" #include "core/providers/dnnl/subgraph/dnnl_subgraph.h" #include "core/providers/dnnl/subgraph/dnnl_subgraph_primitive.h" -#include "core/providers/shared_library/provider_wrappedtypes.h" +#include "core/providers/shared_library/provider_api.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/migraphx/migraphx_execution_provider.h b/onnxruntime/core/providers/migraphx/migraphx_execution_provider.h index e17e1ef9c9c70..52c8e4611c188 100644 --- a/onnxruntime/core/providers/migraphx/migraphx_execution_provider.h +++ b/onnxruntime/core/providers/migraphx/migraphx_execution_provider.h @@ -11,7 +11,7 @@ #include "core/platform/ort_mutex.h" #include "core/providers/migraphx/migraphx_execution_provider_info.h" #include "core/providers/migraphx/migraphx_inc.h" -#include "core/providers/shared_library/provider_wrappedtypes.h" +#include "core/providers/shared_library/provider_api.h" #include #include diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h index 7317b6eff2b19..0caed996562a9 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h @@ -8,7 +8,7 @@ #include "core/framework/execution_provider.h" #include "core/providers/nnapi/nnapi_builtin/nnapi_api_helper.h" #include "core/providers/nnapi/nnapi_provider_factory.h" -#include "core/providers/shared_library/provider_wrappedtypes.h" +#include "core/providers/shared_library/provider_api.h" struct NnApi; namespace onnxruntime { diff --git a/onnxruntime/core/providers/qnn/qnn_execution_provider.h b/onnxruntime/core/providers/qnn/qnn_execution_provider.h index ad86881b16905..daf55a83da1fc 100644 --- a/onnxruntime/core/providers/qnn/qnn_execution_provider.h +++ b/onnxruntime/core/providers/qnn/qnn_execution_provider.h @@ -9,7 +9,7 @@ #include "core/providers/qnn/builder/qnn_backend_manager.h" #include "core/providers/qnn/builder/qnn_model.h" #include "core/providers/qnn/builder/qnn_graph_configs_helper.h" -#include "core/providers/shared_library/provider_wrappedtypes.h" +#include "core/providers/shared_library/provider_api.h" #include "core/graph/model.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/webnn/webnn_execution_provider.h b/onnxruntime/core/providers/webnn/webnn_execution_provider.h index c923985325ffd..9900393aeeec6 100644 --- a/onnxruntime/core/providers/webnn/webnn_execution_provider.h +++ b/onnxruntime/core/providers/webnn/webnn_execution_provider.h @@ -7,7 +7,7 @@ #include "core/common/inlined_containers.h" #include "core/framework/execution_provider.h" #include "core/providers/webnn/builders/helper.h" -#include "core/providers/shared_library/provider_wrappedtypes.h" +#include "core/providers/shared_library/provider_api.h" #include #include From 67a65b1186fd08f1b6d9ad13b0521a55ef35a0bb Mon Sep 17 00:00:00 2001 From: Lei Cao Date: Mon, 22 Jan 2024 16:16:13 -0800 Subject: [PATCH 12/14] typo: add forward declaration for ModelMetadefIdGenerator in provider_api.h --- onnxruntime/core/providers/shared_library/provider_api.h | 1 + 1 file changed, 1 insertion(+) diff --git a/onnxruntime/core/providers/shared_library/provider_api.h b/onnxruntime/core/providers/shared_library/provider_api.h index 6d6bd230454d9..1e3a528d87721 100644 --- a/onnxruntime/core/providers/shared_library/provider_api.h +++ b/onnxruntime/core/providers/shared_library/provider_api.h @@ -157,6 +157,7 @@ struct Tensor; struct SparseTensor; class TensorSeq; class SessionState; +class ModelMetadefIdGenerator; class If; class Loop; From 67dab28c05e66f6ee40db117243d15cf515cf720 Mon Sep 17 00:00:00 2001 From: Lei Cao Date: Mon, 22 Jan 2024 17:41:28 -0800 Subject: [PATCH 13/14] add #ifndef SHARED_PROVIDER in model_metadef_id_generator.h to avoid redefinition --- onnxruntime/core/framework/model_metadef_id_generator.h | 4 ++++ onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc | 3 +-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/onnxruntime/core/framework/model_metadef_id_generator.h b/onnxruntime/core/framework/model_metadef_id_generator.h index 82f68c42b5c35..a9b8cd48d0f6c 100644 --- a/onnxruntime/core/framework/model_metadef_id_generator.h +++ b/onnxruntime/core/framework/model_metadef_id_generator.h @@ -1,6 +1,9 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +// if SHARED_PROVIDER is defined (in provider_api.h), use the definition in provider_wrappedytypes.h to avoid redefinition. +// make sure provider_api.h is included before this header +#ifndef SHARED_PROVIDER #pragma once #include #include "core/common/basic_types.h" @@ -29,3 +32,4 @@ class ModelMetadefIdGenerator { }; } // namespace onnxruntime +#endif diff --git a/onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc b/onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc index 4686f58070342..3d5b18540cafc 100644 --- a/onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc +++ b/onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc @@ -5,8 +5,6 @@ #pragma warning(disable : 4996) #endif -#include "core/providers/dnnl/dnnl_execution_provider.h" - #include #include #include @@ -16,6 +14,7 @@ #include "core/platform/ort_mutex.h" #include "core/providers/shared_library/provider_api.h" +#include "core/providers/dnnl/dnnl_execution_provider.h" #include "core/providers/dnnl/dnnl_fwd.h" #include "core/providers/dnnl/dnnl_node_capability.h" From 560b42fb5f0bed1f0cef76f69cb52616a8cef1f0 Mon Sep 17 00:00:00 2001 From: Lei Cao Date: Wed, 24 Jan 2024 15:52:49 -0800 Subject: [PATCH 14/14] use pointer instead of ModelMetadefIdGenerator instance in EP and create from ORT --- onnxruntime/core/framework/model_metadef_id_generator.h | 4 ---- onnxruntime/core/providers/cann/cann_execution_provider.cc | 3 ++- onnxruntime/core/providers/cann/cann_execution_provider.h | 2 +- onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc | 5 +++-- onnxruntime/core/providers/dnnl/dnnl_execution_provider.h | 3 +-- .../core/providers/migraphx/migraphx_execution_provider.cc | 4 +++- .../core/providers/migraphx/migraphx_execution_provider.h | 3 +-- .../core/providers/shared_library/provider_interfaces.h | 2 ++ .../core/providers/shared_library/provider_wrappedtypes.h | 2 ++ onnxruntime/core/session/provider_bridge_ort.cc | 2 ++ 10 files changed, 17 insertions(+), 13 deletions(-) diff --git a/onnxruntime/core/framework/model_metadef_id_generator.h b/onnxruntime/core/framework/model_metadef_id_generator.h index a9b8cd48d0f6c..82f68c42b5c35 100644 --- a/onnxruntime/core/framework/model_metadef_id_generator.h +++ b/onnxruntime/core/framework/model_metadef_id_generator.h @@ -1,9 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -// if SHARED_PROVIDER is defined (in provider_api.h), use the definition in provider_wrappedytypes.h to avoid redefinition. -// make sure provider_api.h is included before this header -#ifndef SHARED_PROVIDER #pragma once #include #include "core/common/basic_types.h" @@ -32,4 +29,3 @@ class ModelMetadefIdGenerator { }; } // namespace onnxruntime -#endif diff --git a/onnxruntime/core/providers/cann/cann_execution_provider.cc b/onnxruntime/core/providers/cann/cann_execution_provider.cc index 74629c95abf2c..752b742805a7c 100644 --- a/onnxruntime/core/providers/cann/cann_execution_provider.cc +++ b/onnxruntime/core/providers/cann/cann_execution_provider.cc @@ -1035,6 +1035,7 @@ CANNExecutionProvider::CANNExecutionProvider(const CANNExecutionProviderInfo& in soc_name_ = aclrtGetSocName(); ORT_ENFORCE(soc_name_ != nullptr, "aclrtGetSocName return nullptr"); + metadef_id_generator_ = ModelMetadefIdGenerator::Create(); } CANNExecutionProvider::~CANNExecutionProvider() { @@ -1196,7 +1197,7 @@ std::unique_ptr CANNExecutionProvider::GetSubGraph( // Generate unique kernel name for CANN subgraph HashValue model_hash = 0; - int id = metadef_id_generator_.GenerateId(graph_viewer, model_hash); + int id = metadef_id_generator_->GenerateId(graph_viewer, model_hash); auto meta_def = IndexedSubGraph_MetaDef::Create(); meta_def->name() = graph_viewer.Name() + "_" + std::to_string(model_hash) + "_" + std::to_string(id); diff --git a/onnxruntime/core/providers/cann/cann_execution_provider.h b/onnxruntime/core/providers/cann/cann_execution_provider.h index 129c44a5a41de..63ae980869c65 100644 --- a/onnxruntime/core/providers/cann/cann_execution_provider.h +++ b/onnxruntime/core/providers/cann/cann_execution_provider.h @@ -81,7 +81,7 @@ class CANNExecutionProvider : public IExecutionProvider { std::unordered_map modelIDs_; std::unordered_map models_; std::unordered_map> names_; - ModelMetadefIdGenerator metadef_id_generator_; + std::unique_ptr metadef_id_generator_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc b/onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc index 3d5b18540cafc..3271dab13f675 100644 --- a/onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc +++ b/onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc @@ -76,6 +76,7 @@ DnnlExecutionProvider::DnnlExecutionProvider(const DnnlExecutionProviderInfo& in // Log the number of threads used LOGS_DEFAULT(INFO) << "Allocated " << omp_get_max_threads() << " OpenMP threads for oneDNN ep\n"; #endif // defined(DNNL_OPENMP) + metadef_id_generator_ = ModelMetadefIdGenerator::Create(); } DnnlExecutionProvider::~DnnlExecutionProvider() { @@ -227,7 +228,7 @@ std::vector> DnnlExecutionProvider::GetCapabi // Assign inputs and outputs to subgraph's meta_def HashValue model_hash; - int metadef_id = metadef_id_generator_.GenerateId(graph_viewer, model_hash); + int metadef_id = metadef_id_generator_->GenerateId(graph_viewer, model_hash); auto meta_def = ::onnxruntime::IndexedSubGraph_MetaDef::Create(); meta_def->name() = "DNNL_" + std::to_string(model_hash) + "_" + std::to_string(metadef_id); meta_def->domain() = kMSDomain; @@ -262,7 +263,7 @@ std::vector> DnnlExecutionProvider::GetCapabi graph_viewer.ToProto(*model_proto->mutable_graph(), false, true); model_proto->set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION); HashValue model_hash; - int metadef_id = metadef_id_generator_.GenerateId(graph_viewer, model_hash); + int metadef_id = metadef_id_generator_->GenerateId(graph_viewer, model_hash); std::fstream dump("DNNL_" + std::to_string(model_hash) + "_" + std::to_string(metadef_id) + ".onnx", std::ios::out | std::ios::trunc | std::ios::binary); model_proto->SerializeToOstream(dump); } diff --git a/onnxruntime/core/providers/dnnl/dnnl_execution_provider.h b/onnxruntime/core/providers/dnnl/dnnl_execution_provider.h index f7e4a5e380cd7..b7fcbb7765180 100644 --- a/onnxruntime/core/providers/dnnl/dnnl_execution_provider.h +++ b/onnxruntime/core/providers/dnnl/dnnl_execution_provider.h @@ -13,7 +13,6 @@ #include "core/providers/dnnl/dnnl_op_manager.h" #include "core/providers/dnnl/subgraph/dnnl_subgraph.h" #include "core/providers/dnnl/subgraph/dnnl_subgraph_primitive.h" -#include "core/framework/model_metadef_id_generator.h" namespace onnxruntime { @@ -42,7 +41,7 @@ class DnnlExecutionProvider : public IExecutionProvider { bool debug_log_ = false; // enable fusion by default bool enable_fusion_ = true; - ModelMetadefIdGenerator metadef_id_generator_; + std::unique_ptr metadef_id_generator_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/migraphx/migraphx_execution_provider.cc b/onnxruntime/core/providers/migraphx/migraphx_execution_provider.cc index 163dbd928728e..40e76a0a67782 100644 --- a/onnxruntime/core/providers/migraphx/migraphx_execution_provider.cc +++ b/onnxruntime/core/providers/migraphx/migraphx_execution_provider.cc @@ -165,6 +165,8 @@ MIGraphXExecutionProvider::MIGraphXExecutionProvider(const MIGraphXExecutionProv MIOPEN_CALL_THROW(miopenCreate(&external_miopen_handle_)); MIOPEN_CALL_THROW(miopenSetStream(external_miopen_handle_, stream_)); + metadef_id_generator_ = ModelMetadefIdGenerator::Create(); + LOGS_DEFAULT(VERBOSE) << "[MIGraphX EP] MIGraphX provider options: " << "device_id: " << device_id_ << ", migraphx_fp16_enable: " << fp16_enable_ @@ -757,7 +759,7 @@ std::unique_ptr MIGraphXExecutionProvider::GetSubGraph(const st // Generate unique kernel name for MIGraphX subgraph uint64_t model_hash = 0; - int id = metadef_id_generator_.GenerateId(graph, model_hash); + int id = metadef_id_generator_->GenerateId(graph, model_hash); std::string subgraph_id = std::to_string(model_hash) + "_" + std::to_string(id); auto meta_def = IndexedSubGraph_MetaDef::Create(); const std::string graph_type = graph.IsSubgraph() ? "subgraph" : "graph"; diff --git a/onnxruntime/core/providers/migraphx/migraphx_execution_provider.h b/onnxruntime/core/providers/migraphx/migraphx_execution_provider.h index f7482b071cee8..d582338c7e067 100644 --- a/onnxruntime/core/providers/migraphx/migraphx_execution_provider.h +++ b/onnxruntime/core/providers/migraphx/migraphx_execution_provider.h @@ -8,7 +8,6 @@ #include "core/framework/arena_extend_strategy.h" #include "core/framework/execution_provider.h" -#include "core/framework/model_metadef_id_generator.h" #include "core/platform/ort_mutex.h" #include "core/providers/migraphx/migraphx_execution_provider_info.h" #include "core/providers/migraphx/migraphx_inc.h" @@ -99,7 +98,7 @@ class MIGraphXExecutionProvider : public IExecutionProvider { AllocatorPtr allocator_; miopenHandle_t external_miopen_handle_ = nullptr; rocblas_handle external_rocblas_handle_ = nullptr; - ModelMetadefIdGenerator metadef_id_generator_; + std::unique_ptr metadef_id_generator_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/shared_library/provider_interfaces.h b/onnxruntime/core/providers/shared_library/provider_interfaces.h index c001d4078389f..a216b2bfc6d04 100644 --- a/onnxruntime/core/providers/shared_library/provider_interfaces.h +++ b/onnxruntime/core/providers/shared_library/provider_interfaces.h @@ -972,6 +972,8 @@ struct ProviderHost { #endif // ModelMetadefIdGenerator + virtual std::unique_ptr ModelMetadefIdGenerator__construct() = 0; + virtual void ModelMetadefIdGenerator__operator_delete(ModelMetadefIdGenerator* p) = 0; virtual int ModelMetadefIdGenerator__GenerateId(const ModelMetadefIdGenerator* p, const GraphViewer& graph_viewer, HashValue& model_hash) = 0; }; diff --git a/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h b/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h index 386ac45d8b7f7..f46c76fd3421b 100644 --- a/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h +++ b/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h @@ -1155,6 +1155,8 @@ class TensorSeq final { class ModelMetadefIdGenerator { public: + static std::unique_ptr Create() { return g_host->ModelMetadefIdGenerator__construct(); } + static void operator delete(void* p) { g_host->ModelMetadefIdGenerator__operator_delete(reinterpret_cast(p)); } int GenerateId(const GraphViewer& graph_viewer, HashValue& model_hash) const { return g_host->ModelMetadefIdGenerator__GenerateId(this, graph_viewer, model_hash); } }; diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index 824b5804688ce..f8bd7f4aa208c 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -1081,6 +1081,8 @@ struct ProviderHostImpl : ProviderHost { void TensorSeq__Reserve(TensorSeq* p, size_t capacity) override { p->Reserve(capacity); } // ModelMetadefIdGenerator(wrapped) + std::unique_ptr ModelMetadefIdGenerator__construct() override { return std::make_unique(); } + void ModelMetadefIdGenerator__operator_delete(ModelMetadefIdGenerator* p) override { delete p; } int ModelMetadefIdGenerator__GenerateId(const ModelMetadefIdGenerator* p, const GraphViewer& graph_viewer, HashValue& model_hash) override { return p->GenerateId(graph_viewer, model_hash); } #if defined(ENABLE_TRAINING) && defined(ORT_USE_NCCL)