diff --git a/include/onnxruntime/core/framework/execution_provider.h b/include/onnxruntime/core/framework/execution_provider.h index 1de0217c7e1fa..31c988f500779 100644 --- a/include/onnxruntime/core/framework/execution_provider.h +++ b/include/onnxruntime/core/framework/execution_provider.h @@ -59,14 +59,11 @@ enum class DataLayout { class IExecutionProvider { protected: - IExecutionProvider(const std::string& type, bool use_metadef_id_creator = false) - : IExecutionProvider(type, OrtDevice(), use_metadef_id_creator) {} + IExecutionProvider(const std::string& type) + : IExecutionProvider(type, OrtDevice()) {} - IExecutionProvider(const std::string& type, OrtDevice device, bool use_metadef_id_creator = false) + IExecutionProvider(const std::string& type, OrtDevice device) : default_device_(device), type_{type} { - if (use_metadef_id_creator) { - metadef_id_generator_ = std::make_unique(); - } } /* @@ -274,19 +271,6 @@ class IExecutionProvider { return logger_; } - /** Generate a unique id that can be used in a MetaDef name. Values are unique for a model instance. - The model hash is also returned if you wish to include that in the MetaDef name to ensure uniqueness across models. - @param graph_viewer[in] Graph viewer that GetCapability was called with. Can be for the main graph or nested graph. - @param model_hash[out] Returns the hash for the main (i.e. top level) graph in the model. - This is created using the model path if available, - or the model input names and the output names from all nodes in the main graph. - @remarks e.g. the TensorRT Execution Provider is used in multiple sessions and the underlying infrastructure caches - compiled kernels, so the name must be unique and deterministic across models and sessions. - NOTE: Ideally this would be a protected method, but to work across the EP bridge it has to be public and - virtual, and ModelMetadefIdGenerator but be defined in the header as well. - */ - virtual int GenerateMetaDefId(const onnxruntime::GraphViewer& graph_viewer, HashValue& model_hash) const; - virtual std::unique_ptr GetProfiler() { return {}; } @@ -340,18 +324,5 @@ class IExecutionProvider { // It will be set when this object is registered to a session const logging::Logger* logger_ = nullptr; - - // helper to generate ids that are unique to model and deterministic, even if the execution provider is shared across - // multiple sessions. - class ModelMetadefIdGenerator { - public: - int GenerateId(const onnxruntime::GraphViewer& graph_viewer, HashValue& model_hash); - - private: - std::unordered_map main_graph_hash_; // map graph instance hash to model contents hash - std::unordered_map model_metadef_id_; // current unique id for model - }; - - std::unique_ptr metadef_id_generator_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/framework/execution_provider.cc b/onnxruntime/core/framework/execution_provider.cc index 7f8009216ce3a..b39924d4c3ff9 100644 --- a/onnxruntime/core/framework/execution_provider.cc +++ b/onnxruntime/core/framework/execution_provider.cc @@ -35,77 +35,4 @@ common::Status IExecutionProvider::Compile(const std::vector& } #endif - -int IExecutionProvider::ModelMetadefIdGenerator::GenerateId(const onnxruntime::GraphViewer& graph_viewer, - HashValue& model_hash) { - model_hash = 0; - - // find the top level graph - const Graph* cur_graph = &graph_viewer.GetGraph(); - while (cur_graph->IsSubgraph()) { - cur_graph = cur_graph->ParentGraph(); - } - - uint32_t instance_hash[4] = {0, 0, 0, 0}; - - const Graph& main_graph = *cur_graph; - - // hash the bytes in the Graph instance. we can't just use the address as a new Graph instance may use - // the same memory (unit tests prove this can occur). the raw bytes of the Graph instance should be a unique - // fingerprint for the instance that can use used as the key to the hash of the model path/contents. - MurmurHash3::x86_128(&main_graph, gsl::narrow_cast(sizeof(Graph)), instance_hash[0], &instance_hash); - HashValue graph_instance_hash = instance_hash[0] | (uint64_t(instance_hash[1]) << 32); - - // if we've already hashed this main graph instance use the cached value - auto entry = main_graph_hash_.find(graph_instance_hash); - if (entry != main_graph_hash_.cend()) { - model_hash = entry->second; - } else { - uint32_t hash[4] = {0, 0, 0, 0}; - - // prefer path the model was loaded from - // this may not be available if the model was loaded from a stream or in-memory bytes - const auto& model_path_str = main_graph.ModelPath().ToPathString(); - if (!model_path_str.empty()) { - MurmurHash3::x86_128(model_path_str.data(), gsl::narrow_cast(model_path_str.size()), hash[0], &hash); - } else { - auto hash_str = [&hash](const std::string& str) { - MurmurHash3::x86_128(str.data(), gsl::narrow_cast(str.size()), hash[0], &hash); - }; - - // fingerprint the main graph by hashing graph inputs and the ordered outputs from each node - for (const auto* node_arg : main_graph.GetInputsIncludingInitializers()) { - hash_str(node_arg->Name()); - } - - // note: process nodes in order defined in model to be deterministic - for (const auto& node : main_graph.Nodes()) { - for (const auto* node_arg : node.OutputDefs()) { - if (node_arg->Exists()) { - hash_str(node_arg->Name()); - } - } - } - } - - model_hash = hash[0] | (uint64_t(hash[1]) << 32); - - main_graph_hash_[graph_instance_hash] = model_hash; - } - - // return the current unique id, and increment to update - return model_metadef_id_[model_hash]++; -} - -int IExecutionProvider::GenerateMetaDefId(const onnxruntime::GraphViewer& graph_viewer, HashValue& model_hash) const { - ORT_ENFORCE(metadef_id_generator_, - "IExecutionProvider constructor must be called with true for use_metadef_id_creator"); - - // if the EP is shared across multiple sessions there's a very small potential for concurrency issues. - // use a lock when generating an id to be paranoid - static OrtMutex mutex; - std::lock_guard lock(mutex); - return metadef_id_generator_->GenerateId(graph_viewer, model_hash); -} - } // namespace onnxruntime diff --git a/onnxruntime/core/framework/model_metadef_id_generator.cc b/onnxruntime/core/framework/model_metadef_id_generator.cc new file mode 100644 index 0000000000000..e51c6ebc29975 --- /dev/null +++ b/onnxruntime/core/framework/model_metadef_id_generator.cc @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#include +#include "model_metadef_id_generator.h" +#include "core/platform/ort_mutex.h" +#include "core/graph/graph_viewer.h" +#include "core/framework/murmurhash3.h" + +namespace onnxruntime { +int ModelMetadefIdGenerator::GenerateId(const onnxruntime::GraphViewer& graph_viewer, + HashValue& model_hash) const { + // if the EP is shared across multiple sessions there's a very small potential for concurrency issues. + // use a lock when generating an id to be paranoid + static OrtMutex mutex; + std::lock_guard lock(mutex); + model_hash = 0; + + // find the top level graph + const Graph* cur_graph = &graph_viewer.GetGraph(); + while (cur_graph->IsSubgraph()) { + cur_graph = cur_graph->ParentGraph(); + } + + uint32_t instance_hash[4] = {0, 0, 0, 0}; + + const Graph& main_graph = *cur_graph; + + // hash the bytes in the Graph instance. we can't just use the address as a new Graph instance may use + // the same memory (unit tests prove this can occur). the raw bytes of the Graph instance should be a unique + // fingerprint for the instance that can use used as the key to the hash of the model path/contents. + MurmurHash3::x86_128(&main_graph, gsl::narrow_cast(sizeof(Graph)), instance_hash[0], &instance_hash); + HashValue graph_instance_hash = instance_hash[0] | (uint64_t(instance_hash[1]) << 32); + + // if we've already hashed this main graph instance use the cached value + auto entry = main_graph_hash_.find(graph_instance_hash); + if (entry != main_graph_hash_.cend()) { + model_hash = entry->second; + } else { + uint32_t hash[4] = {0, 0, 0, 0}; + + // prefer path the model was loaded from + // this may not be available if the model was loaded from a stream or in-memory bytes + const auto& model_path_str = main_graph.ModelPath().ToPathString(); + if (!model_path_str.empty()) { + MurmurHash3::x86_128(model_path_str.data(), gsl::narrow_cast(model_path_str.size()), hash[0], &hash); + } else { + auto hash_str = [&hash](const std::string& str) { + MurmurHash3::x86_128(str.data(), gsl::narrow_cast(str.size()), hash[0], &hash); + }; + + // fingerprint the main graph by hashing graph inputs and the ordered outputs from each node + for (const auto* node_arg : main_graph.GetInputsIncludingInitializers()) { + hash_str(node_arg->Name()); + } + + // note: process nodes in order defined in model to be deterministic + for (const auto& node : main_graph.Nodes()) { + for (const auto* node_arg : node.OutputDefs()) { + if (node_arg->Exists()) { + hash_str(node_arg->Name()); + } + } + } + } + + model_hash = hash[0] | (uint64_t(hash[1]) << 32); + + main_graph_hash_[graph_instance_hash] = model_hash; + } + + // return the current unique id, and increment to update + return model_metadef_id_[model_hash]++; +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/model_metadef_id_generator.h b/onnxruntime/core/framework/model_metadef_id_generator.h new file mode 100644 index 0000000000000..82f68c42b5c35 --- /dev/null +++ b/onnxruntime/core/framework/model_metadef_id_generator.h @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include "core/common/basic_types.h" +namespace onnxruntime { +class GraphViewer; + +/// +/// helper to generate ids that are unique to model and deterministic, even if the execution provider is shared across +/// multiple sessions. +/// +class ModelMetadefIdGenerator { + public: + /** Generate a unique id that can be used in a MetaDef name. Values are unique for a model instance. + The model hash is also returned if you wish to include that in the MetaDef name to ensure uniqueness across models. + @param graph_viewer[in] Graph viewer that GetCapability was called with. Can be for the main graph or nested graph. + @param model_hash[out] Returns the hash for the main (i.e. top level) graph in the model. + This is created using the model path if available, + or the model input names and the output names from all nodes in the main graph. + */ + int GenerateId(const onnxruntime::GraphViewer& graph_viewer, HashValue& model_hash) const; + + private: + // mutable as these are caches so we can minimize the hashing required on each usage of GenerateId + mutable std::unordered_map main_graph_hash_; // map graph instance hash to model contents hash + mutable std::unordered_map model_metadef_id_; // current unique id for model +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cann/cann_execution_provider.cc b/onnxruntime/core/providers/cann/cann_execution_provider.cc index 127c37bd84d0f..752b742805a7c 100644 --- a/onnxruntime/core/providers/cann/cann_execution_provider.cc +++ b/onnxruntime/core/providers/cann/cann_execution_provider.cc @@ -9,7 +9,6 @@ #include #include -#include "core/providers/shared_library/provider_api.h" #define ORT_API_MANUAL_INIT #include "core/session/onnxruntime_cxx_api.h" #include "core/providers/cann/cann_execution_provider.h" @@ -1029,13 +1028,14 @@ Status RegisterCANNKernels(KernelRegistry& kernel_registry) { } // namespace cann CANNExecutionProvider::CANNExecutionProvider(const CANNExecutionProviderInfo& info) - : IExecutionProvider{onnxruntime::kCannExecutionProvider, OrtDevice(OrtDevice::NPU, OrtDevice::MemType::DEFAULT, info.device_id), true}, info_{info} { + : IExecutionProvider{onnxruntime::kCannExecutionProvider, OrtDevice(OrtDevice::NPU, OrtDevice::MemType::DEFAULT, info.device_id)}, info_{info} { InitProviderOrtApi(); CANN_CALL_THROW(aclrtSetDevice(info_.device_id)); soc_name_ = aclrtGetSocName(); ORT_ENFORCE(soc_name_ != nullptr, "aclrtGetSocName return nullptr"); + metadef_id_generator_ = ModelMetadefIdGenerator::Create(); } CANNExecutionProvider::~CANNExecutionProvider() { @@ -1197,7 +1197,7 @@ std::unique_ptr CANNExecutionProvider::GetSubGraph( // Generate unique kernel name for CANN subgraph HashValue model_hash = 0; - int id = GenerateMetaDefId(graph_viewer, model_hash); + int id = metadef_id_generator_->GenerateId(graph_viewer, model_hash); auto meta_def = IndexedSubGraph_MetaDef::Create(); meta_def->name() = graph_viewer.Name() + "_" + std::to_string(model_hash) + "_" + std::to_string(id); diff --git a/onnxruntime/core/providers/cann/cann_execution_provider.h b/onnxruntime/core/providers/cann/cann_execution_provider.h index 76d3d9c331563..63ae980869c65 100644 --- a/onnxruntime/core/providers/cann/cann_execution_provider.h +++ b/onnxruntime/core/providers/cann/cann_execution_provider.h @@ -81,6 +81,7 @@ class CANNExecutionProvider : public IExecutionProvider { std::unordered_map modelIDs_; std::unordered_map models_; std::unordered_map> names_; + std::unique_ptr metadef_id_generator_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/coreml/coreml_execution_provider.cc b/onnxruntime/core/providers/coreml/coreml_execution_provider.cc index c9973671ffa28..c133f7b82aba4 100644 --- a/onnxruntime/core/providers/coreml/coreml_execution_provider.cc +++ b/onnxruntime/core/providers/coreml/coreml_execution_provider.cc @@ -24,7 +24,7 @@ namespace onnxruntime { constexpr const char* COREML = "CoreML"; CoreMLExecutionProvider::CoreMLExecutionProvider(uint32_t coreml_flags) - : IExecutionProvider{onnxruntime::kCoreMLExecutionProvider, true}, + : IExecutionProvider{onnxruntime::kCoreMLExecutionProvider}, coreml_flags_(coreml_flags) { } @@ -54,7 +54,7 @@ CoreMLExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_vie const auto gen_metadef_name = [&]() { HashValue model_hash; - int metadef_id = GenerateMetaDefId(graph_viewer, model_hash); + int metadef_id = metadef_id_generator_.GenerateId(graph_viewer, model_hash); return MakeString(COREML, "_", model_hash, "_", metadef_id); }; diff --git a/onnxruntime/core/providers/coreml/coreml_execution_provider.h b/onnxruntime/core/providers/coreml/coreml_execution_provider.h index 67050e8079cf9..0201739547dd1 100644 --- a/onnxruntime/core/providers/coreml/coreml_execution_provider.h +++ b/onnxruntime/core/providers/coreml/coreml_execution_provider.h @@ -4,6 +4,7 @@ #pragma once #include "core/framework/execution_provider.h" +#include "core/framework/model_metadef_id_generator.h" #include "core/providers/coreml/coreml_provider_factory.h" namespace onnxruntime { @@ -34,5 +35,6 @@ class CoreMLExecutionProvider : public IExecutionProvider { #ifdef __APPLE__ std::unordered_map> coreml_models_; #endif + ModelMetadefIdGenerator metadef_id_generator_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc b/onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc index 05eb0091a8c83..3271dab13f675 100644 --- a/onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc +++ b/onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc @@ -5,8 +5,6 @@ #pragma warning(disable : 4996) #endif -#include "core/providers/dnnl/dnnl_execution_provider.h" - #include #include #include @@ -16,6 +14,7 @@ #include "core/platform/ort_mutex.h" #include "core/providers/shared_library/provider_api.h" +#include "core/providers/dnnl/dnnl_execution_provider.h" #include "core/providers/dnnl/dnnl_fwd.h" #include "core/providers/dnnl/dnnl_node_capability.h" @@ -30,7 +29,7 @@ constexpr const char* DNNL = "Dnnl"; constexpr const char* DNNL_CPU = "DnnlCpu"; DnnlExecutionProvider::DnnlExecutionProvider(const DnnlExecutionProviderInfo& info) - : IExecutionProvider{onnxruntime::kDnnlExecutionProvider, true}, + : IExecutionProvider{onnxruntime::kDnnlExecutionProvider}, info_(info) { InitProviderOrtApi(); @@ -77,8 +76,8 @@ DnnlExecutionProvider::DnnlExecutionProvider(const DnnlExecutionProviderInfo& in // Log the number of threads used LOGS_DEFAULT(INFO) << "Allocated " << omp_get_max_threads() << " OpenMP threads for oneDNN ep\n"; #endif // defined(DNNL_OPENMP) - -} // namespace onnxruntime + metadef_id_generator_ = ModelMetadefIdGenerator::Create(); +} DnnlExecutionProvider::~DnnlExecutionProvider() { } @@ -229,7 +228,7 @@ std::vector> DnnlExecutionProvider::GetCapabi // Assign inputs and outputs to subgraph's meta_def HashValue model_hash; - int metadef_id = GenerateMetaDefId(graph_viewer, model_hash); + int metadef_id = metadef_id_generator_->GenerateId(graph_viewer, model_hash); auto meta_def = ::onnxruntime::IndexedSubGraph_MetaDef::Create(); meta_def->name() = "DNNL_" + std::to_string(model_hash) + "_" + std::to_string(metadef_id); meta_def->domain() = kMSDomain; @@ -264,7 +263,7 @@ std::vector> DnnlExecutionProvider::GetCapabi graph_viewer.ToProto(*model_proto->mutable_graph(), false, true); model_proto->set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION); HashValue model_hash; - int metadef_id = GenerateMetaDefId(graph_viewer, model_hash); + int metadef_id = metadef_id_generator_->GenerateId(graph_viewer, model_hash); std::fstream dump("DNNL_" + std::to_string(model_hash) + "_" + std::to_string(metadef_id) + ".onnx", std::ios::out | std::ios::trunc | std::ios::binary); model_proto->SerializeToOstream(dump); } diff --git a/onnxruntime/core/providers/dnnl/dnnl_execution_provider.h b/onnxruntime/core/providers/dnnl/dnnl_execution_provider.h index 41062ccb4bc1b..b7fcbb7765180 100644 --- a/onnxruntime/core/providers/dnnl/dnnl_execution_provider.h +++ b/onnxruntime/core/providers/dnnl/dnnl_execution_provider.h @@ -41,6 +41,7 @@ class DnnlExecutionProvider : public IExecutionProvider { bool debug_log_ = false; // enable fusion by default bool enable_fusion_ = true; + std::unique_ptr metadef_id_generator_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/js/js_execution_provider.cc b/onnxruntime/core/providers/js/js_execution_provider.cc index c2ff2ebc39e13..21bf09b274aff 100644 --- a/onnxruntime/core/providers/js/js_execution_provider.cc +++ b/onnxruntime/core/providers/js/js_execution_provider.cc @@ -680,7 +680,7 @@ std::unique_ptr RegisterKernels() { using namespace js; JsExecutionProvider::JsExecutionProvider(const JsExecutionProviderInfo& info) - : IExecutionProvider{kJsExecutionProvider, OrtDevice(OrtDevice::GPU, OrtDevice::MemType::DEFAULT, 0), true}, + : IExecutionProvider{kJsExecutionProvider, OrtDevice(OrtDevice::GPU, OrtDevice::MemType::DEFAULT, 0)}, preferred_data_layout_{info.data_layout} { } diff --git a/onnxruntime/core/providers/migraphx/migraphx_execution_provider.cc b/onnxruntime/core/providers/migraphx/migraphx_execution_provider.cc index 8bfa66710e2fc..40e76a0a67782 100644 --- a/onnxruntime/core/providers/migraphx/migraphx_execution_provider.cc +++ b/onnxruntime/core/providers/migraphx/migraphx_execution_provider.cc @@ -102,7 +102,7 @@ std::shared_ptr MIGraphXExecutionProvider::GetKernelRegistry() c } MIGraphXExecutionProvider::MIGraphXExecutionProvider(const MIGraphXExecutionProviderInfo& info) - : IExecutionProvider{onnxruntime::kMIGraphXExecutionProvider, OrtDevice(OrtDevice::GPU, OrtDevice::MemType::DEFAULT, info.device_id), true}, device_id_(info.device_id) { + : IExecutionProvider{onnxruntime::kMIGraphXExecutionProvider, OrtDevice(OrtDevice::GPU, OrtDevice::MemType::DEFAULT, info.device_id)}, device_id_(info.device_id) { InitProviderOrtApi(); // Set GPU device to be used HIP_CALL_THROW(hipSetDevice(device_id_)); @@ -165,6 +165,8 @@ MIGraphXExecutionProvider::MIGraphXExecutionProvider(const MIGraphXExecutionProv MIOPEN_CALL_THROW(miopenCreate(&external_miopen_handle_)); MIOPEN_CALL_THROW(miopenSetStream(external_miopen_handle_, stream_)); + metadef_id_generator_ = ModelMetadefIdGenerator::Create(); + LOGS_DEFAULT(VERBOSE) << "[MIGraphX EP] MIGraphX provider options: " << "device_id: " << device_id_ << ", migraphx_fp16_enable: " << fp16_enable_ @@ -757,7 +759,7 @@ std::unique_ptr MIGraphXExecutionProvider::GetSubGraph(const st // Generate unique kernel name for MIGraphX subgraph uint64_t model_hash = 0; - int id = GenerateMetaDefId(graph, model_hash); + int id = metadef_id_generator_->GenerateId(graph, model_hash); std::string subgraph_id = std::to_string(model_hash) + "_" + std::to_string(id); auto meta_def = IndexedSubGraph_MetaDef::Create(); const std::string graph_type = graph.IsSubgraph() ? "subgraph" : "graph"; diff --git a/onnxruntime/core/providers/migraphx/migraphx_execution_provider.h b/onnxruntime/core/providers/migraphx/migraphx_execution_provider.h index c094be51012e4..d582338c7e067 100644 --- a/onnxruntime/core/providers/migraphx/migraphx_execution_provider.h +++ b/onnxruntime/core/providers/migraphx/migraphx_execution_provider.h @@ -98,6 +98,7 @@ class MIGraphXExecutionProvider : public IExecutionProvider { AllocatorPtr allocator_; miopenHandle_t external_miopen_handle_ = nullptr; rocblas_handle external_rocblas_handle_ = nullptr; + std::unique_ptr metadef_id_generator_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc index 727917ad9232e..b04703d7611ee 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc @@ -50,7 +50,7 @@ std::unordered_set GetPartitioningStopOps(const optional& partitioning_stop_ops_list) - : IExecutionProvider{onnxruntime::kNnapiExecutionProvider, true}, + : IExecutionProvider{onnxruntime::kNnapiExecutionProvider}, nnapi_flags_(nnapi_flags), partitioning_stop_ops_(GetPartitioningStopOps(partitioning_stop_ops_list)) { nnapi_handle_ = NnApiImplementation(); @@ -176,7 +176,7 @@ NnapiExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_view const auto gen_metadef_name = [&]() { HashValue model_hash; - int metadef_id = GenerateMetaDefId(graph_viewer, model_hash); + int metadef_id = metadef_id_generator_.GenerateId(graph_viewer, model_hash); return MakeString(NNAPI, "_", model_hash, "_", metadef_id); }; diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h index e4911511e6db0..460616c41991f 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h @@ -6,6 +6,7 @@ #include "core/common/inlined_containers_fwd.h" #include "core/common/optional.h" #include "core/framework/execution_provider.h" +#include "core/framework/model_metadef_id_generator.h" #include "core/providers/nnapi/nnapi_builtin/nnapi_api_helper.h" #include "core/providers/nnapi/nnapi_provider_factory.h" @@ -48,5 +49,6 @@ class NnapiExecutionProvider : public IExecutionProvider { const NnApi* nnapi_handle_ = nullptr; nnapi::DeviceWrapperVector nnapi_target_devices_; nnapi::TargetDeviceOption target_device_option_; + ModelMetadefIdGenerator metadef_id_generator_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/partitioning_utils.h b/onnxruntime/core/providers/partitioning_utils.h index f9d5f7403f17b..136725c2f7250 100644 --- a/onnxruntime/core/providers/partitioning_utils.h +++ b/onnxruntime/core/providers/partitioning_utils.h @@ -40,7 +40,7 @@ using OnGroupClosedFn = std::function& group /** Called to create a metadef name. -Most likely should call IExecutionProvider::GenerateMetaDefId. +Most likely should call ModelMetadefIdGenerator.GenerateId. See onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.cc for example usage. @return The metadef name. diff --git a/onnxruntime/core/providers/qnn/qnn_execution_provider.cc b/onnxruntime/core/providers/qnn/qnn_execution_provider.cc index 0310cc2bc8f26..5f4e2e62f063e 100644 --- a/onnxruntime/core/providers/qnn/qnn_execution_provider.cc +++ b/onnxruntime/core/providers/qnn/qnn_execution_provider.cc @@ -129,7 +129,7 @@ static void ParseHtpArchitecture(const std::string& htp_arch_string, QnnHtpDevic QNNExecutionProvider::QNNExecutionProvider(const ProviderOptions& provider_options_map, const SessionOptions* session_options) - : IExecutionProvider{onnxruntime::kQnnExecutionProvider, true} { + : IExecutionProvider{onnxruntime::kQnnExecutionProvider} { if (session_options) { disable_cpu_ep_fallback_ = session_options->config_options.GetConfigOrDefault( kOrtSessionOptionsDisableCPUEPFallback, "0") == "1"; @@ -472,7 +472,7 @@ QNNExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_viewer const auto gen_metadef_name = [&]() { uint64_t model_hash; - int metadef_id = GenerateMetaDefId(graph_viewer, model_hash); + int metadef_id = metadef_id_generator_.GenerateId(graph_viewer, model_hash); return MakeString(QNN, "_", model_hash, "_", metadef_id); }; diff --git a/onnxruntime/core/providers/qnn/qnn_execution_provider.h b/onnxruntime/core/providers/qnn/qnn_execution_provider.h index 3f75be0efebcd..09bcb24db4dc2 100644 --- a/onnxruntime/core/providers/qnn/qnn_execution_provider.h +++ b/onnxruntime/core/providers/qnn/qnn_execution_provider.h @@ -5,6 +5,7 @@ #include "core/framework/execution_provider.h" #include "core/framework/session_options.h" +#include "core/framework/model_metadef_id_generator.h" #include "core/graph/model.h" #include #include "core/providers/qnn/builder/qnn_backend_manager.h" @@ -71,6 +72,7 @@ class QNNExecutionProvider : public IExecutionProvider { bool qnn_context_embed_mode_ = true; int32_t vtcm_size_in_mb_ = 0; std::unique_ptr qnn_ep_context_model_; + ModelMetadefIdGenerator metadef_id_generator_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/shared_library/provider_api.h b/onnxruntime/core/providers/shared_library/provider_api.h index 53ba4874c643c..1e3a528d87721 100644 --- a/onnxruntime/core/providers/shared_library/provider_api.h +++ b/onnxruntime/core/providers/shared_library/provider_api.h @@ -142,7 +142,7 @@ struct KernelDefBuilder; struct KernelRegistry; struct Function; struct Graph; -struct GraphViewer; +class GraphViewer; enum class DataLayout; struct Model; struct Path; @@ -157,6 +157,7 @@ struct Tensor; struct SparseTensor; class TensorSeq; class SessionState; +class ModelMetadefIdGenerator; class If; class Loop; diff --git a/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc b/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc index a3155fe6b86cf..a16d2890b47c2 100644 --- a/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc +++ b/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc @@ -329,10 +329,6 @@ common::Status IExecutionProvider::Compile(const std::vector& return g_host->IExecutionProvider__Compile(this, fused_nodes_and_graphs, node_compute_funcs); } -int IExecutionProvider::GenerateMetaDefId(const onnxruntime::GraphViewer& graph_viewer, HashValue& model_hash) const { - return g_host->IExecutionProvider__GenerateMetaDefId(this, graph_viewer, model_hash); -} - #ifdef USE_TENSORRT std::unique_ptr CreateCUDAAllocator(int16_t device_id, const char* name) { return g_host->CreateCUDAAllocator(device_id, name); diff --git a/onnxruntime/core/providers/shared_library/provider_interfaces.h b/onnxruntime/core/providers/shared_library/provider_interfaces.h index 21c14ce784a38..a216b2bfc6d04 100644 --- a/onnxruntime/core/providers/shared_library/provider_interfaces.h +++ b/onnxruntime/core/providers/shared_library/provider_interfaces.h @@ -229,8 +229,6 @@ struct ProviderHost { virtual common::Status IExecutionProvider__Compile(IExecutionProvider* p, const std::vector& fused_nodes_and_graphs, std::vector& node_compute_funcs) = 0; - virtual int IExecutionProvider__GenerateMetaDefId(const IExecutionProvider* p, const onnxruntime::GraphViewer& graph_viewer, HashValue& model_hash) = 0; - // Status virtual std::string Status__ToString(const Status* p) = 0; @@ -972,6 +970,11 @@ struct ProviderHost { #if !defined(ORT_MINIMAL_BUILD) || defined(ORT_MINIMAL_BUILD_CUSTOM_OPS) virtual Status LoadDynamicLibrary(onnxruntime::PathString library_name) = 0; #endif + + // ModelMetadefIdGenerator + virtual std::unique_ptr ModelMetadefIdGenerator__construct() = 0; + virtual void ModelMetadefIdGenerator__operator_delete(ModelMetadefIdGenerator* p) = 0; + virtual int ModelMetadefIdGenerator__GenerateId(const ModelMetadefIdGenerator* p, const GraphViewer& graph_viewer, HashValue& model_hash) = 0; }; #if defined(_MSC_VER) && !defined(__clang__) diff --git a/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h b/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h index eaf8ef459cf00..f46c76fd3421b 100644 --- a/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h +++ b/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h @@ -750,7 +750,8 @@ struct Graph final { PROVIDER_DISALLOW_ALL(Graph) }; -struct GraphViewer final { +class GraphViewer final { + public: static void operator delete(void* p) { g_host->GraphViewer__operator_delete(reinterpret_cast(p)); } std::unique_ptr CreateModel(const logging::Logger& logger) const { return g_host->GraphViewer__CreateModel(this, logger); } @@ -1152,6 +1153,13 @@ class TensorSeq final { void Reserve(size_t capacity) { g_host->TensorSeq__Reserve(this, capacity); } }; +class ModelMetadefIdGenerator { + public: + static std::unique_ptr Create() { return g_host->ModelMetadefIdGenerator__construct(); } + static void operator delete(void* p) { g_host->ModelMetadefIdGenerator__operator_delete(reinterpret_cast(p)); } + int GenerateId(const GraphViewer& graph_viewer, HashValue& model_hash) const { return g_host->ModelMetadefIdGenerator__GenerateId(this, graph_viewer, model_hash); } +}; + template <> inline gsl::span Tensor::DataAsSpan() const { return g_host->Tensor__DataAsSpan_int64(this); } diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc index fe6b959b962de..c709ca31f8643 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc @@ -1310,7 +1310,7 @@ TensorrtExecutionProvider::PerThreadContext& TensorrtExecutionProvider::GetPerTh } TensorrtExecutionProvider::TensorrtExecutionProvider(const TensorrtExecutionProviderInfo& info) - : IExecutionProvider{onnxruntime::kTensorrtExecutionProvider, OrtDevice(OrtDevice::GPU, OrtDevice::MemType::DEFAULT, info.device_id), true}, info_(info), device_id_(info.device_id) { + : IExecutionProvider{onnxruntime::kTensorrtExecutionProvider, OrtDevice(OrtDevice::GPU, OrtDevice::MemType::DEFAULT, info.device_id)}, info_(info), device_id_(info.device_id) { InitProviderOrtApi(); CUDA_CALL_THROW(cudaSetDevice(device_id_)); diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_utils.h b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_utils.h index a8e3ae3ddf6ec..92cce0c203927 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_utils.h +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_utils.h @@ -497,7 +497,15 @@ void RemoveCachesByType(const std::string& root, std::string file_extension) { } } -// Helper class to generate engine id via model name/model content/env metadata +/** + * + * Helper class to generate engine id via model name/model content/env metadata + * + * + * The TensorRT Execution Provider is used in multiple sessions and the underlying infrastructure caches + * compiled kernels, so the name must be unique and deterministic across models and sessions. + * + */ HashValue TRTGenerateId(const GraphViewer& graph_viewer) { HashValue model_hash = 0; diff --git a/onnxruntime/core/providers/webnn/webnn_execution_provider.cc b/onnxruntime/core/providers/webnn/webnn_execution_provider.cc index 2922cf9540a8e..257a8fa74d930 100644 --- a/onnxruntime/core/providers/webnn/webnn_execution_provider.cc +++ b/onnxruntime/core/providers/webnn/webnn_execution_provider.cc @@ -19,7 +19,7 @@ namespace onnxruntime { WebNNExecutionProvider::WebNNExecutionProvider(const std::string& webnn_device_flags, const std::string& webnn_threads_number, const std::string& webnn_power_flags) - : IExecutionProvider{onnxruntime::kWebNNExecutionProvider, true} { + : IExecutionProvider{onnxruntime::kWebNNExecutionProvider} { // Create WebNN context and graph builder. const emscripten::val ml = emscripten::val::global("navigator")["ml"]; if (!ml.as()) { @@ -168,7 +168,7 @@ WebNNExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_view // Assign inputs and outputs to subgraph's meta_def. uint64_t model_hash; - int metadef_id = GenerateMetaDefId(graph_viewer, model_hash); + int metadef_id = metadef_id_generator_.GenerateId(graph_viewer, model_hash); auto meta_def = std::make_unique<::onnxruntime::IndexedSubGraph::MetaDef>(); meta_def->name = "WEBNN_" + std::to_string(model_hash) + "_" + std::to_string(metadef_id); meta_def->domain = kMSDomain; diff --git a/onnxruntime/core/providers/webnn/webnn_execution_provider.h b/onnxruntime/core/providers/webnn/webnn_execution_provider.h index 13a475327dc0c..d9cfa5f17c0d4 100644 --- a/onnxruntime/core/providers/webnn/webnn_execution_provider.h +++ b/onnxruntime/core/providers/webnn/webnn_execution_provider.h @@ -6,6 +6,7 @@ #include "core/common/inlined_containers.h" #include "core/framework/execution_provider.h" +#include "core/framework/model_metadef_id_generator.h" #include "core/providers/webnn/builders/helper.h" #include @@ -48,5 +49,6 @@ class WebNNExecutionProvider : public IExecutionProvider { DataLayout preferred_layout_; webnn::WebnnDeviceType wnn_device_type_; InlinedHashMap> models_; + ModelMetadefIdGenerator metadef_id_generator_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/xnnpack/xnnpack_execution_provider.cc b/onnxruntime/core/providers/xnnpack/xnnpack_execution_provider.cc index a2a776df439e4..eafbfae6f01e1 100644 --- a/onnxruntime/core/providers/xnnpack/xnnpack_execution_provider.cc +++ b/onnxruntime/core/providers/xnnpack/xnnpack_execution_provider.cc @@ -155,7 +155,7 @@ std::unique_ptr RegisterKernels() { using namespace xnnpack; XnnpackExecutionProvider::XnnpackExecutionProvider(const XnnpackExecutionProviderInfo& info) - : IExecutionProvider{kXnnpackExecutionProvider, true} { + : IExecutionProvider{kXnnpackExecutionProvider} { int xnn_thread_pool_size = info.xnn_thread_pool_size; int ort_thread_pool_size = info.session_options ? info.session_options->intra_op_param.thread_pool_size : 1; bool allow_intra_op_spinning = (info.session_options == nullptr) || diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index 3269c9f0f4e4b..f8bd7f4aa208c 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -30,6 +30,7 @@ #include "core/framework/sparse_utils.h" #include "core/graph/graph_proto_serializer.h" #include "core/framework/murmurhash3.h" +#include "core/framework/model_metadef_id_generator.h" #include "core/session/onnxruntime_c_api.h" #include "core/common/string_helper.h" @@ -317,10 +318,6 @@ struct ProviderHostImpl : ProviderHost { return p->IExecutionProvider::Compile(fused_nodes_and_graphs, node_compute_funcs); } - int IExecutionProvider__GenerateMetaDefId(const IExecutionProvider* p, const onnxruntime::GraphViewer& graph_viewer, HashValue& model_hash) override { - return p->IExecutionProvider::GenerateMetaDefId(graph_viewer, model_hash); - } - // Status (direct) std::string Status__ToString(const Status* p) override { return p->Status::ToString(); } @@ -1083,6 +1080,11 @@ struct ProviderHostImpl : ProviderHost { void TensorSeq__Add(TensorSeq* p, Tensor&& tensor) override { p->Add(std::move(tensor)); } void TensorSeq__Reserve(TensorSeq* p, size_t capacity) override { p->Reserve(capacity); } + // ModelMetadefIdGenerator(wrapped) + std::unique_ptr ModelMetadefIdGenerator__construct() override { return std::make_unique(); } + void ModelMetadefIdGenerator__operator_delete(ModelMetadefIdGenerator* p) override { delete p; } + int ModelMetadefIdGenerator__GenerateId(const ModelMetadefIdGenerator* p, const GraphViewer& graph_viewer, HashValue& model_hash) override { return p->GenerateId(graph_viewer, model_hash); } + #if defined(ENABLE_TRAINING) && defined(ORT_USE_NCCL) training::DistributedRunContext& GetDistributedRunContextInstance() override { return training::DistributedRunContext::GetInstance(); } #endif diff --git a/onnxruntime/test/framework/execution_provider_test.cc b/onnxruntime/test/framework/execution_provider_test.cc index 5a7351a766fa3..390fda7bfc5ad 100644 --- a/onnxruntime/test/framework/execution_provider_test.cc +++ b/onnxruntime/test/framework/execution_provider_test.cc @@ -6,6 +6,7 @@ #include "test_utils.h" #include "test/test_environment.h" #include "test/util/include/asserts.h" +#include "core/framework/model_metadef_id_generator.h" #include "gtest/gtest.h" @@ -18,11 +19,14 @@ class TestEP : public IExecutionProvider { static constexpr const char* kEPType = "TestEP"; public: - TestEP() : IExecutionProvider{kEPType, true} {} + TestEP() : IExecutionProvider{kEPType} {} int GetId(const GraphViewer& viewer, HashValue& model_hash) { - return GenerateMetaDefId(viewer, model_hash); + return metadef_id_generator_.GenerateId(viewer, model_hash); } + + private: + ModelMetadefIdGenerator metadef_id_generator_; }; TEST(ExecutionProviderTest, MetadefIdGeneratorUsingModelPath) { diff --git a/onnxruntime/test/framework/tunable_op_test.cc b/onnxruntime/test/framework/tunable_op_test.cc index 19253e1a5bd2c..6fe0754db40d3 100644 --- a/onnxruntime/test/framework/tunable_op_test.cc +++ b/onnxruntime/test/framework/tunable_op_test.cc @@ -82,7 +82,7 @@ class TestEP : public IExecutionProvider { TestTuningContext tuning_ctx_{this}; public: - TestEP() : IExecutionProvider{kEPType, true} {} + TestEP() : IExecutionProvider{kEPType} {} ITuningContext* GetTuningContext() const override { return const_cast(&tuning_ctx_); diff --git a/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.cc b/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.cc index 957443c23e7c3..0167f7a7718b1 100644 --- a/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.cc +++ b/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.cc @@ -85,7 +85,7 @@ constexpr const char* INTERNAL_TESTING_EP = "InternalTestingEP"; InternalTestingExecutionProvider::InternalTestingExecutionProvider(const std::unordered_set& ops, const std::unordered_set& stop_ops, DataLayout preferred_layout) - : IExecutionProvider{utils::kInternalTestingExecutionProvider, true}, + : IExecutionProvider{utils::kInternalTestingExecutionProvider}, ep_name_{INTERNAL_TESTING_EP}, ops_{ops}, stop_ops_{stop_ops}, @@ -212,7 +212,7 @@ InternalTestingExecutionProvider::GetCapability(const onnxruntime::GraphViewer& // create functor to generate a guaranteed unique metadef id auto generate_metadef_name = [this, &graph_viewer]() { HashValue model_hash; - int metadef_id = GenerateMetaDefId(graph_viewer, model_hash); + int metadef_id = metadef_id_generator_.GenerateId(graph_viewer, model_hash); auto meta_def = std::make_unique<::onnxruntime::IndexedSubGraph::MetaDef>(); return ep_name_ + "_" + std::to_string(model_hash) + "_" + std::to_string(metadef_id); }; diff --git a/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.h b/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.h index 6103352627667..6615eb82f2b05 100644 --- a/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.h +++ b/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.h @@ -4,6 +4,7 @@ #pragma once #include #include "core/framework/execution_provider.h" +#include "core/framework/model_metadef_id_generator.h" namespace onnxruntime { namespace internal_testing_ep { @@ -82,6 +83,7 @@ class InternalTestingExecutionProvider : public IExecutionProvider { // per-instance kernel registry so tests using static kernels don't clash. // shared_ptr as required by IExecutionProvider::GetKernelRegistry std::shared_ptr kernel_registry_; + ModelMetadefIdGenerator metadef_id_generator_; }; } // namespace internal_testing_ep