[WIP] Out-Tree EP feature

include/onnxruntime/core/session/environment.h

@@ -88,6 +88,10 @@
    */
   Status CreateAndRegisterAllocatorV2(const std::string& provider_type, const OrtMemoryInfo& mem_info, const std::unordered_map<std::string, std::string>& options, const OrtArenaCfg* arena_cfg = nullptr);
 
+  void InsertCustomEp(const char* ep_name, OrtExecutionProviderFactory* ep_factory);
+
+  const std::unordered_map<std::string, std::unique_ptr<OrtExecutionProviderFactory>>& GetCustomEpFactories() const { return custom_ep_factories_; }
+
  private:
   ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(Environment);
   Status Initialize(std::unique_ptr<logging::LoggingManager> logging_manager,
@@ -99,5 +103,6 @@
   std::unique_ptr<onnxruntime::concurrency::ThreadPool> inter_op_thread_pool_;
   bool create_global_thread_pools_{false};
   std::vector<AllocatorPtr> shared_allocators_;
+  std::unordered_map<std::string, std::unique_ptr<OrtExecutionProviderFactory>> custom_ep_factories_;
 };
 }  // namespace onnxruntime

include/onnxruntime/core/session/onnxruntime_c_api.h

@@ -304,6 +304,9 @@
 ORT_RUNTIME_CLASS(OpAttr);
 ORT_RUNTIME_CLASS(Logger);
 ORT_RUNTIME_CLASS(ShapeInferContext);
+ORT_RUNTIME_CLASS(ExecutionProvider);
+ORT_RUNTIME_CLASS(ExecutionProviderFactory);
+ORT_RUNTIME_CLASS(Graph);
 
 #ifdef _WIN32
 typedef _Return_type_success_(return == 0) OrtStatus* OrtStatusPtr;
@@ -681,6 +684,16 @@
   const char*(ORT_API_CALL* GetVersionString)(void)NO_EXCEPTION;
 };
 
+typedef struct OrtExecutionProvider {
+  //void(ORT_API_CALL* GetCapability)(const OrtExecutionProvider* this_, const OrtGraph* graph, _Out_ int* cnt, _Outptr_ OrtComputeCapability** compute_capability);
+  //void(ORT_API_CALL* Compile)(OrtExecutionProvider* this_, const OrtGraph* graph, const OrtNode* node, int size, _Out_ int* cnt, _Outptr_ OrtNodeComputeInfo** node_compute_info);
+  const char* type;
+} OrtExecutionProvider;
+
+typedef struct OrtExecutionProviderFactory {
+  void*(ORT_API_CALL* CreateExecutionProvider)(OrtExecutionProviderFactory* this_, const char* const* ep_option_keys, const char* const* ep_option_values, size_t option_size);
+} OrtExecutionProviderFactory;
+
 typedef struct OrtApiBase OrtApiBase;
 
 /** \brief The Onnxruntime library's entry point to access the C API
@@ -4665,6 +4678,11 @@
                   _In_reads_(num_external_initializer_files) char* const* external_initializer_file_buffer_array,
                   _In_reads_(num_external_initializer_files) const size_t* external_initializer_file_lengths,
                   size_t num_external_initializer_files);
+
+  ORT_API2_STATUS(RegisterOrtExecutionProviderLibrary, _In_ const ORTCHAR_T* lib_path, _In_ OrtEnv* env, _In_ const char* ep_name);
+
+  ORT_API2_STATUS(SessionOptionsAppendOrtExecutionProvider, _In_ OrtSessionOptions* options, _In_ const char* ep_name,
+                   _In_reads_(num_keys) const char* const* provider_options_keys, _In_reads_(num_keys) const char* const* provider_options_values, _In_ size_t num_keys);
 };
 
 /*
@@ -4825,6 +4843,8 @@
  */
 ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_Tensorrt, _In_ OrtSessionOptions* options, int device_id);
 
+ORT_API(bool, OrtGraph_IsConstantInitializer, const OrtGraph* graph, const char* name, bool check_outer_scope);
+
 #ifdef __cplusplus
 }
 #endif

onnxruntime/core/framework/provider_adapter.h

@@ -0,0 +1,14 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+#include "core/session/onnxruntime_c_api.h"
+
+namespace onnxruntime {
+class ExecutionProviderAdapter : public IExecutionProvider {
+  public:
+  ExecutionProviderAdapter(OrtExecutionProvider* ep) : IExecutionProvider(ep->type), ep_impl_(ep) {}
+  private:
+  OrtExecutionProvider* ep_impl_;
+};
+}

onnxruntime/core/framework/session_options.h

@@ -15,6 +15,7 @@
 #include "core/session/onnxruntime_c_api.h"
 #include "core/optimizer/graph_transformer_level.h"
 #include "core/util/thread_utils.h"
+#include "core/framework/provider_options.h"
 
 #if !defined(ORT_MINIMAL_BUILD) || defined(ORT_MINIMAL_BUILD_CUSTOM_OPS)
 #include "core/framework/library_handles.h"
@@ -184,6 +185,8 @@ struct SessionOptions {
   // User specified logging func and param
   OrtLoggingFunction user_logging_function = nullptr;
   void* user_logging_param = nullptr;
+
+  ProviderOptionsMap custom_ep_options;
 };
 
 inline std::ostream& operator<<(std::ostream& os, const SessionOptions& session_options) {

onnxruntime/core/session/environment.cc

@@ -348,4 +348,9 @@ Status Environment::CreateAndRegisterAllocatorV2(const std::string& provider_typ
   return Status{ONNXRUNTIME, common::INVALID_ARGUMENT, provider_type + " is not implemented in CreateAndRegisterAllocatorV2()"};
 }
 
+void Environment::InsertCustomEp(const char* ep_name, OrtExecutionProviderFactory* ep_factory) {
+  std::unique_ptr<OrtExecutionProviderFactory> p(ep_factory);
+  custom_ep_factories_.insert({ep_name, std::move(p)});  // TODO(leca): review
+}
+
 }  // namespace onnxruntime

onnxruntime/core/session/inference_session.cc

@@ -89,6 +89,7 @@
 #include "core/framework/stream_execution_context.h"
 #include "orttraining/core/optimizer/memory_optimizer/memory_optimizer.h"
 #endif
+#include "core/framework/provider_adapter.h"
 
 using namespace ONNX_NAMESPACE;
 using namespace onnxruntime::common;
@@ -1655,6 +1656,22 @@ common::Status InferenceSession::Initialize() {
     const Env& env = Env::Default();
     env.GetTelemetryProvider().LogSessionCreationStart();
 
+    const std::unordered_map<std::string, std::unique_ptr<OrtExecutionProviderFactory>>& custom_ep_factories = environment_.GetCustomEpFactories();
+    if (custom_ep_factories.size() > 0) {
+      for (auto const& [ep_name, ep_factory] : custom_ep_factories) {
+        if (session_options_.custom_ep_options.find(ep_name) != session_options_.custom_ep_options.end()) {
+          std::vector<const char*> keys, values;
+          for (auto const& [op_k, op_v] : session_options_.custom_ep_options[ep_name]) {
+            keys.push_back(op_k.c_str());
+            values.push_back(op_v.c_str());
+          }
+          OrtExecutionProvider* ep = reinterpret_cast<OrtExecutionProvider*>(ep_factory->CreateExecutionProvider(ep_factory.get(), keys.data(), values.data(), keys.size()));
+          std::unique_ptr<ExecutionProviderAdapter> ep_adapter = std::make_unique<ExecutionProviderAdapter>(ep);
+          ORT_RETURN_IF_ERROR(RegisterExecutionProvider(std::move(ep_adapter)));
+        }
+      }
+    }
+
     bool have_cpu_ep = false;
 
     {

onnxruntime/core/session/onnxruntime_c_api.cc

@@ -22,6 +22,7 @@
 #include "core/common/safeint.h"
 #include "core/graph/constants.h"
 #include "core/graph/graph.h"
+#include "core/graph/graph_viewer.h"
 #include "core/framework/allocator.h"
 #include "core/framework/tensor.h"
 #include "core/framework/ort_value.h"
@@ -2353,6 +2354,30 @@ ORT_API(const OrtTrainingApi*, OrtApis::GetTrainingApi, uint32_t version) {
 #endif
 }
 
+ORT_API_STATUS_IMPL(OrtApis::RegisterOrtExecutionProviderLibrary, _In_ const char* lib_path, _In_ OrtEnv* env, _In_ const char* ep_name) {
+  API_IMPL_BEGIN
+  void* handle = nullptr;
+  ORT_THROW_IF_ERROR(Env::Default().LoadDynamicLibrary(ToPathString(lib_path), false, &handle));
+  if (handle) {
+    OrtExecutionProviderFactory* (*symbol)();
+    ORT_THROW_IF_ERROR(Env::Default().GetSymbolFromLibrary(handle, "RegisterCustomEp", (void**)&symbol));
+    env->InsertCustomEp(ep_name, symbol());
+    return nullptr;
+  }
+  return CreateStatus(ORT_RUNTIME_EXCEPTION, "cannot load the shared library for out-tree EP");
+  API_IMPL_END
+}
+
+ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendOrtExecutionProvider, _In_ OrtSessionOptions* options, _In_ const char* ep_name,
+                    _In_reads_(num_keys) const char* const* provider_options_keys, _In_reads_(num_keys) const char* const* provider_options_values, _In_ size_t num_keys) {
+  std::unordered_map<std::string, std::string> kv;
+  for (size_t i = 0; i < num_keys; i++) {
+    kv.insert({provider_options_keys[i], provider_options_values[i]});
+  }
+  options->value.custom_ep_options.insert({ep_name, kv});
+  return nullptr;
+}
+
 static constexpr OrtApiBase ort_api_base = {
     &OrtApis::GetApi,
     &OrtApis::GetVersionString};
@@ -2730,6 +2755,9 @@ static constexpr OrtApi ort_api_1_to_19 = {
     &OrtApis::KernelInfoGetAllocator,
     &OrtApis::AddExternalInitializersFromFilesInMemory,
     // End of Version 18 - DO NOT MODIFY ABOVE (see above text for more information)
+
+    &OrtApis::RegisterOrtExecutionProviderLibrary,
+    &OrtApis::SessionOptionsAppendOrtExecutionProvider,
 };
 
 // OrtApiBase can never change as there is no way to know what version of OrtApiBase is returned by OrtGetApiBase.
@@ -2802,3 +2830,8 @@ DEFINE_RELEASE_ORT_OBJECT_FUNCTION(Value, OrtValue)
 DEFINE_RELEASE_ORT_OBJECT_FUNCTION(RunOptions, OrtRunOptions)
 DEFINE_RELEASE_ORT_OBJECT_FUNCTION(Session, ::onnxruntime::InferenceSession)
 DEFINE_RELEASE_ORT_OBJECT_FUNCTION(ModelMetadata, ::onnxruntime::ModelMetadata)
+
+ORT_API(bool, OrtGraph_IsConstantInitializer, const OrtGraph* graph, const char* name, bool check_outer_scope) {
+  ::onnxruntime::GraphViewer graph_viewer(*(reinterpret_cast<const ::onnxruntime::Graph*>(graph)));
+  return graph_viewer.IsConstantInitializer(std::string(name), check_outer_scope);
+}

onnxruntime/core/session/ort_apis.h

@@ -523,4 +523,9 @@ ORT_API_STATUS_IMPL(SessionOptionsAppendExecutionProvider_VitisAI, _In_ OrtSessi
 ORT_API_STATUS_IMPL(KernelContext_GetScratchBuffer, _In_ const OrtKernelContext* context, _In_ const OrtMemoryInfo* mem_info, _In_ size_t count_or_bytes, _Outptr_ void** out);
 
 ORT_API_STATUS_IMPL(KernelInfoGetAllocator, _In_ const OrtKernelInfo* info, _In_ OrtMemType mem_type, _Outptr_ OrtAllocator** out);
+
+ORT_API_STATUS_IMPL(RegisterOrtExecutionProviderLibrary, _In_ const ORTCHAR_T* lib_path, _In_ OrtEnv* env, _In_ const char* ep_name);
+
+ORT_API_STATUS_IMPL(SessionOptionsAppendOrtExecutionProvider, _In_ OrtSessionOptions* options, _In_ const char* ep_name,
+                    _In_reads_(num_keys) const char* const* provider_options_keys, _In_reads_(num_keys) const char* const* provider_options_values, _In_ size_t num_keys);
 }  // namespace OrtApis

onnxruntime/core/session/ort_env.cc

@@ -110,3 +110,7 @@ onnxruntime::common::Status OrtEnv::UnregisterAllocator(const OrtMemoryInfo& mem
 onnxruntime::common::Status OrtEnv::CreateAndRegisterAllocatorV2(const std::string& provider_type, const OrtMemoryInfo& mem_info, const std::unordered_map<std::string, std::string>& options, const OrtArenaCfg* arena_cfg) {
   return value_->CreateAndRegisterAllocatorV2(provider_type, mem_info, options, arena_cfg);
 }
+
+void OrtEnv::InsertCustomEp(const char* ep_name, OrtExecutionProviderFactory* ep_factory) {
+  value_->InsertCustomEp(ep_name, ep_factory);
+}

onnxruntime/core/session/ort_env.h

@@ -65,6 +65,8 @@ struct OrtEnv {
   ~OrtEnv();
   onnxruntime::common::Status CreateAndRegisterAllocatorV2(const std::string& provider_type, const OrtMemoryInfo& mem_info, const std::unordered_map<std::string, std::string>& options, const OrtArenaCfg* arena_cfg = nullptr);
 
+  void InsertCustomEp(const char* ep_name, OrtExecutionProviderFactory* ep_factory);
+
  private:
   static std::unique_ptr<OrtEnv> p_instance_;
   static onnxruntime::OrtMutex m_;

samples/c_test/CMakeLists.txt

@@ -0,0 +1,10 @@
+# usage:
+# cd build/
+# cmake -S ../ -B ./
+# cmake --build ./
+cmake_minimum_required(VERSION 3.26)
+project(TestOutTreeEp)
+add_executable(TestOutTreeEp test.cpp)
+
+target_include_directories(TestOutTreeEp PUBLIC "../../include/onnxruntime")
+target_link_libraries(TestOutTreeEp PUBLIC "/home/leca/code/onnxruntime/build/Linux/Debug/libonnxruntime.so")

samples/c_test/test.cpp

@@ -0,0 +1,20 @@
+#include "core/session/onnxruntime_c_api.h"
+#include <vector>
+
+inline void THROW_ON_ERROR(OrtStatus* status) {
+    if (status != nullptr) abort();
+}
+
+int main() {
+    const OrtApi* g_ort = OrtGetApiBase()->GetApi(ORT_API_VERSION);
+    OrtEnv* p_env = nullptr;
+    OrtLoggingLevel log_level = OrtLoggingLevel::ORT_LOGGING_LEVEL_INFO;
+    THROW_ON_ERROR(g_ort->CreateEnv(log_level, "", &p_env));
+    THROW_ON_ERROR(g_ort->RegisterOrtExecutionProviderLibrary("/home/leca/code/onnxruntime/samples/outTreeEp/build/liboutTreeEp.so", p_env, "outTreeEp"));
+
+    OrtSessionOptions* so = nullptr;
+    THROW_ON_ERROR(g_ort->CreateSessionOptions(&so));
+    std::vector<const char*> keys{"int_property", "str_property"}, values{"3", "strvalue"};
+    THROW_ON_ERROR(g_ort->SessionOptionsAppendOrtExecutionProvider(so, "outTreeEp", keys.data(), values.data(), keys.size()));
+    return 0;
+}

samples/outTreeEp/CMakeLists.txt

@@ -0,0 +1,9 @@
+# usage:
+# cd build/
+# cmake -S ../ -B ./
+# cmake --build ./
+cmake_minimum_required(VERSION 3.26)
+project(outTreeEp VERSION 1.0)
+set(CMAKE_CXX_STANDARD 17)
+add_library(outTreeEp SHARED out_tree_ep.cc)
+target_include_directories(outTreeEp PUBLIC "../../include/onnxruntime")

samples/outTreeEp/out_tree_ep.cc

@@ -0,0 +1,29 @@
+#include "out_tree_ep.h"
+#include <memory>
+namespace onnxruntime {
+
+OutTreeEpFactory::OutTreeEpFactory() {
+    OrtExecutionProviderFactory::CreateExecutionProvider = [](OrtExecutionProviderFactory* this_, const char* const* ep_option_keys, const char* const* ep_option_values, size_t option_size) -> void* {
+        OutTreeEpInfo info;
+        for (size_t i = 0; i < option_size; i++) {
+            if (!strcmp(ep_option_keys[i], "int_property")) info.int_property = std::atoi(ep_option_values[i]);
+            else if (!strcmp(ep_option_keys[i], "str_property")) info.str_property = ep_option_values[i];
+            // TODO(leca): else throw
+        }
+        std::unique_ptr<OutTreeEp> ret = std::make_unique<OutTreeEp>("outTreeEp", std::move(info));
+        return ret.release();
+    };
+}
+
+}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+OrtExecutionProviderFactory* RegisterCustomEp() {
+    std::unique_ptr<onnxruntime::OutTreeEpFactory> ret = std::make_unique<onnxruntime::OutTreeEpFactory>();
+    return ret.release();
+}
+#ifdef __cplusplus
+}
+#endif

samples/outTreeEp/out_tree_ep.h

@@ -0,0 +1,36 @@
+#pragma once
+#include "core/session/onnxruntime_c_api.h"
+#include <string>
+
+#ifdef _WIN32
+#define EXPORT_API __declspec(dllexport)
+#else
+#define EXPORT_API
+#endif
+
+namespace onnxruntime {
+
+struct OutTreeEpInfo {
+    int int_property;
+    std::string str_property;
+};
+
+struct OutTreeEp : public OrtExecutionProvider {
+    OutTreeEp(const char* ep_type, const OutTreeEpInfo& ep_info) : info(ep_info) { type = ep_type; }
+    OutTreeEpInfo info;
+};
+
+struct OutTreeEpFactory : public OrtExecutionProviderFactory {
+    OutTreeEpFactory();
+};
+}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+EXPORT_API OrtExecutionProviderFactory* RegisterCustomEp();
+
+#ifdef __cplusplus
+}
+#endif