[WIP] Out-Tree EP feature

cmake/onnxruntime_unittests.cmake

@@ -483,6 +483,10 @@ if (NOT onnxruntime_MINIMAL_BUILD)
   list(APPEND onnxruntime_shared_lib_test_SRC ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_inference.cc)
 endif()
 
+if (onnxruntime_TEST_TENSORRT_EP_PLUGIN)
+    list(APPEND onnxruntime_shared_lib_test_SRC ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_trt_ep_plugin.cc)
+endif()
+
 if(onnxruntime_RUN_ONNX_TESTS)
   list(APPEND onnxruntime_shared_lib_test_SRC ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_io_types.cc)
 endif()
@@ -1227,7 +1231,7 @@ if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP)
       endif()
       if (${CMAKE_SYSTEM_NAME} MATCHES "AIX")
         list(APPEND onnxruntime_perf_test_libs onnxruntime_graph onnxruntime_session onnxruntime_providers onnxruntime_framework onnxruntime_util onnxruntime_mlas onnxruntime_optimizer onnxruntime_flatbuffers iconv re2 gtest absl_failure_signal_handler absl_examine_stack absl_flags_parse  absl_flags_usage absl_flags_usage_internal)
-    endif()
+      endif()
       target_link_libraries(onnxruntime_perf_test PRIVATE ${onnxruntime_perf_test_libs} Threads::Threads)
       if(WIN32)
         target_link_libraries(onnxruntime_perf_test PRIVATE debug dbghelp advapi32)

include/onnxruntime/core/framework/execution_provider.h

@@ -75,6 +75,8 @@ class IExecutionProvider {
   */
   const OrtDevice default_device_;
 
+  bool builtin_ep_ = true;
+
  public:
   virtual ~IExecutionProvider() = default;
 
@@ -325,6 +327,8 @@ class IExecutionProvider {
     return InlinedVector<const Node*>();
   }
 
+  bool IsBuiltInEp() const { return builtin_ep_; }
+
  private:
   const std::string type_;
 

include/onnxruntime/core/framework/ort_type_constraints.h

@@ -0,0 +1,15 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+#include "core/session/onnxruntime_c_api_ep.h"
+#include <unordered_map>
+#include <string>
+#include <set>
+
+struct OrtTypeConstraints {
+    bool AddTypeConstraint(const char* type_symbol, ONNXTensorElementDataType type);
+    inline const std::unordered_map<std::string, std::set<ONNXTensorElementDataType>>& GetTypeConstraints() const { return type_constraints_; };
+private:
+    std::unordered_map<std::string, std::set<ONNXTensorElementDataType>> type_constraints_;
+};

include/onnxruntime/core/session/environment.h

@@ -5,11 +5,13 @@
 
 #include <atomic>
 #include <memory>
+#include <unordered_set>
 #include "core/common/common.h"
 #include "core/common/status.h"
 #include "core/platform/threadpool.h"
 #include "core/common/logging/logging.h"
 #include "core/framework/allocator.h"
+#include "core/session/onnxruntime_c_api_ep.h"
 
 struct OrtThreadingOptions;
 namespace onnxruntime {
@@ -88,6 +90,15 @@ class Environment {
    */
   Status CreateAndRegisterAllocatorV2(const std::string& provider_type, const OrtMemoryInfo& mem_info, const std::unordered_map<std::string, std::string>& options, const OrtArenaCfg* arena_cfg = nullptr);
 
+  // TODO(leca): return Status to handle corner cases (plugin EP Factory already exists, etc.)
+  void InsertPluginEpFactory(const char* ep_name, OrtExecutionProviderFactory* ep_factory);
+
+  OrtExecutionProviderFactory* GetPluginExecutionProviderFactory(const std::string& ep_name);
+
+  Status DeletePluginEpFactory(const char* ep_name);
+
+  std::unordered_set<std::string> GetPluginEpFactoryNames();
+
  private:
   ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(Environment);
   Status Initialize(std::unique_ptr<logging::LoggingManager> logging_manager,
@@ -99,5 +110,6 @@ class Environment {
   std::unique_ptr<onnxruntime::concurrency::ThreadPool> inter_op_thread_pool_;
   bool create_global_thread_pools_{false};
   std::vector<AllocatorPtr> shared_allocators_;
+  std::unordered_map<std::string, std::unique_ptr<OrtExecutionProviderFactory>> plugin_ep_factories_;
 };
 }  // namespace onnxruntime

include/onnxruntime/core/session/onnxruntime_c_api.h

@@ -304,6 +304,12 @@ ORT_RUNTIME_CLASS(Op);
 ORT_RUNTIME_CLASS(OpAttr);
 ORT_RUNTIME_CLASS(Logger);
 ORT_RUNTIME_CLASS(ShapeInferContext);
+ORT_RUNTIME_CLASS(KernelInfo);
+ORT_RUNTIME_CLASS(KernelContext);
+ORT_RUNTIME_CLASS(CustomOp);
+ORT_RUNTIME_CLASS(KernelRegistry);
+ORT_RUNTIME_CLASS(TypeConstraints);
+ORT_RUNTIME_CLASS(Device);
 
 #ifdef _WIN32
 typedef _Return_type_success_(return == 0) OrtStatus* OrtStatusPtr;
@@ -364,13 +370,6 @@ typedef enum OrtLanguageProjection {
   ORT_PROJECTION_NODEJS = 6,
 } OrtLanguageProjection;
 
-struct OrtKernelInfo;
-typedef struct OrtKernelInfo OrtKernelInfo;
-struct OrtKernelContext;
-typedef struct OrtKernelContext OrtKernelContext;
-struct OrtCustomOp;
-typedef struct OrtCustomOp OrtCustomOp;
-
 typedef enum OrtAllocatorType {
   OrtInvalidAllocator = -1,
   OrtDeviceAllocator = 0,
@@ -395,6 +394,13 @@ typedef enum OrtMemoryInfoDeviceType {
   OrtMemoryInfoDeviceType_FPGA = 2
 } OrtMemoryInfoDeviceType;
 
+typedef enum OrtMemoryType {
+  OrtMemoryType_Default = 0,
+  OrtMemoryType_CUDA_PINNED = 1,
+  OrtMemoryType_HIP_PINNED = 2,
+  OrtMemoryType_CANN_PINNED = 3,
+} OrtMemoryType;
+
 /** \brief Algorithm to use for cuDNN Convolution Op
  */
 typedef enum OrtCudnnConvAlgoSearch {
@@ -658,6 +664,9 @@ typedef struct OrtApi OrtApi;
 struct OrtTrainingApi;
 typedef struct OrtTrainingApi OrtTrainingApi;
 
+struct OrtGraphApi;
+typedef struct OrtGraphApi OrtGraphApi;
+
 /** \brief The helper interface to get the right version of OrtApi
  *
  * Get a pointer to this structure through ::OrtGetApiBase
@@ -4310,9 +4319,6 @@ struct OrtApi {
    */
   const char*(ORT_API_CALL* GetBuildInfoString)(void);
 
-  /// \name OrtROCMProviderOptions
-  /// @{
-
   /** \brief Create an OrtROCMProviderOptions
    *
    * \param[out] out Newly created ::OrtROCMProviderOptions. Must be released with OrtApi::ReleaseROCMProviderOptions
@@ -4665,7 +4671,128 @@ struct OrtApi {
                   _In_reads_(num_external_initializer_files) char* const* external_initializer_file_buffer_array,
                   _In_reads_(num_external_initializer_files) const size_t* external_initializer_file_lengths,
                   size_t num_external_initializer_files);
-};
+
+  /** \brief Create OrtDevice object.
+   *
+   * \param[in] device_type
+   * \param[in] memory_type
+   * \param[in] device_id
+   * \param[out] out OrtDevice object
+   *
+   * \since Version 1.xx.
+   */
+  ORT_API2_STATUS(CreateDevice, _In_ enum OrtMemoryInfoDeviceType device_type, _In_ enum OrtMemoryType memory_type, _In_ int16_t device_id, _Outptr_ const OrtDevice** out);
+
+  /** \brief Get OrtMemoryInfoDeviceType property from OrtDevice object.
+   *
+   * \param[in] device OrtDevice object
+   * \param[out] out OrtMemoryInfoDeviceType property
+   *
+   * \since Version 1.xx.
+   */
+  ORT_API2_STATUS(DeviceGetType, _In_ const OrtDevice* device, _Out_ OrtMemoryInfoDeviceType* out);
+
+  /** \brief Get OrtMemoryType property from OrtDevice object.
+   *
+   * \param[in] device OrtDevice object
+   * \param[out] out OrtMemoryType property
+   *
+   * \since Version 1.xx.
+   */
+  ORT_API2_STATUS(DeviceGetMemoryType, _In_ const OrtDevice* device, _Out_ OrtMemoryType* out);
+
+  /** \brief Get device id property from OrtDevice object.
+   *
+   * \param[in] device OrtDevice object
+   * \param[out] out device id property
+   *
+   * \since Version 1.xx.
+   */
+  ORT_API2_STATUS(DeviceGetId, _In_ const OrtDevice* device, _Out_ int16_t* out);
+
+  /** \brief Release OrtDevice object.
+   *
+   * \since Version 1.xx.
+   */
+  ORT_CLASS_RELEASE(Device);
+
+  /** \brief Register the plugin ExecutionProvider library
+   *
+   * The plugin ExecutionProvider library will be loaded and EP factory object will be created and saved in OrtEnv object
+   *
+   * \param[in] lib_path the path of the plugin ExecutionProvider library
+   * \param[in] env OrtEnv object
+   * \param[in] ep_name the plugin ExecutionProvider name
+   *
+   * \since Version 1.xx.
+   */
+  ORT_API2_STATUS(RegisterPluginExecutionProviderLibrary, _In_ const ORTCHAR_T* lib_path, _In_ OrtEnv* env, _In_ const char* ep_name);
+
+  /** \brief Unregister the plugin ExecutionProvider library
+   *
+   * The plugin ExecutionProvider factory will be removed from OrtEnv object
+   *
+   * \param[in] env OrtEnv object
+   * \param[in] ep_name the plugin ExecutionProvider name
+   *
+   * \since Version 1.xx.
+   */
+  ORT_API2_STATUS(UnregisterPluginExecutionProviderLibrary, _In_ OrtEnv* env, _In_ const char* ep_name);
+
+  /** \brief Append the plugin ExecutionProvider factory into the session option with provider options
+   *
+   * \param[in] options OrtSessionOptions object
+   * \param[in] ep_name the plugin ExecutionProvider name
+   * \param[in] env OrtEnv object
+   * \param[in] provider_options_keys provider options' keys
+   * \param[in] provider_options_values provider options' values
+   * \param[in] num_keys the number of the provider options' key-value pairs
+   *
+   * \since Version 1.xx.
+   */
+  ORT_API2_STATUS(SessionOptionsAppendPluginExecutionProvider, _In_ OrtSessionOptions* options, _In_ const char* ep_name, _In_ OrtEnv* env,
+                   _In_reads_(num_keys) const char* const* provider_options_keys, _In_reads_(num_keys) const char* const* provider_options_values, _In_ size_t num_keys);
+
+  /** \brief Create OrtTypeConstraints object
+   *
+   * \param[out] OrtTypeConstraints object
+   *
+   * \since Version 1.xx.
+   */
+  ORT_API2_STATUS(CreateOrtTypeConstraints, _Outptr_ OrtTypeConstraints** type_constraints);
+
+  /** \brief Add a specific type constraint into OrtTypeConstraints object
+   *
+   * \param[in] type_constraints OrtTypeConstraints object
+   * \param[in] type_symbol symbol string to represent a specific type
+   * \param[in] type a specific type
+   *
+   * \since Version 1.xx.
+   */
+  ORT_API2_STATUS(AddTypeConstraint, _In_ OrtTypeConstraints* type_constraints, _In_ const char* type_symbol, ONNXTensorElementDataType type);
+
+  /** \brief Release OrtTypeConstraints object.
+   *
+   * \since Version 1.xx.
+   */
+  ORT_CLASS_RELEASE(TypeConstraints);
+
+  /** \brief Create KernelCreateInfo with custom op and type constraints, and register it
+   *
+   * \param[in] kernel_registry Opaque pointer of KernelRegistry object
+   * \param[in] custom_op Custom Op where the kernel compute function is defined
+   * \param[in] type_constraints
+   *
+   * \since Version 1.xx.
+   */
+  ORT_API2_STATUS(OrtKernelRegistry_RegisterKernel, _In_ OrtKernelRegistry* kernel_registry, _In_ OrtCustomOp* custom_op, _In_ OrtTypeConstraints* type_constraints);
+
+  /** \brief Get Graph API
+   *
+   * \since Version 1.xx.
+   */
+  const OrtGraphApi*(ORT_API_CALL* GetGraphApi)(uint32_t version)NO_EXCEPTION;
+};  // struct OrtApi
 
 /*
  * Steps to use a custom op:
@@ -4693,6 +4820,15 @@ typedef enum OrtCustomOpInputOutputCharacteristic {
  * the implementor of the custom op.
  */
 struct OrtCustomOp {
+#ifdef __cplusplus
+  OrtCustomOp() : CreateKernel{nullptr}, GetName{nullptr}, GetExecutionProviderType{nullptr}, GetInputType{nullptr},
+                  GetInputTypeCount{nullptr}, GetOutputType{nullptr}, GetOutputTypeCount{nullptr}, KernelCompute{nullptr},
+                  KernelDestroy{nullptr}, GetInputCharacteristic{nullptr}, GetOutputCharacteristic{nullptr},
+                  GetInputMemoryType{nullptr}, GetVariadicInputMinArity{nullptr}, GetVariadicInputHomogeneity{nullptr},
+                  GetVariadicOutputMinArity{nullptr}, GetVariadicOutputHomogeneity{nullptr}, CreateKernelV2{nullptr},
+                  KernelComputeV2{nullptr}, InferOutputShapeFn{nullptr}, GetStartVersion{nullptr}, GetEndVersion{nullptr},
+                  GetMayInplace{nullptr}, ReleaseMayInplace{nullptr}, GetAliasMap{nullptr}, ReleaseAliasMap{nullptr} {}
+#endif
   uint32_t version;  // Must be initialized to ORT_API_VERSION
 
   // This callback creates the kernel, which is a user defined