Add API for NPU Device Selection in the DML EP

include/onnxruntime/core/providers/dml/dml_provider_factory.h

@@ -30,6 +30,23 @@
 extern "C" {
 #endif
 
+enum OrtDmlPerformancePreference {
+  Default = 0,
+  HighPerformance = 1,
+  LowPower = 2
+};
+
+enum OrtDmlDeviceFilter {
+  Gpu = 1,
+  Npu = 2,
+  Both = 3
+};
+
+struct OrtDmlDeviceOptions {
+  OrtDmlPerformancePreference perf_pref;
+  OrtDmlDeviceFilter dev_filter;
+};
+
 /**
  * [[deprecated]]
  * This export is deprecated.
@@ -99,6 +116,13 @@
    * This API gets the D3D12 resource when an OrtValue has been allocated by the DML EP.
    */
   ORT_API2_STATUS(GetD3D12ResourceFromAllocation, _In_ OrtAllocator* provider, _In_ void* dml_resource, _Out_ ID3D12Resource** d3d_resource);
+
+  /**
+   * SessionOptionsAppendExecutionProvider_DML2
+   * Creates a DirectML Execution Provider given the supplied device options that contain a performance preference
+   * (high power, low power, or defult) and a device filter (None, GPU, or NPU).
+   */
+  ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_DML2, _In_ OrtSessionOptions* options, OrtDmlDeviceOptions* device_opts);
 };
 
 #ifdef __cplusplus

onnxruntime/core/providers/dml/dml_provider_factory.cc

@@ -1,6 +1,9 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
+#include <dxcore.h>
+#include <vector>
+
 #include <DirectML.h>
 #ifndef _GAMING_XBOX
 #include <dxgi1_4.h>
@@ -172,6 +175,41 @@
   return CreateExecutionProviderFactory_DML(dml_device.Get(), cmd_queue.Get());
 }
 
+std::shared_ptr<IExecutionProviderFactory> DMLProviderFactoryCreator::CreateDXCore(ComPtr<IDXCoreAdapter> dxcore_device) {
+  // Create D3D12 Device from DXCore Adapter
+  ComPtr<ID3D12Device> d3d12_device;
+  ORT_THROW_IF_FAILED(D3D12CreateDevice(dxcore_device.Get(), D3D_FEATURE_LEVEL_11_0, IID_GRAPHICS_PPV_ARGS(d3d12_device.ReleaseAndGetAddressOf())));
+
+  // Create DML Device from D3D12 device
+  D3D12_COMMAND_QUEUE_DESC cmd_queue_desc = {};
+  cmd_queue_desc.Type = D3D12_COMMAND_LIST_TYPE_DIRECT;
+  cmd_queue_desc.Flags = D3D12_COMMAND_QUEUE_FLAG_DISABLE_GPU_TIMEOUT;
+
+  ComPtr<ID3D12CommandQueue> cmd_queue;
+  ORT_THROW_IF_FAILED(d3d12_device->CreateCommandQueue(&cmd_queue_desc, IID_GRAPHICS_PPV_ARGS(cmd_queue.ReleaseAndGetAddressOf())));
+
+  DML_CREATE_DEVICE_FLAGS flags = DML_CREATE_DEVICE_FLAG_NONE;
+
+  // In debug builds, enable the DML debug layer if the D3D12 debug layer is also enabled
+#if _DEBUG && !_GAMING_XBOX
+  ComPtr<ID3D12DebugDevice> debug_device;
+  (void)d3d12_device->QueryInterface(IID_PPV_ARGS(&debug_device));  // ignore failure
+  const bool is_d3d12_debug_layer_enabled = (debug_device != nullptr);
+
+  if (is_d3d12_debug_layer_enabled) {
+    flags |= DML_CREATE_DEVICE_FLAG_DEBUG;
+  }
+#endif
+
+  ComPtr<IDMLDevice> dml_device;
+  ORT_THROW_IF_FAILED(DMLCreateDevice1(d3d12_device.Get(),
+                                   flags,
+                                   DML_FEATURE_LEVEL_5_0,
+                                   IID_PPV_ARGS(&dml_device)));
+
+  return CreateExecutionProviderFactory_DML(dml_device.Get(), cmd_queue.Get());
+}
+
 }  // namespace onnxruntime
 
 // [[deprecated]]
@@ -216,6 +254,107 @@
   API_IMPL_END
 }
 
+bool IsHardwareAdapter(IDXCoreAdapter* adapter) {
+    bool is_hardware{ false };
+    THROW_IF_FAILED(adapter->GetProperty(
+        DXCoreAdapterProperty::IsHardware,
+        &is_hardware));
+    return is_hardware;
+}
+
+bool IsGPU(IDXCoreAdapter* compute_adapter) {
+    // Only considering hardware adapters
+    if (!IsHardwareAdapter(compute_adapter))
+        return false;
+    return compute_adapter->IsAttributeSupported(DXCORE_ADAPTER_ATTRIBUTE_D3D12_GRAPHICS);
+}
+
+bool IsNPU(IDXCoreAdapter* compute_adapter) {
+    // Only considering hardware adapters
+    if (!IsHardwareAdapter(compute_adapter))
+        return false;
+    return !(compute_adapter->IsAttributeSupported(DXCORE_ADAPTER_ATTRIBUTE_D3D12_GRAPHICS));
+}
+
+ORT_API_STATUS_IMPL(SessionOptionsAppendExecutionProvider_DML2, _In_ OrtSessionOptions* options, OrtDmlDeviceOptions* device_opts) {
+API_IMPL_BEGIN
+    OrtDmlPerformancePreference perf_pref = device_opts->perf_pref;
+    OrtDmlDeviceFilter dev_filter = device_opts->dev_filter;
+
+    // Create DXCore Adapter Factory
+    ComPtr<IDXCoreAdapterFactory> adapterFactory;
+    ORT_THROW_IF_FAILED(::DXCoreCreateAdapterFactory(adapterFactory.GetAddressOf()));
+
+    // Get a list of all the adapters that support compute
+    ComPtr<IDXCoreAdapterList> d3D12CoreComputeAdapters;
+    GUID attributes[]{ DXCORE_ADAPTER_ATTRIBUTE_D3D12_CORE_COMPUTE };
+    ORT_THROW_IF_FAILED(
+        adapterFactory->CreateAdapterList(_countof(attributes),
+            attributes,
+            d3D12CoreComputeAdapters.GetAddressOf()));
+
+    const uint32_t count{ d3D12CoreComputeAdapters->GetAdapterCount() };
+    int compute_only_device_id = -1;
+
+    // Struct for holding each adapter
+    enum class DeviceType { GPU, NPU, BadDevice};
+    struct AdapterInfo {
+        ComPtr<IDXCoreAdapter> Adapter;
+        DeviceType Type; // GPU or NPU
+    };
+
+    // Returns an adapter type based on the user supplied device filter
+    std::function<DeviceType (IDXCoreAdapter*)> filterAdapterTypeQuery[4] = {
+        nullptr, // enum cannot take 0 value
+        // Only GPUs are considered so all other devices aren't needed
+        [](IDXCoreAdapter* adapter){ return IsGPU(adapter) ? DeviceType::GPU : DeviceType::BadDevice; },
+        // Only NPUs are considered so all other devices aren't needed
+        [](IDXCoreAdapter* adapter){ return IsNPU(adapter) ? DeviceType::NPU : DeviceType::BadDevice; },
+        // Both GPUs and NPUs are considered
+        [](IDXCoreAdapter* adapter){ return IsGPU(adapter) ? DeviceType::GPU :
+        IsNPU(adapter) ? DeviceType::NPU : DeviceType::BadDevice; }
+    };
+
+    auto selected_adapters = std::vector<AdapterInfo>();
+    // Iterate through all compute capable adapters
+    for (uint32_t i = 0; i < count; ++i)
+    {
+        ComPtr<IDXCoreAdapter> candidateAdapter;
+        ORT_THROW_IF_FAILED(
+            d3D12CoreComputeAdapters->GetAdapter(i, candidateAdapter.GetAddressOf()));
+
+        // Add the adapters that are valid based on the device filter (GPU, NPU, or Both)
+        auto adapterType = filterAdapterTypeQuery[static_cast<int>(dev_filter)](candidateAdapter.Get());
+        if (adapterType != DeviceType::BadDevice) {
+            selected_adapters.push_back(AdapterInfo{candidateAdapter, adapterType});
+        }
+    }
+
+    // When considering both GPUs and NPUs sort them by performance preference
+    // of Default (Gpus first), HighPerformance (GPUs first), or LowPower (NPUs first)
+    if (dev_filter == OrtDmlDeviceFilter::Both) // considering both NPUs and GPUs
+    {
+        struct SortingPolicy
+        {
+            bool gpusFirst_ = true;
+            SortingPolicy(bool gpusFirst = true) : gpusFirst_(gpusFirst){}
+            bool operator()(const AdapterInfo& a, const AdapterInfo& b) {
+                return gpusFirst_ ? a.Type > b.Type : a.Type < b.Type;
+            }
+        };
+        auto sortingPolicy = SortingPolicy(perf_pref != OrtDmlPerformancePreference::LowPower);
+        std::sort(selected_adapters.begin(), selected_adapters.end(), sortingPolicy);
+    }
+
+    // check if num adapters is 0 (no adapters?)
+    ComPtr<IDXCoreAdapter> highest_pri_adapter = selected_adapters[0].Adapter;
+    // return the create function for a dxcore device
+    options->provider_factories.push_back(onnxruntime::DMLProviderFactoryCreator::CreateDXCore(
+        highest_pri_adapter));
+  return nullptr;
+API_IMPL_END
+}
+
 ORT_API_STATUS_IMPL(GetD3D12ResourceFromAllocation, _In_ OrtAllocator* ort_allocator, _In_ void* allocation, _Out_ ID3D12Resource** d3d_resource) {
   API_IMPL_BEGIN
 #ifdef USE_DML

onnxruntime/core/providers/dml/dml_provider_factory_creator.h

@@ -10,11 +10,14 @@
 #include "core/providers/providers.h"
 #include "core/providers/dml/dml_provider_factory.h"
 
+#include <dxcore.h>
+
 namespace onnxruntime {
 
 struct DMLProviderFactoryCreator {
   static std::shared_ptr<IExecutionProviderFactory> Create(int device_id);
   static std::shared_ptr<IExecutionProviderFactory> Create(int device_id, bool skip_software_device_check);
+  static std::shared_ptr<IExecutionProviderFactory> CreateDXCore(Microsoft::WRL::ComPtr<IDXCoreAdapter> dxcore_device);
   static Microsoft::WRL::ComPtr<ID3D12Device> CreateD3D12Device(int device_id, bool skip_software_device_check);
 };
 }  // namespace onnxruntime