Vtable hooking for d3d12, hopefully fixes overlays

src/d3d12/D3D12_Hooks.cpp

@@ -120,6 +120,17 @@ void D3D12::ExecuteCommandLists(ID3D12CommandQueue* apCommandQueue, UINT aNumCom
     d3d12.m_realExecuteCommandLists(apCommandQueue, aNumCommandLists, apcpCommandLists);
 }
 
+void* ApplyHook(void** vtable, size_t index, void* target)
+{
+    DWORD oldProtect;
+    VirtualProtect(vtable + index, 8, PAGE_EXECUTE_READWRITE, &oldProtect);
+    auto ret = vtable[index];
+    vtable[index] = target;
+    VirtualProtect(vtable + index, 8, oldProtect, nullptr);
+
+    return ret;
+}
+
 void D3D12::Hook()
 {
     int d3d12FailedHooksCount = 0;
@@ -153,7 +164,8 @@ void D3D12::Hook()
     }
     else
     {
-        if (kiero::bind(140, reinterpret_cast<void**>(&m_realPresentD3D12), &Present) != kiero::Status::Success)
+        m_realPresentD3D12 = (decltype(m_realPresentD3D12))ApplyHook(kiero::getSwapChainVtable(), 8, &Present);
+        if (m_realPresentD3D12 == nullptr)
         {
             spdlog::error("{0} Present hook failed!", d3d12type);
             ++d3d12FailedHooksCount;
@@ -164,7 +176,8 @@ void D3D12::Hook()
             ++d3d12CompleteHooksCount;
         }
 
-        if (kiero::bind(145, reinterpret_cast<void**>(&m_realResizeBuffersD3D12), &ResizeBuffers) != kiero::Status::Success)
+        m_realResizeBuffersD3D12 = (decltype(m_realResizeBuffersD3D12))ApplyHook(kiero::getSwapChainVtable(), 13, &ResizeBuffers);
+        if (m_realResizeBuffersD3D12 == nullptr) // 13
         {
             spdlog::error("{0} ResizeBuffers hook failed!", d3d12type);
             ++d3d12FailedHooksCount;
@@ -176,7 +189,8 @@ void D3D12::Hook()
         }
     }
 
-    if (kiero::bind(54, reinterpret_cast<void**>(&m_realExecuteCommandLists), &ExecuteCommandLists) != kiero::Status::Success)
+    m_realExecuteCommandLists = (decltype(m_realExecuteCommandLists))ApplyHook(kiero::getCommandQueueVtable(), 10, &ExecuteCommandLists);
+    if (m_realExecuteCommandLists == nullptr) // 10
     {
         spdlog::error("{0} ExecuteCommandLists hook failed!", d3d12type);
         ++d3d12FailedHooksCount;

src/kiero/kiero.cpp

@@ -10,6 +10,9 @@
 
 static bool g_kieroInitialized = false;
 static uint150_t* g_methodsTable = NULL;
+static void** g_swapChainVtable = nullptr;
+static void** g_commandListVtable = nullptr;
+static void** g_commandQueueVtable = nullptr;
 static uintptr_t g_commandQueueOffset = 0;
 static bool g_isDownLevelDevice = false;
 
@@ -211,6 +214,11 @@ kiero::Status::Enum kiero::init()
     }
 
     g_methodsTable = (uint150_t*)::calloc(176, sizeof(uint150_t));
+
+    g_swapChainVtable = *(void***)swapChain.operator IDXGISwapChain3 *();
+    g_commandListVtable = *(void***)commandList.operator ID3D12GraphicsCommandList *();
+    g_commandQueueVtable = *(void***)commandQueue.operator ID3D12CommandQueue*();
+
     ::memcpy(g_methodsTable, *(uint150_t**)(void*)device, 44 * sizeof(uint150_t));
     ::memcpy(g_methodsTable + 44, *(uint150_t**)(void*)commandQueue, 19 * sizeof(uint150_t));
     ::memcpy(g_methodsTable + 44 + 19, *(uint150_t**)(void*)commandAllocator, 9 * sizeof(uint150_t));
@@ -269,6 +277,21 @@ void kiero::unbind(uint16_t _index)
     }
 }
 
+void** kiero::getSwapChainVtable()
+{
+    return g_swapChainVtable;
+}
+
+void** kiero::getCommandListVtable()
+{
+    return g_commandListVtable;
+}
+
+void** kiero::getCommandQueueVtable()
+{
+    return g_commandQueueVtable;
+}
+
 uint150_t* kiero::getMethodsTable()
 {
     return g_methodsTable;

src/kiero/kiero.h

@@ -41,6 +41,10 @@ namespace kiero
     Status::Enum bind(uint16_t index, void** original, void* function);
     void unbind(uint16_t index);
 
+    void** getSwapChainVtable();
+    void** getCommandListVtable();
+    void** getCommandQueueVtable();
+
     uint150_t* getMethodsTable();
     uintptr_t getCommandQueueOffset();
     bool isDownLevelDevice();