From 160d17eabb6b05979fa11f80ab9446e9a6e55c0c Mon Sep 17 00:00:00 2001
From: Andrei Alexeyev <akari@taisei-project.org>
Date: Fri, 8 Nov 2024 14:18:51 +0100
Subject: [PATCH] GPU Vulkan: release submitted command buffer after defrag

Fixes #11429
---
 src/gpu/vulkan/SDL_gpu_vulkan.c | 38 ++++++++++++++++++++++-----------
 1 file changed, 25 insertions(+), 13 deletions(-)

diff --git a/src/gpu/vulkan/SDL_gpu_vulkan.c b/src/gpu/vulkan/SDL_gpu_vulkan.c
index 29951c609595c..92c2df9463165 100644
--- a/src/gpu/vulkan/SDL_gpu_vulkan.c
+++ b/src/gpu/vulkan/SDL_gpu_vulkan.c
@@ -10219,6 +10219,22 @@ static SDL_GPUFence *VULKAN_SubmitAndAcquireFence(
     return (SDL_GPUFence *)vulkanCommandBuffer->inFlightFence;
 }
 
+static void VULKAN_INTERNAL_ReleaseCommandBuffer(VulkanCommandBuffer *vulkanCommandBuffer)
+{
+    VulkanRenderer *renderer = vulkanCommandBuffer->renderer;
+
+    if (renderer->submittedCommandBufferCount + 1 >= renderer->submittedCommandBufferCapacity) {
+        renderer->submittedCommandBufferCapacity = renderer->submittedCommandBufferCount + 1;
+
+        renderer->submittedCommandBuffers = SDL_realloc(
+            renderer->submittedCommandBuffers,
+            sizeof(VulkanCommandBuffer *) * renderer->submittedCommandBufferCapacity);
+    }
+
+    renderer->submittedCommandBuffers[renderer->submittedCommandBufferCount] = vulkanCommandBuffer;
+    renderer->submittedCommandBufferCount += 1;
+}
+
 static bool VULKAN_Submit(
     SDL_GPUCommandBuffer *commandBuffer)
 {
@@ -10292,19 +10308,6 @@ static bool VULKAN_Submit(
         CHECK_VULKAN_ERROR_AND_RETURN(vulkanResult, vkQueueSubmit, false)
     }
 
-    // Mark command buffers as submitted
-
-    if (renderer->submittedCommandBufferCount + 1 >= renderer->submittedCommandBufferCapacity) {
-        renderer->submittedCommandBufferCapacity = renderer->submittedCommandBufferCount + 1;
-
-        renderer->submittedCommandBuffers = SDL_realloc(
-            renderer->submittedCommandBuffers,
-            sizeof(VulkanCommandBuffer *) * renderer->submittedCommandBufferCapacity);
-    }
-
-    renderer->submittedCommandBuffers[renderer->submittedCommandBufferCount] = vulkanCommandBuffer;
-    renderer->submittedCommandBufferCount += 1;
-
     // Present, if applicable
     bool result = true;
 
@@ -10336,6 +10339,11 @@ static bool VULKAN_Submit(
                 presentData->windowData->needsSwapchainRecreate = true;
             }
         } else {
+            if (presentResult != VK_SUCCESS) {
+                VULKAN_INTERNAL_ReleaseCommandBuffer(vulkanCommandBuffer);
+                SDL_UnlockMutex(renderer->submitLock);
+            }
+
             CHECK_VULKAN_ERROR_AND_RETURN(presentResult, vkQueuePresentKHR, false)
         }
 
@@ -10391,6 +10399,10 @@ static bool VULKAN_Submit(
         result = VULKAN_INTERNAL_DefragmentMemory(renderer);
     }
 
+    // Mark command buffer as submitted
+    // This must happen after defrag, because it will try to acquire new command buffers.
+    VULKAN_INTERNAL_ReleaseCommandBuffer(vulkanCommandBuffer);
+
     SDL_UnlockMutex(renderer->submitLock);
 
     return result;