vulkan: use subpass dependencies for layout transitions when switchin…

…g canvases.
love2d · Oct 1, 2024 · eeec242 · eeec242
1 parent 3a6833f
commit eeec242
Show file tree

Hide file tree

Showing 4 changed files with 69 additions and 61 deletions.
diff --git a/src/modules/graphics/vulkan/Graphics.cpp b/src/modules/graphics/vulkan/Graphics.cpp
@@ -1210,8 +1210,8 @@ static bool computeDispatchBarrierFlags(Shader *shader, VkAccessFlags &dstAccess
  // TODO: this is pretty messy.
  VkAccessFlags texAccessFlags = 0;
  VkPipelineStageFlags texStageFlags = 0;
- const PixelFormatInfo &formatInfo = getPixelFormatInfo(tex->getPixelFormat());
- Vulkan::setImageLayoutTransitionOptions(false, tex->isRenderTarget(), formatInfo, VK_IMAGE_LAYOUT_GENERAL, texAccessFlags, texStageFlags);
+ bool depthStencil  = isPixelFormatDepthStencil(tex->getPixelFormat());
+ Vulkan::setImageLayoutTransitionOptions(false, tex->isRenderTarget(), depthStencil, VK_IMAGE_LAYOUT_GENERAL, texAccessFlags, texStageFlags);
 
  dstAccessFlags |= texAccessFlags;
  dstStageFlags |= texStageFlags;
@@ -2197,6 +2197,25 @@ VkRenderPass Graphics::createRenderPass(RenderPassConfiguration &configuration)
  VkSubpassDescription subPass{};
  subPass.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS;
 
+ VkSubpassDependency beginDependency{};
+ beginDependency.srcSubpass = VK_SUBPASS_EXTERNAL;
+ beginDependency.dstSubpass = 0;
+ beginDependency.srcStageMask = VK_PIPELINE_STAGE_TRANSFER_BIT;
+ beginDependency.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
+ beginDependency.dstStageMask = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT
+ | VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT;
+ beginDependency.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT
+ | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT;
+
+ VkSubpassDependency endDependency{};
+ endDependency.srcSubpass = 0;
+ endDependency.dstSubpass = VK_SUBPASS_EXTERNAL;
+ endDependency.srcStageMask = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT;
+ endDependency.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT
+ | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT;
+ endDependency.dstStageMask = VK_PIPELINE_STAGE_TRANSFER_BIT;
+ endDependency.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
+
  std::vector<VkAttachmentDescription> attachments;
  std::vector<VkAttachmentReference> colorAttachmentRefs;
 
@@ -2215,9 +2234,21 @@ VkRenderPass Graphics::createRenderPass(RenderPassConfiguration &configuration)
  colorDescription.storeOp = VK_ATTACHMENT_STORE_OP_STORE;
  colorDescription.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
  colorDescription.stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE;
- colorDescription.initialLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
- colorDescription.finalLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
+ colorDescription.initialLayout = colorAttachment.layout;
+ colorDescription.finalLayout = colorAttachment.layout;
  attachments.push_back(colorDescription);
+
+ VkAccessFlags texBeginAccessFlags = 0;
+ VkPipelineStageFlags texBeginStageFlags = 0;
+ Vulkan::setImageLayoutTransitionOptions(true, true, false, colorAttachment.layout, texBeginAccessFlags, texBeginStageFlags);
+ beginDependency.srcAccessMask |= texBeginAccessFlags;
+ beginDependency.srcStageMask |= texBeginStageFlags;
+
+ VkAccessFlags texEndAccessFlags = 0;
+ VkPipelineStageFlags texEndStageFlags = 0;
+ Vulkan::setImageLayoutTransitionOptions(false, true, false, colorAttachment.layout, texEndAccessFlags, texEndStageFlags);
+ endDependency.dstAccessMask |= texEndAccessFlags;
+ endDependency.dstStageMask |= texEndStageFlags;
  }
 
  subPass.colorAttachmentCount = static_cast<uint32_t>(colorAttachmentRefs.size());
@@ -2237,9 +2268,21 @@ VkRenderPass Graphics::createRenderPass(RenderPassConfiguration &configuration)
  depthStencilAttachment.storeOp = VK_ATTACHMENT_STORE_OP_STORE;
  depthStencilAttachment.stencilLoadOp = configuration.staticData.depthStencilAttachment.stencilLoadOp;
  depthStencilAttachment.stencilStoreOp = VK_ATTACHMENT_STORE_OP_STORE;
- depthStencilAttachment.initialLayout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL;
- depthStencilAttachment.finalLayout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL;
+ depthStencilAttachment.initialLayout = configuration.staticData.depthStencilAttachment.layout;
+ depthStencilAttachment.finalLayout = configuration.staticData.depthStencilAttachment.layout;
  attachments.push_back(depthStencilAttachment);
+
+ VkAccessFlags texBeginAccessFlags = 0;
+ VkPipelineStageFlags texBeginStageFlags = 0;
+ Vulkan::setImageLayoutTransitionOptions(true, true, true, configuration.staticData.depthStencilAttachment.layout, texBeginAccessFlags, texBeginStageFlags);
+ beginDependency.srcAccessMask |= texBeginAccessFlags;
+ beginDependency.srcStageMask |= texBeginStageFlags;
+
+ VkAccessFlags texEndAccessFlags = 0;
+ VkPipelineStageFlags texEndStageFlags = 0;
+ Vulkan::setImageLayoutTransitionOptions(false, true, true, configuration.staticData.depthStencilAttachment.layout, texEndAccessFlags, texEndStageFlags);
+ endDependency.dstAccessMask |= texEndAccessFlags;
+ endDependency.dstStageMask |= texEndStageFlags;
  }
 
  VkAttachmentReference colorAttachmentResolveRef{};
@@ -2261,23 +2304,7 @@ VkRenderPass Graphics::createRenderPass(RenderPassConfiguration &configuration)
  attachments.push_back(colorAttachmentResolve);
  }
 
- VkSubpassDependency dependency{};
- dependency.srcSubpass = VK_SUBPASS_EXTERNAL;
- dependency.dstSubpass = 0;
- dependency.srcStageMask = VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT;
- dependency.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
- dependency.dstStageMask = VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT;
- dependency.dstAccessMask = VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT;
-
- VkSubpassDependency readbackDependency{};
- readbackDependency.srcSubpass = 0;
- readbackDependency.dstSubpass = VK_SUBPASS_EXTERNAL;
- readbackDependency.srcStageMask = VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT;
- readbackDependency.srcAccessMask = VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT;
- readbackDependency.dstStageMask = VK_PIPELINE_STAGE_TRANSFER_BIT;
- readbackDependency.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
-
- std::array<VkSubpassDependency, 2> dependencies = { dependency, readbackDependency };
+ std::array<VkSubpassDependency, 2> dependencies = { beginDependency, endDependency };
 
  VkRenderPassCreateInfo createInfo{};
  createInfo.sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO;
@@ -2486,14 +2513,13 @@ void Graphics::setDefaultRenderPass()
  renderPassState.msaa = msaaSamples;
  renderPassState.numColorAttachments = 1;
  renderPassState.packedColorAttachmentFormats = (uint8)swapChainPixelFormat;
- renderPassState.transitionImages.clear();
 
  RenderPassConfiguration renderPassConfiguration{};
 
- renderPassConfiguration.colorAttachments.push_back({ swapChainImageFormat, VK_ATTACHMENT_LOAD_OP_LOAD, msaaSamples });
+ renderPassConfiguration.colorAttachments.push_back({ swapChainImageFormat, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, VK_ATTACHMENT_LOAD_OP_LOAD, msaaSamples });
 
  VkFormat dsformat = backbufferHasDepth || backbufferHasStencil ? depthStencilFormat : VK_FORMAT_UNDEFINED;
- renderPassConfiguration.staticData.depthStencilAttachment = { dsformat, VK_ATTACHMENT_LOAD_OP_LOAD, VK_ATTACHMENT_LOAD_OP_LOAD, msaaSamples };
+ renderPassConfiguration.staticData.depthStencilAttachment = { dsformat, VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL, VK_ATTACHMENT_LOAD_OP_LOAD, VK_ATTACHMENT_LOAD_OP_LOAD, msaaSamples };
  if (msaaSamples & VK_SAMPLE_COUNT_1_BIT)
  renderPassConfiguration.staticData.resolve = false;
  else
@@ -2555,42 +2581,28 @@ void Graphics::setRenderPass(const RenderTargets &rts, int pixelw, int pixelh, b
  for (const auto &color : rts.colors)
  renderPassConfiguration.colorAttachments.push_back({ 
  Vulkan::getTextureFormat(color.texture->getPixelFormat()).internalFormat,
+ ((Texture*)color.texture)->getImageLayout(),
  VK_ATTACHMENT_LOAD_OP_LOAD,
  dynamic_cast<Texture*>(color.texture)->getMsaaSamples() });
  if (rts.depthStencil.texture != nullptr)
  renderPassConfiguration.staticData.depthStencilAttachment = {
  Vulkan::getTextureFormat(rts.depthStencil.texture->getPixelFormat()).internalFormat,
+ ((Texture*)rts.depthStencil.texture)->getImageLayout(),
  VK_ATTACHMENT_LOAD_OP_LOAD,
  VK_ATTACHMENT_LOAD_OP_LOAD,
  dynamic_cast<Texture*>(rts.depthStencil.texture)->getMsaaSamples() };
 
  FramebufferConfiguration configuration{};
 
- std::vector<std::tuple<VkImage, PixelFormat, bool, VkImageLayout, VkImageLayout, int, int>> transitionImages;
-
  for (const auto &color : rts.colors)
  {
  auto tex = (Texture*)color.texture;
  configuration.colorViews.push_back(tex->getRenderTargetView(color.mipmap, color.slice));
- const Texture::ViewInfo &viewinfo = tex->getRootViewInfo();
- VkImageLayout imagelayout = tex->getImageLayout();
- if (imagelayout != VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL)
- {
- transitionImages.push_back({ (VkImage)tex->getHandle(), tex->getPixelFormat(), tex->isRenderTarget(), imagelayout, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
- viewinfo.startMipmap + color.mipmap, viewinfo.startLayer + color.slice });
- }
  }
  if (rts.depthStencil.texture != nullptr)
  {
  auto tex = (Texture*)rts.depthStencil.texture;
  configuration.staticData.depthView = tex->getRenderTargetView(rts.depthStencil.mipmap, rts.depthStencil.slice);
- const Texture::ViewInfo &viewinfo = tex->getRootViewInfo();
- VkImageLayout imagelayout = tex->getImageLayout();
- if (imagelayout != VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL)
- {
- transitionImages.push_back({ (VkImage)tex->getHandle(), tex->getPixelFormat(), tex->isRenderTarget(), imagelayout, VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL,
- viewinfo.startMipmap + rts.depthStencil.mipmap, viewinfo.startLayer + rts.depthStencil.slice });
- }
  }
 
  configuration.staticData.width = static_cast<uint32_t>(pixelw);
@@ -2619,7 +2631,6 @@ void Graphics::setRenderPass(const RenderTargets &rts, int pixelw, int pixelh, b
  renderPassState.packedColorAttachmentFormats = 0;
  for (size_t i = 0; i < rts.colors.size(); i++)
  renderPassState.packedColorAttachmentFormats |= ((uint64)rts.colors[i].texture->getPixelFormat()) << (i * 8ull);
- renderPassState.transitionImages = std::move(transitionImages);
 }
 
 void Graphics::startRenderPass()
@@ -2644,9 +2655,6 @@ void Graphics::startRenderPass()
  renderPassState.framebufferConfiguration.staticData.renderPass = renderPassState.beginInfo.renderPass;
  renderPassState.beginInfo.framebuffer = getFramebuffer(renderPassState.framebufferConfiguration);
 
- for (const auto &[image, format, renderTarget, imageLayout, renderLayout, rootmip, rootlayer] : renderPassState.transitionImages)
- Vulkan::cmdTransitionImageLayout(commandBuffers.at(currentFrame), image, format, renderTarget, imageLayout, renderLayout, rootmip, 1, rootlayer, 1);
-
  vkCmdBeginRenderPass(commandBuffers.at(currentFrame), &renderPassState.beginInfo, VK_SUBPASS_CONTENTS_INLINE);
 
  applyScissor();
@@ -2658,9 +2666,6 @@ void Graphics::endRenderPass()
 
  vkCmdEndRenderPass(commandBuffers.at(currentFrame));
 
- for (const auto &[image, format, renderTarget, imageLayout, renderLayout, rootmip, rootlayer] : renderPassState.transitionImages)
- Vulkan::cmdTransitionImageLayout(commandBuffers.at(currentFrame), image, format, renderTarget, renderLayout, imageLayout, rootmip, 1, rootlayer, 1);
-
  for (auto &colorAttachment : renderPassState.renderPassConfiguration.colorAttachments)
  colorAttachment.loadOp = VK_ATTACHMENT_LOAD_OP_LOAD;
 

diff --git a/src/modules/graphics/vulkan/Graphics.h b/src/modules/graphics/vulkan/Graphics.h
@@ -49,12 +49,14 @@ namespace vulkan
 struct ColorAttachment
 {
  VkFormat format = VK_FORMAT_UNDEFINED;
+ VkImageLayout layout = VK_IMAGE_LAYOUT_UNDEFINED;
  VkAttachmentLoadOp loadOp = VK_ATTACHMENT_LOAD_OP_LOAD;
  VkSampleCountFlagBits msaaSamples = VK_SAMPLE_COUNT_1_BIT;
 
  bool operator==(const ColorAttachment &attachment) const
  {
- return format == attachment.format && 
+ return format == attachment.format &&
+ layout == attachment.layout &&
  loadOp == attachment.loadOp &&
  msaaSamples == attachment.msaaSamples;
  }
@@ -63,13 +65,15 @@ struct ColorAttachment
 struct DepthStencilAttachment
 {
  VkFormat format = VK_FORMAT_UNDEFINED;
+ VkImageLayout layout = VK_IMAGE_LAYOUT_UNDEFINED;
  VkAttachmentLoadOp depthLoadOp = VK_ATTACHMENT_LOAD_OP_LOAD;
  VkAttachmentLoadOp stencilLoadOp = VK_ATTACHMENT_LOAD_OP_LOAD;
  VkSampleCountFlagBits msaaSamples = VK_SAMPLE_COUNT_1_BIT;
 
  bool operator==(const DepthStencilAttachment &attachment) const
  {
  return format == attachment.format &&
+ layout == attachment.layout &&
  depthLoadOp == attachment.depthLoadOp &&
  stencilLoadOp == attachment.stencilLoadOp &&
  msaaSamples == attachment.msaaSamples;
@@ -196,7 +200,6 @@ struct RenderpassState
  RenderPassConfiguration renderPassConfiguration{};
  FramebufferConfiguration framebufferConfiguration{};
  VkPipeline pipeline = VK_NULL_HANDLE;
- std::vector<std::tuple<VkImage, PixelFormat, bool, VkImageLayout, VkImageLayout, int, int>> transitionImages;
  uint32_t numColorAttachments = 0;
  uint64 packedColorAttachmentFormats = 0;
  float width = 0.0f;

diff --git a/src/modules/graphics/vulkan/Vulkan.cpp b/src/modules/graphics/vulkan/Vulkan.cpp
@@ -822,7 +822,7 @@ VkIndexType Vulkan::getVulkanIndexBufferType(IndexDataType type)
  }
 }
 
-void Vulkan::setImageLayoutTransitionOptions(bool previous, bool renderTarget, const PixelFormatInfo &info, VkImageLayout layout, VkAccessFlags &accessMask, VkPipelineStageFlags &stageFlags)
+void Vulkan::setImageLayoutTransitionOptions(bool previous, bool renderTarget, bool depthStencil, VkImageLayout layout, VkAccessFlags &accessMask, VkPipelineStageFlags &stageFlags)
 {
  switch (layout)
  {
@@ -840,16 +840,16 @@ void Vulkan::setImageLayoutTransitionOptions(bool previous, bool renderTarget, c
  stageFlags = VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_VERTEX_SHADER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT;
  if (renderTarget)
  {
- if (info.color)
- {
- accessMask |= VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
- stageFlags |= VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
- }
- if (info.depth || info.stencil)
+ if (depthStencil)
  {
  accessMask |= VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT;
  stageFlags |= VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT;
  }
+ else
+ {
+ accessMask |= VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
+ stageFlags |= VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
+ }
  }
  break;
  case VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL:
@@ -900,8 +900,8 @@ void Vulkan::cmdTransitionImageLayout(VkCommandBuffer commandBuffer, VkImage ima
  VkPipelineStageFlags sourceStage;
  VkPipelineStageFlags destinationStage;
 
- setImageLayoutTransitionOptions(true, renderTarget, info, oldLayout, barrier.srcAccessMask, sourceStage);
- setImageLayoutTransitionOptions(false, renderTarget, info, newLayout, barrier.dstAccessMask, destinationStage);
+ setImageLayoutTransitionOptions(true, renderTarget, info.depth || info.stencil, oldLayout, barrier.srcAccessMask, sourceStage);
+ setImageLayoutTransitionOptions(false, renderTarget, info.depth || info.stencil, newLayout, barrier.dstAccessMask, destinationStage);
 
  if (info.color)
  barrier.subresourceRange.aspectMask |= VK_IMAGE_ASPECT_COLOR_BIT;

diff --git a/src/modules/graphics/vulkan/Vulkan.h b/src/modules/graphics/vulkan/Vulkan.h
@@ -81,7 +81,7 @@ class Vulkan
  static VkIndexType getVulkanIndexBufferType(IndexDataType type);
 
  static void setImageLayoutTransitionOptions(
- bool previous, bool renderTarget, const PixelFormatInfo &info, VkImageLayout layout, VkAccessFlags &accessMask, VkPipelineStageFlags &stageFlags);
+ bool previous, bool renderTarget, bool depthStencil, VkImageLayout layout, VkAccessFlags &accessMask, VkPipelineStageFlags &stageFlags);
 
  static void cmdTransitionImageLayout(
  VkCommandBuffer, VkImage, PixelFormat format, bool renderTarget, VkImageLayout oldLayout, VkImageLayout newLayout,