diff --git a/filament/backend/src/vulkan/VulkanCommands.cpp b/filament/backend/src/vulkan/VulkanCommands.cpp index 4f9957753d3..95c31657e13 100644 --- a/filament/backend/src/vulkan/VulkanCommands.cpp +++ b/filament/backend/src/vulkan/VulkanCommands.cpp @@ -47,7 +47,8 @@ VulkanCmdFence::VulkanCmdFence(VkFence ifence) VulkanCommandBuffer::VulkanCommandBuffer(VulkanResourceAllocator* allocator, VkDevice device, VkCommandPool pool) - : mResourceManager(allocator) { + : mResourceManager(allocator), + mPipeline(VK_NULL_HANDLE) { // Create the low-level command buffer. const VkCommandBufferAllocateInfo allocateInfo{ .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, diff --git a/filament/backend/src/vulkan/VulkanCommands.h b/filament/backend/src/vulkan/VulkanCommands.h index c3e6697b9d4..e3c7a92f190 100644 --- a/filament/backend/src/vulkan/VulkanCommands.h +++ b/filament/backend/src/vulkan/VulkanCommands.h @@ -89,6 +89,15 @@ struct VulkanCommandBuffer { inline void reset() { fence.reset(); mResourceManager.clear(); + mPipeline = VK_NULL_HANDLE; + } + + inline void setPipeline(VkPipeline pipeline) { + mPipeline = pipeline; + } + + inline VkPipeline pipeline() const { + return mPipeline; } inline VkCommandBuffer buffer() const { @@ -103,6 +112,7 @@ struct VulkanCommandBuffer { private: VulkanAcquireOnlyResourceManager mResourceManager; VkCommandBuffer mBuffer; + VkPipeline mPipeline; }; // Allows classes to be notified after a new command buffer has been activated. diff --git a/filament/backend/src/vulkan/VulkanDriver.cpp b/filament/backend/src/vulkan/VulkanDriver.cpp index 5b8d6de9f88..e1cd3d7cf74 100644 --- a/filament/backend/src/vulkan/VulkanDriver.cpp +++ b/filament/backend/src/vulkan/VulkanDriver.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "vulkan/VulkanDriver.h" +#include "VulkanDriver.h" #include "CommandStreamDispatcher.h" #include "DataReshaper.h" @@ -102,6 +102,15 @@ VulkanTexture* createEmptyTexture(VkDevice device, VkPhysicalDevice physicalDevi return emptyTexture; } +VulkanBufferObject* createEmptyBufferObject(VmaAllocator allocator, VulkanStagePool& stagePool, + VulkanCommands* commands) { + VulkanBufferObject* obj = + new VulkanBufferObject(allocator, stagePool, 1, BufferObjectBinding::UNIFORM); + uint8_t byte = 0; + obj->buffer.loadFromCpu(commands->get().buffer(), &byte, 0, 1); + return obj; +} + #if FVK_ENABLED(FVK_DEBUG_VALIDATION) VKAPI_ATTR VkBool32 VKAPI_CALL debugReportCallback(VkDebugReportFlagsEXT flags, VkDebugReportObjectTypeEXT objectType, uint64_t object, size_t location, @@ -214,12 +223,14 @@ VulkanDriver::VulkanDriver(VulkanPlatform* platform, VulkanContext const& contex mThreadSafeResourceManager(&mResourceAllocator), mCommands(mPlatform->getDevice(), mPlatform->getGraphicsQueue(), mPlatform->getGraphicsQueueFamilyIndex(), &mContext, &mResourceAllocator), - mPipelineCache(&mResourceAllocator), + mPipelineLayoutCache(mPlatform->getDevice(), &mResourceAllocator), + mPipelineCache(mPlatform->getDevice(), mAllocator), mStagePool(mAllocator, &mCommands), mFramebufferCache(mPlatform->getDevice()), mSamplerCache(mPlatform->getDevice()), mBlitter(mPlatform->getPhysicalDevice(), &mCommands), mReadPixels(mPlatform->getDevice()), + mDescriptorSetManager(mPlatform->getDevice(), &mResourceAllocator), mIsSRGBSwapChainSupported(mPlatform->getCustomization().isSRGBSwapChainSupported) { #if FVK_ENABLED(FVK_DEBUG_DEBUG_UTILS) @@ -243,13 +254,17 @@ VulkanDriver::VulkanDriver(VulkanPlatform* platform, VulkanContext const& contex #endif mTimestamps = std::make_unique(mPlatform->getDevice()); - mCommands.setObserver(&mPipelineCache); - mPipelineCache.setDevice(mPlatform->getDevice(), mAllocator); mEmptyTexture = createEmptyTexture(mPlatform->getDevice(), mPlatform->getPhysicalDevice(), mContext, mAllocator, &mCommands, mStagePool); + mEmptyBufferObject = createEmptyBufferObject(mAllocator, mStagePool, &mCommands); - mPipelineCache.setDummyTexture(mEmptyTexture->getPrimaryImageView()); + mDescriptorSetManager.setPlaceHolders(mSamplerCache.getSampler({}), mEmptyTexture, + mEmptyBufferObject); + + mGetPipelineFunction = [this](VulkanDescriptorSetLayoutList const& layouts) { + return mPipelineLayoutCache.getLayout(layouts); + }; } VulkanDriver::~VulkanDriver() noexcept = default; @@ -310,13 +325,14 @@ ShaderModel VulkanDriver::getShaderModel() const noexcept { } void VulkanDriver::terminate() { + delete mEmptyBufferObject; + delete mEmptyTexture; + // Command buffers should come first since it might have commands depending on resources that // are about to be destroyed. mCommands.terminate(); - delete mEmptyTexture; mResourceManager.clear(); - mTimestamps.reset(); mBlitter.terminate(); @@ -329,6 +345,12 @@ void VulkanDriver::terminate() { mPipelineCache.terminate(); mFramebufferCache.reset(); mSamplerCache.terminate(); + mDescriptorSetManager.terminate(); + mPipelineLayoutCache.terminate(); + +#if FVK_ENABLED(FVK_DEBUG_RESOURCE_LEAK) + mResourceAllocator.print(); +#endif vmaDestroyAllocator(mAllocator); @@ -360,6 +382,8 @@ void VulkanDriver::collectGarbage() { mCommands.gc(); mStagePool.gc(); mFramebufferCache.gc(); + mPipelineCache.gc(); + mDescriptorSetManager.gc(); #if FVK_ENABLED(FVK_DEBUG_RESOURCE_LEAK) mResourceAllocator.print(); @@ -489,9 +513,6 @@ void VulkanDriver::destroyBufferObject(Handle boh) { return; } auto bufferObject = mResourceAllocator.handle_cast(boh); - if (bufferObject->bindingType == BufferObjectBinding::UNIFORM) { - mPipelineCache.unbindUniformBuffer(bufferObject->buffer.getGpuBuffer()); - } mResourceManager.release(bufferObject); } @@ -542,6 +563,7 @@ void VulkanDriver::destroyProgram(Handle ph) { return; } auto vkprogram = mResourceAllocator.handle_cast(ph); + mDescriptorSetManager.clearProgram(vkprogram); mResourceManager.release(vkprogram); } @@ -1429,12 +1451,7 @@ void VulkanDriver::endRenderPass(int) { 0, 1, &barrier, 0, nullptr, 0, nullptr); } - if (mCurrentRenderPass.currentSubpass > 0) { - for (uint32_t i = 0; i < VulkanPipelineCache::INPUT_ATTACHMENT_COUNT; i++) { - mPipelineCache.bindInputAttachment(i, {}); - } - mCurrentRenderPass.currentSubpass = 0; - } + mDescriptorSetManager.clearState(); mCurrentRenderPass.renderTarget = nullptr; mCurrentRenderPass.renderPass = VK_NULL_HANDLE; FVK_SYSTRACE_END(); @@ -1453,15 +1470,9 @@ void VulkanDriver::nextSubpass(int) { mPipelineCache.bindRenderPass(mCurrentRenderPass.renderPass, ++mCurrentRenderPass.currentSubpass); - for (uint32_t i = 0; i < VulkanPipelineCache::INPUT_ATTACHMENT_COUNT; i++) { - if ((1 << i) & mCurrentRenderPass.params.subpassMask) { - VulkanAttachment subpassInput = renderTarget->getColor(i); - VkDescriptorImageInfo info = { - .imageView = subpassInput.getImageView(VK_IMAGE_ASPECT_COLOR_BIT), - .imageLayout = ImgUtil::getVkLayout(subpassInput.getLayout()), - }; - mPipelineCache.bindInputAttachment(i, info); - } + if (mCurrentRenderPass.params.subpassMask & 0x1) { + VulkanAttachment subpassInput = renderTarget->getColor(0); + mDescriptorSetManager.updateInputAttachment({}, subpassInput); } } @@ -1501,25 +1512,24 @@ void VulkanDriver::commit(Handle sch) { void VulkanDriver::bindUniformBuffer(uint32_t index, Handle boh) { auto* bo = mResourceAllocator.handle_cast(boh); - const VkDeviceSize offset = 0; - const VkDeviceSize size = VK_WHOLE_SIZE; - mPipelineCache.bindUniformBufferObject((uint32_t) index, bo, offset, size); + VkDeviceSize const offset = 0; + VkDeviceSize const size = VK_WHOLE_SIZE; + mDescriptorSetManager.updateBuffer({}, (uint32_t) index, bo, offset, size); } void VulkanDriver::bindBufferRange(BufferObjectBinding bindingType, uint32_t index, Handle boh, uint32_t offset, uint32_t size) { - assert_invariant(bindingType == BufferObjectBinding::SHADER_STORAGE || - bindingType == BufferObjectBinding::UNIFORM); + assert_invariant(bindingType == BufferObjectBinding::UNIFORM); // TODO: implement BufferObjectBinding::SHADER_STORAGE case auto* bo = mResourceAllocator.handle_cast(boh); - mPipelineCache.bindUniformBufferObject((uint32_t)index, bo, offset, size); + mDescriptorSetManager.updateBuffer({}, (uint32_t) index, bo, offset, size); } void VulkanDriver::unbindBuffer(BufferObjectBinding bindingType, uint32_t index) { - // TODO: implement unbindBuffer() + mDescriptorSetManager.clearBuffer((uint32_t) index); } void VulkanDriver::bindSamplers(uint32_t index, Handle sbh) { @@ -1767,21 +1777,14 @@ void VulkanDriver::bindPipeline(PipelineState pipelineState) { // where "SamplerBinding" is the integer in the GLSL, and SamplerGroupBinding is the abstract // Filament concept used to form groups of samplers. - VkDescriptorImageInfo samplerInfo[VulkanPipelineCache::SAMPLER_BINDING_COUNT] = {}; - VulkanTexture* samplerTextures[VulkanPipelineCache::SAMPLER_BINDING_COUNT] = {nullptr}; - auto const& bindingToSamplerIndex = program->getBindingToSamplerIndex(); - UsageFlags usage = program->getUsage(); - #if FVK_ENABLED_DEBUG_SAMPLER_NAME auto const& bindingToName = program->getBindingToName(); #endif for (auto binding: program->getBindings()) { uint16_t const indexPair = bindingToSamplerIndex[binding]; - if (indexPair == 0xffff) { - usage = VulkanPipelineCache::disableUsageFlags(binding, usage); continue; } @@ -1790,18 +1793,14 @@ void VulkanDriver::bindPipeline(PipelineState pipelineState) { VulkanSamplerGroup* vksb = mSamplerBindings[samplerGroupInd]; if (!vksb) { - usage = VulkanPipelineCache::disableUsageFlags(binding, usage); continue; } SamplerDescriptor const* boundSampler = ((SamplerDescriptor*) vksb->sb->data()) + samplerInd; if (UTILS_UNLIKELY(!boundSampler->t)) { - usage = VulkanPipelineCache::disableUsageFlags(binding, usage); continue; } - VulkanTexture* texture = mResourceAllocator.handle_cast(boundSampler->t); - VkImageViewType const expectedType = texture->getViewType(); // TODO: can this uninitialized check be checked in a higher layer? // This fallback path is very flaky because the dummy texture might not have @@ -1815,43 +1814,20 @@ void VulkanDriver::bindPipeline(PipelineState pipelineState) { texture = mEmptyTexture; } - SamplerParams const& samplerParams = boundSampler->s; - VkSampler const vksampler = mSamplerCache.getSampler(samplerParams); - #if FVK_ENABLED_DEBUG_SAMPLER_NAME VulkanDriver::DebugUtils::setName(VK_OBJECT_TYPE_SAMPLER, reinterpret_cast(vksampler), bindingToName[binding].c_str()); + VulkanDriver::DebugUtils::setName(VK_OBJECT_TYPE_SAMPLER, + reinterpret_cast(samplerInfo.sampler), bindingToName[binding].c_str()); #endif - VkImageView imageView = VK_NULL_HANDLE; - VkImageSubresourceRange const range = texture->getPrimaryViewRange(); - if (any(texture->usage & TextureUsage::DEPTH_ATTACHMENT) && - expectedType == VK_IMAGE_VIEW_TYPE_2D) { - // If the sampler is part of a mipmapped depth texture, where one of the level *can* be - // an attachment, then the sampler for this texture has the same view properties as a - // view for an attachment. Therefore, we can use getAttachmentView to get a - // corresponding VkImageView. - imageView = texture->getAttachmentView(range); - } else { - imageView = texture->getViewForType(range, expectedType); - } + VkSampler const vksampler = mSamplerCache.getSampler(boundSampler->s); - samplerInfo[binding] = { - .sampler = vksampler, - .imageView = imageView, - .imageLayout = ImgUtil::getVkLayout(texture->getPrimaryImageLayout()) - }; - samplerTextures[binding] = texture; + mDescriptorSetManager.updateSampler({}, binding, texture, vksampler); } - mPipelineCache.bindSamplers(samplerInfo, samplerTextures, usage); - - // Bind a new pipeline if the pipeline state changed. - // If allocation failed, skip the draw call and bail. We do not emit an error since the - // validation layer will already do so. - if (!mPipelineCache.bindPipeline(commands)) { - return; - } + mPipelineCache.bindLayout(mDescriptorSetManager.bind(commands, program, mGetPipelineFunction)); + mPipelineCache.bindPipeline(commands); FVK_SYSTRACE_END(); } @@ -1891,12 +1867,8 @@ void VulkanDriver::draw2(uint32_t indexOffset, uint32_t indexCount, uint32_t ins VulkanCommandBuffer& commands = mCommands.get(); VkCommandBuffer cmdbuffer = commands.buffer(); - // Bind new descriptor sets if they need to change. - // If descriptor set allocation failed, skip the draw call and bail. No need to emit an error - // message since the validation layers already do so. - if (!mPipelineCache.bindDescriptors(cmdbuffer)) { - return; - } + // Bind "dynamic" UBOs if they need to change. + mDescriptorSetManager.dynamicBind(&commands, {}); // Finally, make the actual draw call. TODO: support subranges const uint32_t firstIndex = indexOffset; diff --git a/filament/backend/src/vulkan/VulkanDriver.h b/filament/backend/src/vulkan/VulkanDriver.h index 98ba9efc721..fca5c45c5be 100644 --- a/filament/backend/src/vulkan/VulkanDriver.h +++ b/filament/backend/src/vulkan/VulkanDriver.h @@ -28,6 +28,8 @@ #include "VulkanSamplerCache.h" #include "VulkanStagePool.h" #include "VulkanUtility.h" +#include "caching/VulkanDescriptorSetManager.h" +#include "caching/VulkanPipelineLayoutCache.h" #include "DriverBase.h" #include "private/backend/Driver.h" @@ -70,6 +72,8 @@ class VulkanDriver final : public DriverBase { #endif // FVK_ENABLED(FVK_DEBUG_DEBUG_UTILS) private: + static constexpr uint8_t MAX_SAMPLER_BINDING_COUNT = Program::SAMPLER_BINDING_COUNT; + void debugCommandBegin(CommandStream* cmds, bool synchronous, const char* methodName) noexcept override; @@ -106,7 +110,9 @@ class VulkanDriver final : public DriverBase { VulkanPlatform* mPlatform = nullptr; std::unique_ptr mTimestamps; + // Placeholder resources VulkanTexture* mEmptyTexture; + VulkanBufferObject* mEmptyBufferObject; VulkanSwapChain* mCurrentSwapChain = nullptr; VulkanRenderTarget* mDefaultRenderTarget = nullptr; @@ -123,13 +129,17 @@ class VulkanDriver final : public DriverBase { VulkanThreadSafeResourceManager mThreadSafeResourceManager; VulkanCommands mCommands; + VulkanPipelineLayoutCache mPipelineLayoutCache; VulkanPipelineCache mPipelineCache; VulkanStagePool mStagePool; VulkanFboCache mFramebufferCache; VulkanSamplerCache mSamplerCache; VulkanBlitter mBlitter; - VulkanSamplerGroup* mSamplerBindings[VulkanPipelineCache::SAMPLER_BINDING_COUNT] = {}; + VulkanSamplerGroup* mSamplerBindings[MAX_SAMPLER_BINDING_COUNT] = {}; VulkanReadPixels mReadPixels; + VulkanDescriptorSetManager mDescriptorSetManager; + + VulkanDescriptorSetManager::GetPipelineLayoutFunction mGetPipelineFunction; bool const mIsSRGBSwapChainSupported; }; diff --git a/filament/backend/src/vulkan/VulkanHandles.cpp b/filament/backend/src/vulkan/VulkanHandles.cpp index 41f02212c88..9d615ce5151 100644 --- a/filament/backend/src/vulkan/VulkanHandles.cpp +++ b/filament/backend/src/vulkan/VulkanHandles.cpp @@ -120,8 +120,30 @@ void addDescriptors(Bitmask mask, } } +inline VkDescriptorSetLayout createDescriptorSetLayout(VkDevice device, + VkDescriptorSetLayoutCreateInfo const& info) { + VkDescriptorSetLayout layout; + vkCreateDescriptorSetLayout(device, &info, VKALLOC, &layout); + return layout; +} + } // anonymous namespace + +VulkanDescriptorSetLayout::VulkanDescriptorSetLayout(VkDevice device, VkDescriptorSetLayoutCreateInfo const& info, + Bitmask const& bitmask) + : VulkanResource(VulkanResourceType::DESCRIPTOR_SET_LAYOUT), + mDevice(device), + vklayout(createDescriptorSetLayout(device, info)), + bitmask(bitmask), + bindings(getBindings(bitmask)), + count(Count::fromLayoutBitmask(bitmask)) { +} + +VulkanDescriptorSetLayout::~VulkanDescriptorSetLayout() { + vkDestroyDescriptorSetLayout(mDevice, vklayout, VKALLOC); +} + VulkanProgram::VulkanProgram(VkDevice device, Program const& builder) noexcept : HwProgram(builder.getName()), VulkanResource(VulkanResourceType::PROGRAM), diff --git a/filament/backend/src/vulkan/VulkanHandles.h b/filament/backend/src/vulkan/VulkanHandles.h index e41112c125d..8abdccfd968 100644 --- a/filament/backend/src/vulkan/VulkanHandles.h +++ b/filament/backend/src/vulkan/VulkanHandles.h @@ -99,13 +99,12 @@ struct VulkanDescriptorSetLayout : public VulkanResource { static_assert(sizeof(Bitmask) % 8 == 0); - explicit VulkanDescriptorSetLayout(VkDescriptorSetLayout layout, Bitmask const& bitmask) - : VulkanResource(VulkanResourceType::DESCRIPTOR_SET_LAYOUT), - vklayout(layout), - bitmask(bitmask), - bindings(getBindings(bitmask)), - count(Count::fromLayoutBitmask(bitmask)) {} + explicit VulkanDescriptorSetLayout(VkDevice device, VkDescriptorSetLayoutCreateInfo const& info, + Bitmask const& bitmask); + ~VulkanDescriptorSetLayout(); + + VkDevice const mDevice; VkDescriptorSetLayout const vklayout; Bitmask const bitmask; diff --git a/filament/backend/src/vulkan/VulkanPipelineCache.cpp b/filament/backend/src/vulkan/VulkanPipelineCache.cpp index 581d6e9beef..7cfc74162e0 100644 --- a/filament/backend/src/vulkan/VulkanPipelineCache.cpp +++ b/filament/backend/src/vulkan/VulkanPipelineCache.cpp @@ -14,8 +14,9 @@ * limitations under the License. */ -#include "vulkan/VulkanMemory.h" -#include "vulkan/VulkanPipelineCache.h" +#include "VulkanPipelineCache.h" +#include "VulkanMemory.h" +#include "caching/VulkanDescriptorSetManager.h" #include #include @@ -34,46 +35,9 @@ using namespace bluevk; namespace filament::backend { -static VkShaderStageFlags getShaderStageFlags(UsageFlags key, uint16_t binding) { - // NOTE: if you modify this function, you also need to modify getUsageFlags. - assert_invariant(binding < MAX_SAMPLER_COUNT); - VkShaderStageFlags flags = 0; - if (key.test(binding)) { - flags |= VK_SHADER_STAGE_VERTEX_BIT; - } - if (key.test(MAX_SAMPLER_COUNT + binding)) { - flags |= VK_SHADER_STAGE_FRAGMENT_BIT; - } - return flags; -} - -UsageFlags VulkanPipelineCache::disableUsageFlags(uint16_t binding, UsageFlags src) { - src.unset(binding); - src.unset(MAX_SAMPLER_COUNT + binding); - return src; -} - -VulkanPipelineCache::VulkanPipelineCache(VulkanResourceAllocator* allocator) - : mResourceAllocator(allocator), - mPipelineBoundResources(allocator) { - mDummyBufferWriteInfo.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; - mDummyBufferWriteInfo.pNext = nullptr; - mDummyBufferWriteInfo.dstArrayElement = 0; - mDummyBufferWriteInfo.descriptorCount = 1; - mDummyBufferWriteInfo.descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER; - mDummyBufferWriteInfo.pImageInfo = nullptr; - mDummyBufferWriteInfo.pBufferInfo = &mDummyBufferInfo; - mDummyBufferWriteInfo.pTexelBufferView = nullptr; - - mDummyTargetInfo.imageLayout = VulkanImageUtility::getVkLayout(VulkanLayout::READ_ONLY); - mDummyTargetWriteInfo.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; - mDummyTargetWriteInfo.pNext = nullptr; - mDummyTargetWriteInfo.dstArrayElement = 0; - mDummyTargetWriteInfo.descriptorCount = 1; - mDummyTargetWriteInfo.descriptorType = VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT; - mDummyTargetWriteInfo.pImageInfo = &mDummyTargetInfo; - mDummyTargetWriteInfo.pBufferInfo = nullptr; - mDummyTargetWriteInfo.pTexelBufferView = nullptr; +VulkanPipelineCache::VulkanPipelineCache(VkDevice device, VmaAllocator allocator) + : mDevice(device), + mAllocator(allocator) { } VulkanPipelineCache::~VulkanPipelineCache() { @@ -81,244 +45,47 @@ VulkanPipelineCache::~VulkanPipelineCache() { // be explicit about teardown order of various components. } -void VulkanPipelineCache::setDevice(VkDevice device, VmaAllocator allocator) { - assert_invariant(mDevice == VK_NULL_HANDLE); - mDevice = device; - mAllocator = allocator; - mDescriptorPool = createDescriptorPool(mDescriptorPoolSize); - - // Formulate some dummy objects and dummy descriptor info used only for clearing out unused - // bindings. This is especially crucial after a texture has been destroyed. Since core Vulkan - // does not allow specifying VK_NULL_HANDLE without the robustness2 extension, we would need to - // change the pipeline layout more frequently if we wanted to get rid of these dummy objects. - - VkBufferCreateInfo bufferInfo { - .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, - .size = 16, - .usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, - }; - VmaAllocationCreateInfo allocInfo { .usage = VMA_MEMORY_USAGE_GPU_ONLY }; - vmaCreateBuffer(mAllocator, &bufferInfo, &allocInfo, &mDummyBuffer, &mDummyMemory, nullptr); - - mDummyBufferInfo.buffer = mDummyBuffer; - mDummyBufferInfo.range = bufferInfo.size; +void VulkanPipelineCache::bindLayout(VkPipelineLayout layout) noexcept { + mPipelineRequirements.layout = layout; } -bool VulkanPipelineCache::bindDescriptors(VkCommandBuffer cmdbuffer) noexcept { - DescriptorMap::iterator descriptorIter = mDescriptorSets.find(mDescriptorRequirements); - - // Check if the required descriptors are already bound. If so, there's no need to do anything. - if (DescEqual equals; UTILS_LIKELY(equals(mBoundDescriptor, mDescriptorRequirements))) { - - // If the pipeline state during an app's first draw call happens to match the default state - // vector of the cache, then the cache is uninitialized and we should not return early. - if (UTILS_LIKELY(!mDescriptorSets.empty())) { - - // Since the descriptors are already bound, they should be found in the cache. - assert_invariant(descriptorIter != mDescriptorSets.end()); - - // Update the LRU "time stamp" (really a count of cmd buf submissions) before returning. - descriptorIter.value().lastUsed = mCurrentTime; - return true; - } - } - +VulkanPipelineCache::PipelineCacheEntry* VulkanPipelineCache::getOrCreatePipeline() noexcept { // If a cached object exists, re-use it, otherwise create a new one. - DescriptorCacheEntry* cacheEntry = UTILS_LIKELY(descriptorIter != mDescriptorSets.end()) ? - &descriptorIter.value() : createDescriptorSets(); - - // If a descriptor set overflow occurred, allow higher levels to handle it gracefully. - assert_invariant(cacheEntry != nullptr); - if (UTILS_UNLIKELY(cacheEntry == nullptr)) { - return false; - } - - cacheEntry->lastUsed = mCurrentTime; - mBoundDescriptor = mDescriptorRequirements; - // This passes the currently "bound" uniform buffer objects to pipeline that will be used in the - // draw call. - auto resourceEntry = mDescriptorResources.find(cacheEntry->id); - if (resourceEntry == mDescriptorResources.end()) { - mDescriptorResources[cacheEntry->id] - = std::make_unique(mResourceAllocator); - resourceEntry = mDescriptorResources.find(cacheEntry->id); - } - resourceEntry->second->acquireAll(&mPipelineBoundResources); - - vkCmdBindDescriptorSets(cmdbuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, - getOrCreatePipelineLayout()->handle, 0, VulkanPipelineCache::DESCRIPTOR_TYPE_COUNT, - cacheEntry->handles.data(), 0, nullptr); - - return true; + if (PipelineMap::iterator pipelineIter = mPipelines.find(mPipelineRequirements); + pipelineIter != mPipelines.end()) { + auto& pipeline = pipelineIter.value(); + pipeline.lastUsed = mCurrentTime; + return &pipeline; + } + auto ret = createPipeline(); + ret->lastUsed = mCurrentTime; + return ret; } -bool VulkanPipelineCache::bindPipeline(VulkanCommandBuffer* commands) noexcept { +void VulkanPipelineCache::bindPipeline(VulkanCommandBuffer* commands) { VkCommandBuffer const cmdbuffer = commands->buffer(); - PipelineMap::iterator pipelineIter = mPipelines.find(mPipelineRequirements); - + PipelineCacheEntry* cacheEntry = getOrCreatePipeline(); // Check if the required pipeline is already bound. - if (PipelineEqual equals; UTILS_LIKELY(equals(mBoundPipeline, mPipelineRequirements))) { - assert_invariant(pipelineIter != mPipelines.end()); - pipelineIter.value().lastUsed = mCurrentTime; - return true; + if (cacheEntry->handle == commands->pipeline()) { + return; } - // If a cached object exists, re-use it, otherwise create a new one. - PipelineCacheEntry* cacheEntry = UTILS_LIKELY(pipelineIter != mPipelines.end()) ? - &pipelineIter.value() : createPipeline(); - // If an error occurred, allow higher levels to handle it gracefully. - assert_invariant(cacheEntry != nullptr); - if (UTILS_UNLIKELY(cacheEntry == nullptr)) { - return false; - } - - cacheEntry->lastUsed = mCurrentTime; - getOrCreatePipelineLayout()->lastUsed = mCurrentTime; + assert_invariant(cacheEntry != nullptr && "Failed to create/find pipeline"); mBoundPipeline = mPipelineRequirements; - vkCmdBindPipeline(cmdbuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, cacheEntry->handle); - return true; + commands->setPipeline(cacheEntry->handle); } void VulkanPipelineCache::bindScissor(VkCommandBuffer cmdbuffer, VkRect2D scissor) noexcept { - if (UTILS_UNLIKELY(!equivalent(mCurrentScissor, scissor))) { - mCurrentScissor = scissor; - vkCmdSetScissor(cmdbuffer, 0, 1, &scissor); - } -} - -VulkanPipelineCache::DescriptorCacheEntry* VulkanPipelineCache::createDescriptorSets() noexcept { - PipelineLayoutCacheEntry* layoutCacheEntry = getOrCreatePipelineLayout(); - - DescriptorCacheEntry descriptorCacheEntry = { - .pipelineLayout = mPipelineRequirements.layout, - .id = mDescriptorCacheEntryCount++, - }; - - // Each of the arenas for this particular layout are guaranteed to have the same size. Check - // the first arena to see if any descriptor sets are available that can be re-claimed. If not, - // create brand new ones (one for each type). They will be added to the arena later, after they - // are no longer used. This occurs during the cleanup phase during command buffer submission. - auto& descriptorSetArenas = layoutCacheEntry->descriptorSetArenas; - if (descriptorSetArenas[0].empty()) { - - // If allocating a new descriptor set from the pool would cause it to overflow, then - // recreate the pool. The number of descriptor sets that have already been allocated from - // the pool is the sum of the "active" descriptor sets (mDescriptorSets) and the "dormant" - // descriptor sets (mDescriptorArenasCount). - // - // NOTE: technically both sides of the inequality below should be multiplied by - // DESCRIPTOR_TYPE_COUNT to get the true number of descriptor sets. - if (mDescriptorSets.size() + mDescriptorArenasCount + 1 > mDescriptorPoolSize) { - growDescriptorPool(); - } - - VkDescriptorSetAllocateInfo allocInfo = {}; - allocInfo.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO; - allocInfo.descriptorPool = mDescriptorPool; - allocInfo.descriptorSetCount = DESCRIPTOR_TYPE_COUNT; - allocInfo.pSetLayouts = layoutCacheEntry->descriptorSetLayouts.data(); - VkResult error = vkAllocateDescriptorSets(mDevice, &allocInfo, - descriptorCacheEntry.handles.data()); - assert_invariant(error == VK_SUCCESS); - if (error != VK_SUCCESS) { - return nullptr; - } - } else { - for (uint32_t i = 0; i < DESCRIPTOR_TYPE_COUNT; ++i) { - descriptorCacheEntry.handles[i] = descriptorSetArenas[i].back(); - descriptorSetArenas[i].pop_back(); - } - assert_invariant(mDescriptorArenasCount > 0); - mDescriptorArenasCount--; - } - - // Rewrite every binding in the new descriptor sets. - VkDescriptorBufferInfo descriptorBuffers[UBUFFER_BINDING_COUNT]; - VkDescriptorImageInfo descriptorSamplers[SAMPLER_BINDING_COUNT]; - VkDescriptorImageInfo descriptorInputAttachments[INPUT_ATTACHMENT_COUNT]; - VkWriteDescriptorSet descriptorWrites[UBUFFER_BINDING_COUNT + SAMPLER_BINDING_COUNT + - INPUT_ATTACHMENT_COUNT]; - uint32_t nwrites = 0; - VkWriteDescriptorSet* writes = descriptorWrites; - nwrites = 0; - for (uint32_t binding = 0; binding < UBUFFER_BINDING_COUNT; binding++) { - VkWriteDescriptorSet& writeInfo = writes[nwrites++]; - if (mDescriptorRequirements.uniformBuffers[binding]) { - VkDescriptorBufferInfo& bufferInfo = descriptorBuffers[binding]; - bufferInfo.buffer = mDescriptorRequirements.uniformBuffers[binding]; - bufferInfo.offset = mDescriptorRequirements.uniformBufferOffsets[binding]; - bufferInfo.range = mDescriptorRequirements.uniformBufferSizes[binding]; - - // We store size with 32 bits, so our "WHOLE" sentinel is different from Vk. - if (bufferInfo.range == WHOLE_SIZE) { - bufferInfo.range = VK_WHOLE_SIZE; - } - - writeInfo.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; - writeInfo.pNext = nullptr; - writeInfo.dstArrayElement = 0; - writeInfo.descriptorCount = 1; - writeInfo.descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER; - writeInfo.pImageInfo = nullptr; - writeInfo.pBufferInfo = &bufferInfo; - writeInfo.pTexelBufferView = nullptr; - } else { - writeInfo = mDummyBufferWriteInfo; - assert_invariant(mDummyBufferWriteInfo.pBufferInfo->buffer); - } - assert_invariant(writeInfo.pBufferInfo->buffer); - writeInfo.dstSet = descriptorCacheEntry.handles[0]; - writeInfo.dstBinding = binding; - } - for (uint32_t binding = 0; binding < SAMPLER_BINDING_COUNT; binding++) { - if (mDescriptorRequirements.samplers[binding].sampler) { - VkWriteDescriptorSet& writeInfo = writes[nwrites++]; - VkDescriptorImageInfo& imageInfo = descriptorSamplers[binding]; - imageInfo = mDescriptorRequirements.samplers[binding]; - writeInfo.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; - writeInfo.pNext = nullptr; - writeInfo.dstArrayElement = 0; - writeInfo.descriptorCount = 1; - writeInfo.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; - writeInfo.pImageInfo = &imageInfo; - writeInfo.pBufferInfo = nullptr; - writeInfo.pTexelBufferView = nullptr; - writeInfo.dstSet = descriptorCacheEntry.handles[1]; - writeInfo.dstBinding = binding; - } - } - for (uint32_t binding = 0; binding < INPUT_ATTACHMENT_COUNT; binding++) { - if (mDescriptorRequirements.inputAttachments[binding].imageView) { - VkWriteDescriptorSet& writeInfo = writes[nwrites++]; - VkDescriptorImageInfo& imageInfo = descriptorInputAttachments[binding]; - imageInfo = mDescriptorRequirements.inputAttachments[binding]; - writeInfo.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; - writeInfo.pNext = nullptr; - writeInfo.dstArrayElement = 0; - writeInfo.descriptorCount = 1; - writeInfo.descriptorType = VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT; - writeInfo.pImageInfo = &imageInfo; - writeInfo.pBufferInfo = nullptr; - writeInfo.pTexelBufferView = nullptr; - writeInfo.dstSet = descriptorCacheEntry.handles[2]; - writeInfo.dstBinding = binding; - } - } - - vkUpdateDescriptorSets(mDevice, nwrites, writes, 0, nullptr); - - return &mDescriptorSets.emplace(mDescriptorRequirements, descriptorCacheEntry).first.value(); + vkCmdSetScissor(cmdbuffer, 0, 1, &scissor); } VulkanPipelineCache::PipelineCacheEntry* VulkanPipelineCache::createPipeline() noexcept { assert_invariant(mPipelineRequirements.shaders[0] && "Vertex shader is not bound."); - - PipelineLayoutCacheEntry* layout = getOrCreatePipelineLayout(); - assert_invariant(layout); + assert_invariant(mPipelineRequirements.layout && "No pipeline layout specified"); VkPipelineShaderStageCreateInfo shaderStages[SHADER_MODULE_COUNT]; shaderStages[0] = VkPipelineShaderStageCreateInfo{}; @@ -387,7 +154,7 @@ VulkanPipelineCache::PipelineCacheEntry* VulkanPipelineCache::createPipeline() n VkGraphicsPipelineCreateInfo pipelineCreateInfo = {}; pipelineCreateInfo.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO; - pipelineCreateInfo.layout = layout->handle; + pipelineCreateInfo.layout = mPipelineRequirements.layout; pipelineCreateInfo.renderPass = mPipelineRequirements.renderPass; pipelineCreateInfo.subpass = mPipelineRequirements.subpassIndex; pipelineCreateInfo.stageCount = hasFragmentShader ? SHADER_MODULE_COUNT : 1; @@ -481,68 +248,6 @@ VulkanPipelineCache::PipelineCacheEntry* VulkanPipelineCache::createPipeline() n return &mPipelines.emplace(mPipelineRequirements, cacheEntry).first.value(); } -VulkanPipelineCache::PipelineLayoutCacheEntry* VulkanPipelineCache::getOrCreatePipelineLayout() noexcept { - auto iter = mPipelineLayouts.find(mPipelineRequirements.layout); - if (UTILS_LIKELY(iter != mPipelineLayouts.end())) { - return &iter.value(); - } - - PipelineLayoutCacheEntry cacheEntry = {}; - - VkDescriptorSetLayoutBinding binding = {}; - binding.descriptorCount = 1; // NOTE: We never use arrays-of-blocks. - binding.stageFlags = VK_SHADER_STAGE_ALL_GRAPHICS; // NOTE: This is potentially non-optimal. - - // First create the descriptor set layout for UBO's. - VkDescriptorSetLayoutBinding ubindings[UBUFFER_BINDING_COUNT]; - binding.descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER; - for (uint32_t i = 0; i < UBUFFER_BINDING_COUNT; i++) { - binding.binding = i; - ubindings[i] = binding; - } - VkDescriptorSetLayoutCreateInfo dlinfo = {}; - dlinfo.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO; - dlinfo.bindingCount = UBUFFER_BINDING_COUNT; - dlinfo.pBindings = ubindings; - vkCreateDescriptorSetLayout(mDevice, &dlinfo, VKALLOC, &cacheEntry.descriptorSetLayouts[0]); - - // Next create the descriptor set layout for samplers. - VkDescriptorSetLayoutBinding sbindings[SAMPLER_BINDING_COUNT]; - binding.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; - for (uint32_t i = 0; i < SAMPLER_BINDING_COUNT; i++) { - binding.stageFlags = getShaderStageFlags(mPipelineRequirements.layout, i); - binding.binding = i; - sbindings[i] = binding; - } - dlinfo.bindingCount = SAMPLER_BINDING_COUNT; - dlinfo.pBindings = sbindings; - vkCreateDescriptorSetLayout(mDevice, &dlinfo, VKALLOC, &cacheEntry.descriptorSetLayouts[1]); - - // Next create the descriptor set layout for input attachments. - VkDescriptorSetLayoutBinding tbindings[INPUT_ATTACHMENT_COUNT]; - binding.descriptorType = VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT; - binding.stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT; - for (uint32_t i = 0; i < INPUT_ATTACHMENT_COUNT; i++) { - binding.binding = i; - tbindings[i] = binding; - } - dlinfo.bindingCount = INPUT_ATTACHMENT_COUNT; - dlinfo.pBindings = tbindings; - vkCreateDescriptorSetLayout(mDevice, &dlinfo, VKALLOC, &cacheEntry.descriptorSetLayouts[2]); - - // Create VkPipelineLayout based on how to resources are bounded. - VkPipelineLayoutCreateInfo pPipelineLayoutCreateInfo = {}; - pPipelineLayoutCreateInfo.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; - pPipelineLayoutCreateInfo.setLayoutCount = cacheEntry.descriptorSetLayouts.size(); - pPipelineLayoutCreateInfo.pSetLayouts = cacheEntry.descriptorSetLayouts.data(); - VkResult result = vkCreatePipelineLayout(mDevice, &pPipelineLayoutCreateInfo, VKALLOC, - &cacheEntry.handle); - if (UTILS_UNLIKELY(result != VK_SUCCESS)) { - return nullptr; - } - return &mPipelineLayouts.emplace(mPipelineRequirements.layout, cacheEntry).first.value(); -} - void VulkanPipelineCache::bindProgram(VulkanProgram* program) noexcept { mPipelineRequirements.shaders[0] = program->getVertexShader(); mPipelineRequirements.shaders[1] = program->getFragmentShader(); @@ -583,97 +288,15 @@ void VulkanPipelineCache::bindVertexArray(VkVertexInputAttributeDescription cons } } -VulkanPipelineCache::UniformBufferBinding VulkanPipelineCache::getUniformBufferBinding( - uint32_t bindingIndex) const noexcept { - auto& key = mDescriptorRequirements; - return { - key.uniformBuffers[bindingIndex], - key.uniformBufferOffsets[bindingIndex], - key.uniformBufferSizes[bindingIndex], - }; -} - -void VulkanPipelineCache::unbindUniformBuffer(VkBuffer uniformBuffer) noexcept { - auto& key = mDescriptorRequirements; - for (uint32_t bindingIndex = 0u; bindingIndex < UBUFFER_BINDING_COUNT; ++bindingIndex) { - if (key.uniformBuffers[bindingIndex] == uniformBuffer) { - key.uniformBuffers[bindingIndex] = {}; - key.uniformBufferSizes[bindingIndex] = {}; - key.uniformBufferOffsets[bindingIndex] = {}; - } - } -} - -void VulkanPipelineCache::unbindImageView(VkImageView imageView) noexcept { - for (auto& sampler : mDescriptorRequirements.samplers) { - if (sampler.imageView == imageView) { - sampler = {}; - } - } - for (auto& target : mDescriptorRequirements.inputAttachments) { - if (target.imageView == imageView) { - target = {}; - } - } -} - -void VulkanPipelineCache::bindUniformBufferObject(uint32_t bindingIndex, - VulkanBufferObject* bufferObject, VkDeviceSize offset, VkDeviceSize size) noexcept { - VkBuffer buffer = bufferObject->buffer.getGpuBuffer(); - - ASSERT_POSTCONDITION(bindingIndex < UBUFFER_BINDING_COUNT, - "Uniform bindings overflow: index = %d, capacity = %d.", bindingIndex, - UBUFFER_BINDING_COUNT); - auto& key = mDescriptorRequirements; - key.uniformBuffers[bindingIndex] = buffer; - - if (size == VK_WHOLE_SIZE) { - size = WHOLE_SIZE; - } - - assert_invariant(offset <= 0xffffffffu); - assert_invariant(size <= 0xffffffffu); - - key.uniformBufferOffsets[bindingIndex] = offset; - key.uniformBufferSizes[bindingIndex] = size; - - mPipelineBoundResources.acquire(bufferObject); -} - -void VulkanPipelineCache::bindSamplers(VkDescriptorImageInfo samplers[SAMPLER_BINDING_COUNT], - VulkanTexture* textures[SAMPLER_BINDING_COUNT], UsageFlags flags) noexcept { - for (uint32_t bindingIndex = 0; bindingIndex < SAMPLER_BINDING_COUNT; bindingIndex++) { - mDescriptorRequirements.samplers[bindingIndex] = samplers[bindingIndex]; - if (textures[bindingIndex]) { - mPipelineBoundResources.acquire(textures[bindingIndex]); - } - } - mPipelineRequirements.layout = flags; -} - -void VulkanPipelineCache::bindInputAttachment(uint32_t bindingIndex, - VkDescriptorImageInfo targetInfo) noexcept { - ASSERT_POSTCONDITION(bindingIndex < INPUT_ATTACHMENT_COUNT, - "Input attachment bindings overflow: index = %d, capacity = %d.", - bindingIndex, INPUT_ATTACHMENT_COUNT); - mDescriptorRequirements.inputAttachments[bindingIndex] = targetInfo; -} - void VulkanPipelineCache::terminate() noexcept { - // Symmetric to createLayoutsAndDescriptors. - destroyLayoutsAndDescriptors(); for (auto& iter : mPipelines) { vkDestroyPipeline(mDevice, iter.second.handle, VKALLOC); } - mPipelineBoundResources.clear(); mPipelines.clear(); mBoundPipeline = {}; - vmaDestroyBuffer(mAllocator, mDummyBuffer, mDummyMemory); - mDummyBuffer = VK_NULL_HANDLE; - mDummyMemory = VK_NULL_HANDLE; } -void VulkanPipelineCache::onCommandBuffer(const VulkanCommandBuffer& commands) { +void VulkanPipelineCache::gc() noexcept { // The timestamp associated with a given cache entry represents "time" as a count of flush // events since the cache was constructed. If any cache entry was most recently used over // FVK_MAX_PIPELINE_AGE flush events in the past, then we can be sure that it is no longer @@ -683,194 +306,22 @@ void VulkanPipelineCache::onCommandBuffer(const VulkanCommandBuffer& commands) { // The Vulkan spec says: "When a command buffer begins recording, all state in that command // buffer is undefined." Therefore, we need to clear all bindings at this time. mBoundPipeline = {}; - mBoundDescriptor = {}; mCurrentScissor = {}; // NOTE: Due to robin_map restrictions, we cannot use auto or range-based loops. - // Check if any bundles in the cache are no longer in use by any command buffer. Descriptors - // from unused bundles are moved back to their respective arenas. - using ConstDescIterator = decltype(mDescriptorSets)::const_iterator; - for (ConstDescIterator iter = mDescriptorSets.begin(); iter != mDescriptorSets.end();) { - const DescriptorCacheEntry& cacheEntry = iter.value(); - if (cacheEntry.lastUsed + FVK_MAX_PIPELINE_AGE < mCurrentTime) { - auto& arenas = mPipelineLayouts[cacheEntry.pipelineLayout].descriptorSetArenas; - for (uint32_t i = 0; i < DESCRIPTOR_TYPE_COUNT; ++i) { - arenas[i].push_back(cacheEntry.handles[i]); - } - ++mDescriptorArenasCount; - mDescriptorResources.erase(cacheEntry.id); - iter = mDescriptorSets.erase(iter); - } else { - ++iter; - } - } - // Evict any pipelines that have not been used in a while. // Any pipeline older than FVK_MAX_COMMAND_BUFFERS can be safely destroyed. - using ConstPipeIterator = decltype(mPipelines)::const_iterator; - for (ConstPipeIterator iter = mPipelines.begin(); iter != mPipelines.end();) { - const PipelineCacheEntry& cacheEntry = iter.value(); - if (cacheEntry.lastUsed + FVK_MAX_PIPELINE_AGE < mCurrentTime) { - vkDestroyPipeline(mDevice, iter->second.handle, VKALLOC); - iter = mPipelines.erase(iter); - } else { - ++iter; - } - } - - // Evict any layouts that have not been used in a while. - using ConstLayoutIterator = decltype(mPipelineLayouts)::const_iterator; - for (ConstLayoutIterator iter = mPipelineLayouts.begin(); iter != mPipelineLayouts.end();) { - const PipelineLayoutCacheEntry& cacheEntry = iter.value(); - if (cacheEntry.lastUsed + FVK_MAX_PIPELINE_AGE < mCurrentTime) { - vkDestroyPipelineLayout(mDevice, iter->second.handle, VKALLOC); - for (auto setLayout : iter->second.descriptorSetLayouts) { - #if FVK_ENABLED(FVK_DEBUG_PIPELINE_CACHE) - PipelineLayoutKey key = iter.key(); - for (auto& pair : mDescriptorSets) { - assert_invariant(pair.second.pipelineLayout != key); - } - #endif - vkDestroyDescriptorSetLayout(mDevice, setLayout, VKALLOC); - } - auto& arenas = iter->second.descriptorSetArenas; - assert_invariant(mDescriptorArenasCount >= arenas[0].size()); - mDescriptorArenasCount -= arenas[0].size(); - for (auto& arena : arenas) { - vkFreeDescriptorSets(mDevice, mDescriptorPool, arena.size(), arena.data()); - } - iter = mPipelineLayouts.erase(iter); - } else { - ++iter; - } - } - - // If there are no descriptors from any extinct pool that are still in use, we can safely - // destroy the extinct pools, which implicitly frees their associated descriptor sets. - bool canPurgeExtinctPools = true; - for (auto& bundle : mExtinctDescriptorBundles) { - if (bundle.lastUsed + FVK_MAX_PIPELINE_AGE >= mCurrentTime) { - canPurgeExtinctPools = false; - break; - } - } - if (canPurgeExtinctPools) { - for (VkDescriptorPool pool : mExtinctDescriptorPools) { - vkDestroyDescriptorPool(mDevice, pool, VKALLOC); - } - mExtinctDescriptorPools.clear(); - - for (auto const& entry : mExtinctDescriptorBundles) { - mDescriptorResources.erase(entry.id); - } - mExtinctDescriptorBundles.clear(); - } -} - -VkDescriptorPool VulkanPipelineCache::createDescriptorPool(uint32_t size) const { - VkDescriptorPoolSize poolSizes[DESCRIPTOR_TYPE_COUNT] = {}; - VkDescriptorPoolCreateInfo poolInfo { - .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO, - .pNext = nullptr, - .flags = VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT, - .maxSets = size * DESCRIPTOR_TYPE_COUNT, - .poolSizeCount = DESCRIPTOR_TYPE_COUNT, - .pPoolSizes = poolSizes - }; - poolSizes[0].type = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER; - poolSizes[0].descriptorCount = poolInfo.maxSets * UBUFFER_BINDING_COUNT; - poolSizes[1].type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; - poolSizes[1].descriptorCount = poolInfo.maxSets * SAMPLER_BINDING_COUNT; - poolSizes[2].type = VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT; - poolSizes[2].descriptorCount = poolInfo.maxSets * INPUT_ATTACHMENT_COUNT; - - VkDescriptorPool pool; - const UTILS_UNUSED VkResult result = vkCreateDescriptorPool(mDevice, &poolInfo, VKALLOC, &pool); - assert_invariant(result == VK_SUCCESS); - return pool; -} - -void VulkanPipelineCache::destroyLayoutsAndDescriptors() noexcept { - // Our current descriptor set strategy can cause the # of descriptor sets to explode in certain - // situations, so it's interesting to report the number that get stuffed into the cache. - #if FVK_ENABLED(FVK_DEBUG_PIPELINE_CACHE) - utils::slog.d << "Destroying " << mDescriptorSets.size() << " bundles of descriptor sets." - << utils::io::endl; - #endif - - mDescriptorSets.clear(); - - // Our current layout bundle strategy can cause the # of layout bundles to explode in certain - // situations, so it's interesting to report the number that get stuffed into the cache. - #if FVK_ENABLED(FVK_DEBUG_PIPELINE_CACHE) - utils::slog.d << "Destroying " << mPipelineLayouts.size() << " pipeline layouts." - << utils::io::endl; - #endif - - for (auto& iter : mPipelineLayouts) { - vkDestroyPipelineLayout(mDevice, iter.second.handle, VKALLOC); - for (auto setLayout : iter.second.descriptorSetLayouts) { - vkDestroyDescriptorSetLayout(mDevice, setLayout, VKALLOC); - } - // There is no need to free descriptor sets individually since destroying the VkDescriptorPool - // implicitly frees them. - } - mPipelineLayouts.clear(); - vkDestroyDescriptorPool(mDevice, mDescriptorPool, VKALLOC); - mDescriptorPool = VK_NULL_HANDLE; - - for (VkDescriptorPool pool : mExtinctDescriptorPools) { - vkDestroyDescriptorPool(mDevice, pool, VKALLOC); - } - mExtinctDescriptorPools.clear(); - mExtinctDescriptorBundles.clear(); - - // Both mDescriptorSets and mExtinctDescriptorBundles have been cleared, so it's safe to call - // clear() on mDescriptorResources. - mDescriptorResources.clear(); - - mBoundDescriptor = {}; -} - -void VulkanPipelineCache::growDescriptorPool() noexcept { - // We need to destroy the old VkDescriptorPool, but we can't do so immediately because many - // of its descriptors are still in use. So, stash it in an "extinct" list. - mExtinctDescriptorPools.push_back(mDescriptorPool); - - // Create the new VkDescriptorPool, twice as big as the old one. - mDescriptorPoolSize *= 2; - mDescriptorPool = createDescriptorPool(mDescriptorPoolSize); - - // Clear out all unused descriptor sets in the arena so they don't get reclaimed. There is no - // need to free them individually since the old VkDescriptorPool will be destroyed. - for (auto iter = mPipelineLayouts.begin(); iter != mPipelineLayouts.end(); ++iter) { - for (auto& arena : iter.value().descriptorSetArenas) { - arena.clear(); - } - } - mDescriptorArenasCount = 0; - - // Move all in-use descriptors from the primary cache into an "extinct" list, so that they will - // later be destroyed rather than reclaimed. - using DescIterator = decltype(mDescriptorSets)::iterator; - for (DescIterator iter = mDescriptorSets.begin(); iter != mDescriptorSets.end(); ++iter) { - mExtinctDescriptorBundles.push_back(iter.value()); - } - mDescriptorSets.clear(); -} - -size_t VulkanPipelineCache::PipelineLayoutKeyHashFn::operator()( - const PipelineLayoutKey& key) const { - std::hash hasher; - auto h0 = hasher(key.getBitsAt(0)); - auto h1 = hasher(key.getBitsAt(1)); - return h0 ^ (h1 << 1); -} - -bool VulkanPipelineCache::PipelineLayoutKeyEqual::operator()(const PipelineLayoutKey& k1, - const PipelineLayoutKey& k2) const { - return k1 == k2; + using ConstPipeIterator = decltype(mPipelines)::const_iterator; + for (ConstPipeIterator iter = mPipelines.begin(); iter != mPipelines.end();) { + const PipelineCacheEntry& cacheEntry = iter.value(); + if (cacheEntry.lastUsed + FVK_MAX_PIPELINE_AGE < mCurrentTime) { + vkDestroyPipeline(mDevice, iter->second.handle, VKALLOC); + iter = mPipelines.erase(iter); + } else { + ++iter; + } + } } bool VulkanPipelineCache::PipelineEqual::operator()(const PipelineKey& k1, @@ -878,31 +329,6 @@ bool VulkanPipelineCache::PipelineEqual::operator()(const PipelineKey& k1, return 0 == memcmp((const void*) &k1, (const void*) &k2, sizeof(k1)); } -bool VulkanPipelineCache::DescEqual::operator()(const DescriptorKey& k1, - const DescriptorKey& k2) const { - for (uint32_t i = 0; i < UBUFFER_BINDING_COUNT; i++) { - if (k1.uniformBuffers[i] != k2.uniformBuffers[i] || - k1.uniformBufferOffsets[i] != k2.uniformBufferOffsets[i] || - k1.uniformBufferSizes[i] != k2.uniformBufferSizes[i]) { - return false; - } - } - for (uint32_t i = 0; i < SAMPLER_BINDING_COUNT; i++) { - if (k1.samplers[i].sampler != k2.samplers[i].sampler || - k1.samplers[i].imageView != k2.samplers[i].imageView || - k1.samplers[i].imageLayout != k2.samplers[i].imageLayout) { - return false; - } - } - for (uint32_t i = 0; i < INPUT_ATTACHMENT_COUNT; i++) { - if (k1.inputAttachments[i].imageView != k2.inputAttachments[i].imageView || - k1.inputAttachments[i].imageLayout != k2.inputAttachments[i].imageLayout) { - return false; - } - } - return true; -} - } // namespace filament::backend #pragma clang diagnostic pop diff --git a/filament/backend/src/vulkan/VulkanPipelineCache.h b/filament/backend/src/vulkan/VulkanPipelineCache.h index 10bc845eb3d..53eaf71287c 100644 --- a/filament/backend/src/vulkan/VulkanPipelineCache.h +++ b/filament/backend/src/vulkan/VulkanPipelineCache.h @@ -17,6 +17,11 @@ #ifndef TNT_FILAMENT_BACKEND_VULKANPIPELINECACHE_H #define TNT_FILAMENT_BACKEND_VULKANPIPELINECACHE_H +#include "VulkanCommands.h" +#include "VulkanMemory.h" +#include "VulkanResources.h" +#include "VulkanUtility.h" + #include #include @@ -34,12 +39,6 @@ #include #include -#include "VulkanCommands.h" - -VK_DEFINE_HANDLE(VmaAllocator) -VK_DEFINE_HANDLE(VmaAllocation) -VK_DEFINE_HANDLE(VmaPool) - namespace filament::backend { struct VulkanProgram; @@ -56,32 +55,14 @@ class VulkanResourceAllocator; // - Assumes that viewport and scissor should be dynamic. (not baked into VkPipeline) // - Assumes that uniform buffers should be visible across all shader stages. // -class VulkanPipelineCache : public CommandBufferObserver { +class VulkanPipelineCache { public: VulkanPipelineCache(VulkanPipelineCache const&) = delete; VulkanPipelineCache& operator=(VulkanPipelineCache const&) = delete; - static constexpr uint32_t UBUFFER_BINDING_COUNT = Program::UNIFORM_BINDING_COUNT; - static constexpr uint32_t SAMPLER_BINDING_COUNT = MAX_SAMPLER_COUNT; - - // We assume only one possible input attachment between two subpasses. See also the subpasses - // definition in VulkanFboCache. - static constexpr uint32_t INPUT_ATTACHMENT_COUNT = 1; - static constexpr uint32_t SHADER_MODULE_COUNT = 2; static constexpr uint32_t VERTEX_ATTRIBUTE_COUNT = MAX_VERTEX_ATTRIBUTE_COUNT; - // Three descriptor set layouts: uniforms, combined image samplers, and input attachments. - static constexpr uint32_t DESCRIPTOR_TYPE_COUNT = 3; - static constexpr uint32_t INITIAL_DESCRIPTOR_SET_POOL_SIZE = 512; - - // The VertexArray POD is an array of buffer targets and an array of attributes that refer to - // those targets. It does not include any references to actual buffers, so you can think of it - // as a vertex assembler configuration. For simplicity it contains fixed-size arrays and does - // not store sizes; all unused entries are simply zeroed out. - struct VertexArray { - }; - // The ProgramBundle contains weak references to the compiled vertex and fragment shaders. struct ProgramBundle { VkShaderModule vertex; @@ -89,8 +70,6 @@ class VulkanPipelineCache : public CommandBufferObserver { VkSpecializationInfo* specializationInfos = nullptr; }; - static UsageFlags disableUsageFlags(uint16_t binding, UsageFlags src); - #pragma clang diagnostic push #pragma clang diagnostic warning "-Wpadded" @@ -133,17 +112,13 @@ class VulkanPipelineCache : public CommandBufferObserver { // Upon construction, the pipeCache initializes some internal state but does not make any Vulkan // calls. On destruction it will free any cached Vulkan objects that haven't already been freed. - VulkanPipelineCache(VulkanResourceAllocator* allocator); + VulkanPipelineCache(VkDevice device, VmaAllocator allocator); ~VulkanPipelineCache(); - void setDevice(VkDevice device, VmaAllocator allocator); - // Creates new descriptor sets if necessary and binds them using vkCmdBindDescriptorSets. - // Returns false if descriptor set allocation fails. - bool bindDescriptors(VkCommandBuffer cmdbuffer) noexcept; + void bindLayout(VkPipelineLayout layout) noexcept; // Creates a new pipeline if necessary and binds it using vkCmdBindPipeline. - // Returns false if an error occurred. - bool bindPipeline(VulkanCommandBuffer* commands) noexcept; + void bindPipeline(VulkanCommandBuffer* commands); // Sets up a new scissor rectangle if it has been dirtied. void bindScissor(VkCommandBuffer cmdbuffer, VkRect2D scissor) noexcept; @@ -153,42 +128,13 @@ class VulkanPipelineCache : public CommandBufferObserver { void bindRasterState(const RasterState& rasterState) noexcept; void bindRenderPass(VkRenderPass renderPass, int subpassIndex) noexcept; void bindPrimitiveTopology(VkPrimitiveTopology topology) noexcept; - void bindUniformBufferObject(uint32_t bindingIndex, VulkanBufferObject* bufferObject, - VkDeviceSize offset = 0, VkDeviceSize size = VK_WHOLE_SIZE) noexcept; - void bindSamplers(VkDescriptorImageInfo samplers[SAMPLER_BINDING_COUNT], - VulkanTexture* textures[SAMPLER_BINDING_COUNT], UsageFlags flags) noexcept; - void bindInputAttachment(uint32_t bindingIndex, VkDescriptorImageInfo imageInfo) noexcept; + void bindVertexArray(VkVertexInputAttributeDescription const* attribDesc, VkVertexInputBindingDescription const* bufferDesc, uint8_t count); - // Gets the current UBO at the given slot, useful for push / pop. - UniformBufferBinding getUniformBufferBinding(uint32_t bindingIndex) const noexcept; - - // Checks if the given uniform is bound to any slot, and if so binds "null" to that slot. - // Also invalidates all cached descriptors that refer to the given buffer. - // This is only necessary when the client knows that the UBO is about to be destroyed. - void unbindUniformBuffer(VkBuffer uniformBuffer) noexcept; - - // Checks if an image view is bound to any sampler, and if so resets that particular slot. - // Also invalidates all cached descriptors that refer to the given image view. - // This is only necessary when the client knows that a texture is about to be destroyed. - void unbindImageView(VkImageView imageView) noexcept; - - // NOTE: In theory we should proffer "unbindSampler" but in practice we never destroy samplers. - // Destroys all managed Vulkan objects. This should be called before changing the VkDevice. void terminate() noexcept; - // vkCmdBindPipeline and vkCmdBindDescriptorSets establish bindings to a specific command - // buffer; they are not global to the device. Therefore we need to be notified when a - // new command buffer becomes active. - void onCommandBuffer(const VulkanCommandBuffer& cmdbuffer) override; - - // Injects a dummy texture that can be used to clear out old descriptor sets. - void setDummyTexture(VkImageView imageView) { - mDummyTargetInfo.imageView = imageView; - } - static VkPrimitiveTopology getPrimitiveTopology(PrimitiveType pt) noexcept { switch (pt) { case PrimitiveType::POINTS: @@ -204,22 +150,9 @@ class VulkanPipelineCache : public CommandBufferObserver { } } -private: - // PIPELINE LAYOUT CACHE KEY - // ------------------------- - - using PipelineLayoutKey = utils::bitset128; - - static_assert(PipelineLayoutKey::BIT_COUNT >= 2 * MAX_SAMPLER_COUNT); - - struct PipelineLayoutKeyHashFn { - size_t operator()(const PipelineLayoutKey& key) const; - }; - - struct PipelineLayoutKeyEqual { - bool operator()(const PipelineLayoutKey& k1, const PipelineLayoutKey& k2) const; - }; + void gc() noexcept; +private: // PIPELINE CACHE KEY // ------------------ @@ -272,10 +205,10 @@ class VulkanPipelineCache : public CommandBufferObserver { VertexInputBindingDescription vertexBuffers[VERTEX_ATTRIBUTE_COUNT]; // 128 : 156 RasterState rasterState; // 16 : 284 uint32_t padding; // 4 : 300 - PipelineLayoutKey layout; // 16 : 304 + VkPipelineLayout layout; // 8 : 304 }; - static_assert(sizeof(PipelineKey) == 320, "PipelineKey must not have implicit padding."); + static_assert(sizeof(PipelineKey) == 312, "PipelineKey must not have implicit padding."); using PipelineHashFn = utils::hash::MurmurHashFn; @@ -283,52 +216,6 @@ class VulkanPipelineCache : public CommandBufferObserver { bool operator()(const PipelineKey& k1, const PipelineKey& k2) const; }; - // DESCRIPTOR SET CACHE KEY - // ------------------------ - - // Equivalent to VkDescriptorImageInfo but with explicit padding. - struct DescriptorImageInfo { - DescriptorImageInfo& operator=(const VkDescriptorImageInfo& that) { - sampler = that.sampler; - imageView = that.imageView; - imageLayout = that.imageLayout; - padding = 0; - return *this; - } - operator VkDescriptorImageInfo() const { return { sampler, imageView, imageLayout }; } - - // TODO: replace the 64-bit sampler handle with `uint32_t samplerParams` and remove the - // padding field. This is possible if we have access to the VulkanSamplerCache. - VkSampler sampler; - - VkImageView imageView; - VkImageLayout imageLayout; - uint32_t padding; - }; - - // We store size with 32 bits, so our "WHOLE" sentinel is different from Vk. - static const uint32_t WHOLE_SIZE = 0xffffffffu; - - // Represents all the Vulkan state that comprises a bound descriptor set. - struct DescriptorKey { - VkBuffer uniformBuffers[UBUFFER_BINDING_COUNT]; // 80 0 - DescriptorImageInfo samplers[SAMPLER_BINDING_COUNT]; // 1488 80 - DescriptorImageInfo inputAttachments[INPUT_ATTACHMENT_COUNT]; // 24 1568 - uint32_t uniformBufferOffsets[UBUFFER_BINDING_COUNT]; // 40 1592 - uint32_t uniformBufferSizes[UBUFFER_BINDING_COUNT]; // 40 1632 - }; - static_assert(offsetof(DescriptorKey, samplers) == 80); - static_assert(offsetof(DescriptorKey, inputAttachments) == 1568); - static_assert(offsetof(DescriptorKey, uniformBufferOffsets) == 1592); - static_assert(offsetof(DescriptorKey, uniformBufferSizes) == 1632); - static_assert(sizeof(DescriptorKey) == 1672, "DescriptorKey must not have implicit padding."); - - using DescHashFn = utils::hash::MurmurHashFn; - - struct DescEqual { - bool operator()(const DescriptorKey& k1, const DescriptorKey& k2) const; - }; - #pragma clang diagnostic pop // CACHE ENTRY STRUCTS @@ -341,16 +228,6 @@ class VulkanPipelineCache : public CommandBufferObserver { using Timestamp = uint64_t; Timestamp mCurrentTime = 0; - // The descriptor set cache entry is a group of descriptor sets that are bound simultaneously. - struct DescriptorCacheEntry { - std::array handles; - Timestamp lastUsed; - PipelineLayoutKey pipelineLayout; - uint32_t id; - }; - uint32_t mDescriptorCacheEntryCount = 0; - - struct PipelineCacheEntry { VkPipeline handle; Timestamp lastUsed; @@ -359,98 +236,36 @@ class VulkanPipelineCache : public CommandBufferObserver { struct PipelineLayoutCacheEntry { VkPipelineLayout handle; Timestamp lastUsed; - - std::array descriptorSetLayouts; - - // Each pipeline layout has 3 arenas of unused descriptors (one for each binding type). - // - // The difference between the "arenas" and the "pool" are as follows. - // - // - The "pool" is a single, centralized factory for all descriptors (VkDescriptorPool). - // - // - Each "arena" is a set of unused (but alive) descriptors that can only be used with a - // specific pipeline layout and a specific binding type. We manually manage each arena. - // The arenas are created in an empty state, and they are gradually populated as new - // descriptors are reclaimed over time. This is quite different from the pool, which is - // given a fixed size when it is constructed. - // - std::array, DESCRIPTOR_TYPE_COUNT> descriptorSetArenas; }; // CACHE CONTAINERS // ---------------- - using PipelineLayoutMap = tsl::robin_map; using PipelineMap = tsl::robin_map; - using DescriptorMap - = tsl::robin_map; - using DescriptorResourceMap - = std::unordered_map>; - PipelineLayoutMap mPipelineLayouts; +private: + + PipelineCacheEntry* getOrCreatePipeline() noexcept; + PipelineMap mPipelines; - DescriptorMap mDescriptorSets; - DescriptorResourceMap mDescriptorResources; // These helpers all return unstable pointers that should not be stored. - DescriptorCacheEntry* createDescriptorSets() noexcept; PipelineCacheEntry* createPipeline() noexcept; PipelineLayoutCacheEntry* getOrCreatePipelineLayout() noexcept; - // Misc helper methods. - void destroyLayoutsAndDescriptors() noexcept; - VkDescriptorPool createDescriptorPool(uint32_t size) const; - void growDescriptorPool() noexcept; - // Immutable state. VkDevice mDevice = VK_NULL_HANDLE; VmaAllocator mAllocator = VK_NULL_HANDLE; // Current requirements for the pipeline layout, pipeline, and descriptor sets. PipelineKey mPipelineRequirements = {}; - DescriptorKey mDescriptorRequirements = {}; // Current bindings for the pipeline and descriptor sets. PipelineKey mBoundPipeline = {}; - DescriptorKey mBoundDescriptor = {}; // Current state for scissoring. VkRect2D mCurrentScissor = {}; - - // The descriptor set pool starts out with a decent number of descriptor sets. The cache can - // grow the pool by re-creating it with a larger size. See growDescriptorPool(). - VkDescriptorPool mDescriptorPool; - - // This describes the number of descriptor sets in mDescriptorPool. Note that this needs to be - // multiplied by DESCRIPTOR_TYPE_COUNT to get the actual number of descriptor sets. Also note - // that the number of low-level "descriptors" (not descriptor *sets*) is actually much more than - // this size. It can be computed only by factoring in UBUFFER_BINDING_COUNT etc. - uint32_t mDescriptorPoolSize = INITIAL_DESCRIPTOR_SET_POOL_SIZE; - - // To get the actual number of descriptor sets that have been allocated from the pool, - // take the sum of mDescriptorArenasCount (these are inactive descriptor sets) and the - // number of entries in the mDescriptorPool map (active descriptor sets). Multiply the result by - // DESCRIPTOR_TYPE_COUNT. - uint32_t mDescriptorArenasCount = 0; - - // After a growth event (i.e. when the VkDescriptorPool is replaced with a bigger version), all - // currently used descriptors are moved into the "extinct" sets so that they can be safely - // destroyed a few frames later. - std::list mExtinctDescriptorPools; - std::list mExtinctDescriptorBundles; - - VkDescriptorBufferInfo mDummyBufferInfo = {}; - VkWriteDescriptorSet mDummyBufferWriteInfo = {}; - VkDescriptorImageInfo mDummyTargetInfo = {}; - VkWriteDescriptorSet mDummyTargetWriteInfo = {}; - - VkBuffer mDummyBuffer; - VmaAllocation mDummyMemory; - - VulkanResourceAllocator* mResourceAllocator; - VulkanAcquireOnlyResourceManager mPipelineBoundResources; }; } // namespace filament::backend diff --git a/filament/backend/src/vulkan/VulkanResourceAllocator.h b/filament/backend/src/vulkan/VulkanResourceAllocator.h index cd63b6d76ab..3afc0d6b965 100644 --- a/filament/backend/src/vulkan/VulkanResourceAllocator.h +++ b/filament/backend/src/vulkan/VulkanResourceAllocator.h @@ -51,8 +51,8 @@ namespace filament::backend { class VulkanResourceAllocator { public: using AllocatorImpl = HandleAllocatorVK; - VulkanResourceAllocator(size_t arenaSize, bool disableUseAfterFreeCheck) - : mHandleAllocatorImpl("Handles", arenaSize, disableUseAfterFreeCheck) + VulkanResourceAllocator(size_t arenaSize, bool disableUseAfterFreeCheck) + : mHandleAllocatorImpl("Handles", arenaSize, disableUseAfterFreeCheck) #if DEBUG_RESOURCE_LEAKS , mDebugOnlyResourceCount(RESOURCE_TYPE_COUNT) { std::memset(mDebugOnlyResourceCount.data(), 0, sizeof(size_t) * RESOURCE_TYPE_COUNT); diff --git a/filament/backend/src/vulkan/caching/VulkanDescriptorSetManager.cpp b/filament/backend/src/vulkan/caching/VulkanDescriptorSetManager.cpp index 77706e92d67..cad4339a1b0 100644 --- a/filament/backend/src/vulkan/caching/VulkanDescriptorSetManager.cpp +++ b/filament/backend/src/vulkan/caching/VulkanDescriptorSetManager.cpp @@ -80,7 +80,8 @@ class DescriptorPool { mCount(count), mCapacity(capacity), mSize(0), - mUnusedCount(0) { + mUnusedCount(0), + mDisableRecycling(false) { DescriptorCount const actual = mCount * capacity; VkDescriptorPoolSize sizes[4]; uint8_t npools = 0; @@ -111,7 +112,7 @@ class DescriptorPool { VkDescriptorPoolCreateInfo info{ .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO, .pNext = nullptr, - .flags = VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT, + .flags = 0, .maxSets = capacity, .poolSizeCount = npools, .pPoolSizes = sizes, @@ -123,9 +124,19 @@ class DescriptorPool { DescriptorPool& operator=(DescriptorPool const&) = delete; ~DescriptorPool() { + // Note that these have to manually destroyed because they were not explicitly ref-counted. + for (auto const& [mask, sets]: mUnused) { + for (auto set: sets) { + mAllocator->destruct(set); + } + } vkDestroyDescriptorPool(mDevice, mPool, VKALLOC); } + void disableRecycling() noexcept { + mDisableRecycling = true; + } + uint16_t const& capacity() { return mCapacity; } @@ -172,6 +183,9 @@ class DescriptorPool { Handle createSet(Bitmask const& layoutMask, VkDescriptorSet vkSet) { return mAllocator->initHandle(mAllocator, vkSet, [this, layoutMask, vkSet]() { + if (mDisableRecycling) { + return; + } // We are recycling - release the set back into the pool. Note that the // vk handle has not changed, but we need to change the backend handle to allow // for proper refcounting of resources referenced in this set. @@ -200,6 +214,8 @@ class DescriptorPool { using UnusedSetMap = std::unordered_map>, BitmaskHashFn, BitmaskEqual>; UnusedSetMap mUnused; + + bool mDisableRecycling; }; // This is an ever-expanding pool of sets where it @@ -244,6 +260,12 @@ class DescriptorInfinitePool { return ret; } + void disableRecycling() noexcept { + for (auto& pool: mPools) { + pool->disableRecycling(); + } + } + private: VkDevice mDevice; VulkanResourceAllocator* mAllocator; @@ -267,21 +289,19 @@ class LayoutCache { ~LayoutCache() { for (auto [key, layout]: mLayouts) { - auto layoutPtr = mAllocator->handle_cast(layout); - vkDestroyDescriptorSetLayout(mDevice, layoutPtr->vklayout, VKALLOC); + mAllocator->destruct(layout); } mLayouts.clear(); } void destroyLayout(Handle handle) { - auto layoutPtr = mAllocator->handle_cast(handle); for (auto [key, layout]: mLayouts) { if (layout == handle) { mLayouts.erase(key); break; } } - vkDestroyDescriptorSetLayout(mDevice, layoutPtr->vklayout, VKALLOC); + mAllocator->destruct(handle); } Handle getLayout(descset::DescriptorSetLayout const& layout) { @@ -339,10 +359,8 @@ class LayoutCache { .bindingCount = count, .pBindings = toBind, }; - - VkDescriptorSetLayout outLayout; - vkCreateDescriptorSetLayout(mDevice, &dlinfo, VKALLOC, &outLayout); - return (mLayouts[key] = mAllocator->initHandle(outLayout, key)); + return (mLayouts[key] = + mAllocator->initHandle(mDevice, dlinfo, key)); } private: @@ -416,7 +434,7 @@ struct SamplerKey { ret.sampler[count] = info.sampler; ret.imageView[count] = info.imageView; ret.imageLayout[count] = info.imageLayout; - }// else keep them as VK_NULL_HANDLEs. + } // else keep them as VK_NULL_HANDLEs. count++; } return ret; @@ -431,16 +449,16 @@ struct SamplerKey { struct InputAttachmentKey { // This count should be fixed. uint8_t count; - uint8_t padding[7]; - VkImageView view = VK_NULL_HANDLE; + uint8_t padding[3]; VkImageLayout imageLayout = VK_IMAGE_LAYOUT_UNDEFINED; + VkImageView view = VK_NULL_HANDLE; static inline InputAttachmentKey key(VkDescriptorImageInfo const& info, VulkanDescriptorSetLayout* layout) { return { .count = (uint8_t) layout->count.inputAttachment, - .view = info.imageView, .imageLayout = info.imageLayout, + .view = info.imageView, }; } @@ -627,29 +645,40 @@ class DescriptorSetCache { public: DescriptorSetCache(VkDevice device, VulkanResourceAllocator* allocator) : mAllocator(allocator), - mDescriptorPool(device, allocator), - mUBOCache(allocator), - mSamplerCache(allocator), - mInputAttachmentCache(allocator) {} + mDescriptorPool(std::make_unique(device, allocator)), + mUBOCache(std::make_unique>(allocator)), + mSamplerCache(std::make_unique>(allocator)), + mInputAttachmentCache( + std::make_unique>(allocator)) {} template inline std::pair get(Key const& key, VulkanDescriptorSetLayout* layout) { if constexpr (std::is_same_v) { - return get(key, mUBOCache, layout); + return get(key, *mUBOCache, layout); } else if constexpr (std::is_same_v) { - return get(key, mSamplerCache, layout); + return get(key, *mSamplerCache, layout); } else if constexpr (std::is_same_v) { - return get(key, mInputAttachmentCache, layout); + return get(key, *mInputAttachmentCache, layout); } PANIC_POSTCONDITION("Unexpected key type"); } + ~DescriptorSetCache() { + // This will prevent the descriptor sets recycling when we destroy descriptor set caches. + mDescriptorPool->disableRecycling(); + + mInputAttachmentCache.reset(); + mSamplerCache.reset(); + mUBOCache.reset(); + mDescriptorPool.reset(); + } + // gc() should be called at the end of everyframe void gc() { - mUBOCache.gc(); - mSamplerCache.gc(); - mInputAttachmentCache.gc(); + mUBOCache->gc(); + mSamplerCache->gc(); + mInputAttachmentCache->gc(); } private: @@ -660,16 +689,18 @@ class DescriptorSetCache { return {set, true}; } auto set = mAllocator->handle_cast( - mDescriptorPool.obtainSet(layout)); + mDescriptorPool->obtainSet(layout)); cache.put(key, set); return {set, false}; } VulkanResourceAllocator* mAllocator; - DescriptorInfinitePool mDescriptorPool; - LRUDescriptorSetCache mUBOCache; - LRUDescriptorSetCache mSamplerCache; - LRUDescriptorSetCache mInputAttachmentCache; + + // We need to heap-allocate so that the destruction can be strictly ordered. + std::unique_ptr mDescriptorPool; + std::unique_ptr> mUBOCache; + std::unique_ptr> mSamplerCache; + std::unique_ptr> mInputAttachmentCache; }; } // anonymous namespace @@ -746,25 +777,38 @@ class VulkanDescriptorSetManager::Impl { mLayoutStash[program] = layouts; } + VulkanDescriptorSetLayoutList outLayouts = layouts; DescriptorSetVkHandles vkDescSets = initDescSetHandles(); VkWriteDescriptorSet descriptorWrites[MAX_BINDINGS]; uint32_t nwrites = 0; + // Use placeholders when necessary for (uint8_t i = 0; i < VulkanDescriptorSetLayout::UNIQUE_DESCRIPTOR_SET_COUNT; ++i) { - auto handle = layouts[i]; - if (!handle) { - assert_invariant(i == INPUT_ATTACHMENT_SET_ID - && "Unexpectedly absent descriptor set layout"); - continue; - } - VulkanDescriptorSetLayout* layout - = mAllocator->handle_cast(handle); - if (!((i == UBO_SET_ID && layout->bitmask.ubo) - || (i == SAMPLER_SET_ID && layout->bitmask.sampler) - || (i == INPUT_ATTACHMENT_SET_ID && layout->bitmask.inputAttachment + if (!layouts[i]) { + if (i == INPUT_ATTACHMENT_SET_ID || + (i == SAMPLER_SET_ID && !layouts[INPUT_ATTACHMENT_SET_ID])) { + continue; + } + outLayouts[i] = getPlaceHolderLayout(i); + } else { + outLayouts[i] = layouts[i]; + auto p = mAllocator->handle_cast(layouts[i]); + if (!((i == UBO_SET_ID && p->bitmask.ubo) + || (i == SAMPLER_SET_ID && p->bitmask.sampler) + || (i == INPUT_ATTACHMENT_SET_ID && p->bitmask.inputAttachment && mInputAttachment.first.texture))) { + outLayouts[i] = getPlaceHolderLayout(i); + } + } + } + + for (uint8_t i = 0; i < VulkanDescriptorSetLayout::UNIQUE_DESCRIPTOR_SET_COUNT; ++i) { + if (!outLayouts[i]) { continue; } + VulkanDescriptorSetLayout* layout + = mAllocator->handle_cast(outLayouts[i]); + bool const usePlaceholder = layouts[i] != outLayouts[i]; auto const& [set, cached] = getSet(i, layout); VkDescriptorSet const vkSet = set->vkSet; @@ -773,7 +817,8 @@ class VulkanDescriptorSetManager::Impl { // Note that we still need to bind the set, but 'cached' means that we found a set with // the exact same content already written, and we would just bind that one instead. - if (cached) { + // We also don't need to write to the placeholder set. + if (cached || usePlaceholder) { continue; } @@ -836,7 +881,7 @@ class VulkanDescriptorSetManager::Impl { vkUpdateDescriptorSets(mDevice, nwrites, descriptorWrites, 0, nullptr); } - VkPipelineLayout const pipelineLayout = getPipelineLayoutFn(layouts); + VkPipelineLayout const pipelineLayout = getPipelineLayoutFn(outLayouts); VkCommandBuffer const cmdbuffer = commands->buffer(); BoundState state{}; @@ -918,6 +963,10 @@ class VulkanDescriptorSetManager::Impl { FVK_SYSTRACE_END(); } + void clearProgram(VulkanProgram* program) noexcept { + mLayoutStash.erase(program); + } + Handle createLayout( descset::DescriptorSetLayout const& description) { return mLayoutCache.getLayout(description); @@ -1031,25 +1080,62 @@ class VulkanDescriptorSetManager::Impl { } } + inline Handle getPlaceHolderLayout(uint8_t setID) { + if (mPlaceholderLayout[setID]) { + return mPlaceholderLayout[setID]; + } + descset::DescriptorSetLayout inputLayout { + .bindings = {{}}, + }; + switch (setID) { + case UBO_SET_ID: + inputLayout.bindings[0] = { + .type = descset::DescriptorType::UNIFORM_BUFFER, + .stageFlags = descset::ShaderStageFlags2::VERTEX, + .binding = 0, + .flags = descset::DescriptorFlags::NONE, + .count = 0, + }; + break; + case SAMPLER_SET_ID: + inputLayout.bindings[0] = { + .type = descset::DescriptorType::SAMPLER, + .stageFlags = descset::ShaderStageFlags2::FRAGMENT, + .binding = 0, + .flags = descset::DescriptorFlags::NONE, + .count = 0, + }; + break; + case INPUT_ATTACHMENT_SET_ID: + inputLayout.bindings[0] = { + .type = descset::DescriptorType::INPUT_ATTACHMENT, + .stageFlags = descset::ShaderStageFlags2::FRAGMENT, + .binding = 0, + .flags = descset::DescriptorFlags::NONE, + .count = 0, + }; + break; + default: + PANIC_POSTCONDITION("Unexpected set id=%d", setID); + } + mPlaceholderLayout[setID] = mLayoutCache.getLayout(inputLayout); + return mPlaceholderLayout[setID]; + } + VkDevice mDevice; VulkanResourceAllocator* mAllocator; LayoutCache mLayoutCache; DescriptorSetCache mDescriptorSetCache; - bool mHaveDynamicUbos; - UBOMap mUboMap; SamplerMap mSamplerMap; std::pair mInputAttachment; - VulkanResourceManager mResources; - VkDescriptorBufferInfo mPlaceHolderBufferInfo; VkDescriptorImageInfo mPlaceHolderImageInfo; - std::unordered_map mLayoutStash; - BoundState mBoundState; + VulkanDescriptorSetLayoutList mPlaceholderLayout = {}; }; VulkanDescriptorSetManager::VulkanDescriptorSetManager(VkDevice device, @@ -1077,6 +1163,10 @@ void VulkanDescriptorSetManager::dynamicBind(VulkanCommandBuffer* commands, mImpl->dynamicBind(commands, uboLayout); } +void VulkanDescriptorSetManager::clearProgram(VulkanProgram* program) noexcept { + mImpl->clearProgram(program); +} + Handle VulkanDescriptorSetManager::createLayout( descset::DescriptorSetLayout const& layout) { return mImpl->createLayout(layout); diff --git a/filament/backend/src/vulkan/caching/VulkanDescriptorSetManager.h b/filament/backend/src/vulkan/caching/VulkanDescriptorSetManager.h index f8871a57aec..2fa0b020fe1 100644 --- a/filament/backend/src/vulkan/caching/VulkanDescriptorSetManager.h +++ b/filament/backend/src/vulkan/caching/VulkanDescriptorSetManager.h @@ -69,6 +69,10 @@ class VulkanDescriptorSetManager { // proper dynamic binding when Filament-side descriptor changes are completed. void dynamicBind(VulkanCommandBuffer* commands, Handle uboLayout); + // TODO: Obsolete after [GDSR]. + // Since we use program pointer as cache key, we need to clear the cache when it's freed. + void clearProgram(VulkanProgram* program) noexcept; + Handle createLayout(descset::DescriptorSetLayout const& layout); void destroyLayout(Handle layout); diff --git a/filament/backend/src/vulkan/caching/VulkanPipelineLayoutCache.cpp b/filament/backend/src/vulkan/caching/VulkanPipelineLayoutCache.cpp index e1d3b0be9c6..ec96418aaf1 100644 --- a/filament/backend/src/vulkan/caching/VulkanPipelineLayoutCache.cpp +++ b/filament/backend/src/vulkan/caching/VulkanPipelineLayoutCache.cpp @@ -24,7 +24,7 @@ VkPipelineLayout VulkanPipelineLayoutCache::getLayout( VulkanDescriptorSetLayoutList const& descriptorSetLayouts) { PipelineLayoutKey key = {VK_NULL_HANDLE}; uint8_t descSetLayoutCount = 0; - for (auto layoutHandle : descriptorSetLayouts) { + for (auto layoutHandle: descriptorSetLayouts) { if (layoutHandle) { auto layout = mAllocator->handle_cast(layoutHandle); key[descSetLayoutCount++] = layout->vklayout; @@ -55,4 +55,10 @@ VkPipelineLayout VulkanPipelineLayoutCache::getLayout( return layout; } +void VulkanPipelineLayoutCache::terminate() noexcept { + for (auto const& [key, entry]: mPipelineLayouts) { + vkDestroyPipelineLayout(mDevice, entry.handle, VKALLOC); + } +} + }// namespace filament::backend diff --git a/filament/backend/src/vulkan/caching/VulkanPipelineLayoutCache.h b/filament/backend/src/vulkan/caching/VulkanPipelineLayoutCache.h index fff9fd2d227..375e6124d23 100644 --- a/filament/backend/src/vulkan/caching/VulkanPipelineLayoutCache.h +++ b/filament/backend/src/vulkan/caching/VulkanPipelineLayoutCache.h @@ -33,6 +33,8 @@ class VulkanPipelineLayoutCache { mAllocator(allocator), mTimestamp(0) {} + void terminate() noexcept; + using PipelineLayoutKey = std::array;