From d5ec7bcc1ed5dbb5d25b1949cb4a94390277ef6e Mon Sep 17 00:00:00 2001 From: Jack Andersen Date: Sun, 27 May 2018 11:35:21 -1000 Subject: [PATCH] Slab allocation of VkDescriptorPools --- include/boo/graphicsdev/Vulkan.hpp | 1 - lib/graphicsdev/Vulkan.cpp | 121 ++++++++++++++++++----------- 2 files changed, 77 insertions(+), 45 deletions(-) diff --git a/include/boo/graphicsdev/Vulkan.hpp b/include/boo/graphicsdev/Vulkan.hpp index 22b7c5c..c1f65d8 100644 --- a/include/boo/graphicsdev/Vulkan.hpp +++ b/include/boo/graphicsdev/Vulkan.hpp @@ -46,7 +46,6 @@ struct VulkanContext std::mutex m_queueLock; VkDescriptorSetLayout m_descSetLayout = VK_NULL_HANDLE; VkPipelineLayout m_pipelinelayout = VK_NULL_HANDLE; - VkDescriptorPool m_descPool = VK_NULL_HANDLE; VkRenderPass m_pass = VK_NULL_HANDLE; VkRenderPass m_passColorOnly = VK_NULL_HANDLE; VkCommandPool m_loadPool = VK_NULL_HANDLE; diff --git a/lib/graphicsdev/Vulkan.cpp b/lib/graphicsdev/Vulkan.cpp index b484a08..680191d 100644 --- a/lib/graphicsdev/Vulkan.cpp +++ b/lib/graphicsdev/Vulkan.cpp @@ -17,7 +17,7 @@ #include "logvisor/logvisor.hpp" -#define BOO_VK_MAX_DESCRIPTOR_SETS 262144 +#define BOO_VK_MAX_DESCRIPTOR_SETS 65536 #undef min #undef max @@ -66,6 +66,7 @@ static logvisor::Module Log("boo::Vulkan"); VulkanContext g_VulkanContext; class VulkanDataFactoryImpl; struct VulkanCommandQueue; +struct VulkanDescriptorPool; struct VulkanShareableShader : IShareableShader { @@ -82,8 +83,11 @@ class VulkanDataFactoryImpl : public VulkanDataFactory, public GraphicsDataFacto friend class VulkanDataFactory::Context; friend struct VulkanData; friend struct VulkanPool; + friend struct VulkanDescriptorPool; + friend struct VulkanShaderDataBinding; IGraphicsContext* m_parent; VulkanContext* m_ctx; + VulkanDescriptorPool* m_descPoolHead = nullptr; std::unordered_map> m_sharedShaders; std::unordered_map m_sourceToBinary; std::vector m_texUnis; @@ -136,10 +140,16 @@ class VulkanDataFactoryImpl : public VulkanDataFactory, public GraphicsDataFacto public: VulkanDataFactoryImpl(IGraphicsContext* parent, VulkanContext* ctx); + ~VulkanDataFactoryImpl() + { + assert(m_descPoolHead == nullptr && "Dangling descriptor pools detected"); + } Platform platform() const {return Platform::Vulkan;} const SystemChar* platformName() const {return _S("Vulkan");} + boo::ObjToken allocateDescriptorSets(VkDescriptorSet* out); + void commitTransaction(const FactoryCommitFunc& __BooTraceArgs); boo::ObjToken newPoolBuffer(BufferUse use, size_t stride, size_t count __BooTraceArgs); @@ -198,7 +208,7 @@ dbgFunc(VkDebugReportFlagsEXT msgFlags, VkDebugReportObjectTypeEXT objType, * That's what would happen without validation layers, so we'll * keep that behavior here. */ - return false; + return VK_FALSE; } static void SetImageLayout(VkCommandBuffer cmd, VkImage image, @@ -665,23 +675,6 @@ void VulkanContext::initDevice() pipelineLayout.pSetLayouts = &m_descSetLayout; ThrowIfFailed(vk::CreatePipelineLayout(m_dev, &pipelineLayout, nullptr, &m_pipelinelayout)); - VkDescriptorPoolSize poolSizes[2] = {}; - VkDescriptorPoolCreateInfo descriptorPoolInfo = {}; - descriptorPoolInfo.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO; - descriptorPoolInfo.flags = VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT; - descriptorPoolInfo.pNext = nullptr; - descriptorPoolInfo.maxSets = BOO_VK_MAX_DESCRIPTOR_SETS; - descriptorPoolInfo.poolSizeCount = 2; - descriptorPoolInfo.pPoolSizes = poolSizes; - - poolSizes[0].type = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER; - poolSizes[0].descriptorCount = BOO_GLSL_MAX_UNIFORM_COUNT * BOO_VK_MAX_DESCRIPTOR_SETS; - - poolSizes[1].type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; - poolSizes[1].descriptorCount = BOO_GLSL_MAX_TEXTURE_COUNT * BOO_VK_MAX_DESCRIPTOR_SETS; - - ThrowIfFailed(vk::CreateDescriptorPool(m_dev, &descriptorPoolInfo, nullptr, &m_descPool)); - std::string gpuName = m_gpuProps.deviceName; Log.report(logvisor::Info, "Initialized %s", gpuName.c_str()); Log.report(logvisor::Info, "Vulkan version %d.%d.%d", @@ -708,12 +701,6 @@ void VulkanContext::destroyDevice() m_pass = VK_NULL_HANDLE; } - if (m_descPool) - { - vk::DestroyDescriptorPool(m_dev, m_descPool, nullptr); - m_descPool = VK_NULL_HANDLE; - } - if (m_pipelinelayout) { vk::DestroyPipelineLayout(m_dev, m_pipelinelayout, nullptr); @@ -1093,6 +1080,66 @@ bool VulkanContext::_resizeSwapChains() return true; } +struct VulkanDescriptorPool : ListNode +{ + VkDescriptorPool m_descPool; + int m_allocatedSets = 0; + + VulkanDescriptorPool(VulkanDataFactoryImpl* factory) + : ListNode(factory) + { + VkDescriptorPoolSize poolSizes[2] = {}; + VkDescriptorPoolCreateInfo descriptorPoolInfo = {}; + descriptorPoolInfo.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO; + descriptorPoolInfo.pNext = nullptr; + descriptorPoolInfo.maxSets = BOO_VK_MAX_DESCRIPTOR_SETS; + descriptorPoolInfo.poolSizeCount = 2; + descriptorPoolInfo.pPoolSizes = poolSizes; + + poolSizes[0].type = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER; + poolSizes[0].descriptorCount = BOO_GLSL_MAX_UNIFORM_COUNT * BOO_VK_MAX_DESCRIPTOR_SETS; + + poolSizes[1].type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; + poolSizes[1].descriptorCount = BOO_GLSL_MAX_TEXTURE_COUNT * BOO_VK_MAX_DESCRIPTOR_SETS; + + ThrowIfFailed(vk::CreateDescriptorPool(factory->m_ctx->m_dev, &descriptorPoolInfo, nullptr, &m_descPool)); + } + + ~VulkanDescriptorPool() + { + vk::DestroyDescriptorPool(m_head->m_ctx->m_dev, m_descPool, nullptr); + } + + std::unique_lock destructorLock() override + { return std::unique_lock{m_head->m_dataMutex}; } + static std::unique_lock _getHeadLock(VulkanDataFactoryImpl* factory) + { return std::unique_lock{factory->m_dataMutex}; } + static VulkanDescriptorPool*& _getHeadPtr(VulkanDataFactoryImpl* factory) + { return factory->m_descPoolHead; } +}; + +boo::ObjToken VulkanDataFactoryImpl::allocateDescriptorSets(VkDescriptorSet* out) +{ + std::lock_guard lk(m_dataMutex); + boo::ObjToken pool; + if (!m_descPoolHead || m_descPoolHead->m_allocatedSets == BOO_VK_MAX_DESCRIPTOR_SETS) + pool = new VulkanDescriptorPool(this); + else + pool = m_descPoolHead; + + VkDescriptorSetLayout layouts[] = {m_ctx->m_descSetLayout, m_ctx->m_descSetLayout}; + VkDescriptorSetAllocateInfo descAllocInfo; + descAllocInfo.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO; + descAllocInfo.pNext = nullptr; + descAllocInfo.descriptorPool = pool->m_descPool; + descAllocInfo.descriptorSetCount = 2; + descAllocInfo.pSetLayouts = layouts; + ThrowIfFailed(vk::AllocateDescriptorSets(m_ctx->m_dev, &descAllocInfo, out)); + pool->m_allocatedSets += 2; + + return pool; +} + struct AllocatedBuffer { VkBuffer m_buffer = VK_NULL_HANDLE; @@ -2414,6 +2461,7 @@ struct VulkanShaderDataBinding : GraphicsDataNode VkDeviceSize m_vboOffs[2][2] = {{},{}}; VkBuffer m_iboBufs[2] = {}; VkDeviceSize m_iboOffs[2] = {}; + boo::ObjToken m_descPool; VkDescriptorSet m_descSets[2] = {}; size_t m_vertOffset; @@ -2425,7 +2473,7 @@ struct VulkanShaderDataBinding : GraphicsDataNode #endif VulkanShaderDataBinding(const boo::ObjToken& d, - VulkanContext* ctx, + VulkanDataFactoryImpl& factory, const boo::ObjToken& pipeline, const boo::ObjToken& vbuf, const boo::ObjToken& instVbuf, @@ -2436,7 +2484,7 @@ struct VulkanShaderDataBinding : GraphicsDataNode const int* bindIdxs, const bool* depthBinds, size_t baseVert, size_t baseInst) : GraphicsDataNode(d), - m_ctx(ctx), + m_ctx(factory.m_ctx), m_pipeline(pipeline), m_vbuf(vbuf), m_instVbuf(instVbuf), @@ -2478,22 +2526,7 @@ struct VulkanShaderDataBinding : GraphicsDataNode size_t totalDescs = ubufCount + texCount; if (totalDescs > 0) - { - VkDescriptorSetLayout layouts[] = {ctx->m_descSetLayout, ctx->m_descSetLayout}; - VkDescriptorSetAllocateInfo descAllocInfo; - descAllocInfo.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO; - descAllocInfo.pNext = nullptr; - descAllocInfo.descriptorPool = ctx->m_descPool; - descAllocInfo.descriptorSetCount = 2; - descAllocInfo.pSetLayouts = layouts; - ThrowIfFailed(vk::AllocateDescriptorSets(ctx->m_dev, &descAllocInfo, m_descSets)); - } - } - - ~VulkanShaderDataBinding() - { - if (m_descSets[0]) - ThrowIfFailed(vk::FreeDescriptorSets(m_ctx->m_dev, m_ctx->m_descPool, 2, m_descSets)); + m_descPool = factory.allocateDescriptorSets(m_descSets); } void commit(VulkanContext* ctx) @@ -3709,7 +3742,7 @@ VulkanDataFactory::Context::newShaderDataBinding( size_t baseVert, size_t baseInst) { VulkanDataFactoryImpl& factory = static_cast(m_parent); - return {new VulkanShaderDataBinding(m_data, factory.m_ctx, pipeline, vbuf, instVbuf, ibuf, + return {new VulkanShaderDataBinding(m_data, factory, pipeline, vbuf, instVbuf, ibuf, ubufCount, ubufs, ubufOffs, ubufSizes, texCount, texs, bindIdxs, bindDepth, baseVert, baseInst)}; }