diff --git a/src/backend/vulkan/BufferVk.cpp b/src/backend/vulkan/BufferVk.cpp index 01b101f3dd..0af6ebd8bc 100644 --- a/src/backend/vulkan/BufferVk.cpp +++ b/src/backend/vulkan/BufferVk.cpp @@ -104,7 +104,7 @@ namespace vulkan { memcpy(memory + start * sizeof(uint32_t), data, count * sizeof(uint32_t)); - ToBackend(GetDevice())->FakeSubmit(); + ToBackend(GetDevice())->GetPendingCommandBuffer(); } void Buffer::MapReadAsyncImpl(uint32_t serial, uint32_t start, uint32_t /*count*/) { @@ -114,7 +114,7 @@ namespace vulkan { MapReadRequestTracker* tracker = ToBackend(GetDevice())->GetMapReadRequestTracker(); tracker->Track(this, serial, memory + start); - ToBackend(GetDevice())->FakeSubmit(); + ToBackend(GetDevice())->GetPendingCommandBuffer(); } void Buffer::UnmapImpl() { diff --git a/src/backend/vulkan/VulkanBackend.cpp b/src/backend/vulkan/VulkanBackend.cpp index e09861dd4b..055849a0f1 100644 --- a/src/backend/vulkan/VulkanBackend.cpp +++ b/src/backend/vulkan/VulkanBackend.cpp @@ -112,6 +112,9 @@ namespace vulkan { } Device::~Device() { + // Immediately forget about all pending commands so we don't try to submit them in Tick + FreeCommands(&pendingCommands); + if (fn.QueueWaitIdle(queue) != VK_SUCCESS) { ASSERT(false); } @@ -124,6 +127,12 @@ namespace vulkan { completedSerial = nextSerial; Tick(); + ASSERT(commandsInFlight.Empty()); + for (auto& commands : unusedCommands) { + FreeCommands(&commands); + } + unusedCommands.clear(); + for (VkFence fence : unusedFences) { fn.DestroyFence(vkDevice, fence, nullptr); } @@ -222,9 +231,14 @@ namespace vulkan { void Device::TickImpl() { CheckPassedFences(); + RecycleCompletedCommands(); mapReadRequestTracker->Tick(completedSerial); memoryAllocator->Tick(completedSerial); + + if (pendingCommands.pool != VK_NULL_HANDLE) { + SubmitPendingCommands(); + } } const VulkanDeviceInfo& Device::GetDeviceInfo() const { @@ -243,23 +257,41 @@ namespace vulkan { return nextSerial; } - VkInstance Device::GetInstance() const { - return instance; + VkCommandBuffer Device::GetPendingCommandBuffer() { + if (pendingCommands.pool == VK_NULL_HANDLE) { + pendingCommands = GetUnusedCommands(); + + VkCommandBufferBeginInfo beginInfo; + beginInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + beginInfo.pNext = nullptr; + beginInfo.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + beginInfo.pInheritanceInfo = nullptr; + + if (fn.BeginCommandBuffer(pendingCommands.commandBuffer, &beginInfo) != VK_SUCCESS) { + ASSERT(false); + } + } + + return pendingCommands.commandBuffer; } - VkDevice Device::GetVkDevice() const { - return vkDevice; - } + void Device::SubmitPendingCommands() { + if (pendingCommands.pool == VK_NULL_HANDLE) { + return; + } + + if (fn.EndCommandBuffer(pendingCommands.commandBuffer) != VK_SUCCESS) { + ASSERT(false); + } - void Device::FakeSubmit() { VkSubmitInfo submitInfo; submitInfo.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; submitInfo.pNext = nullptr; submitInfo.waitSemaphoreCount = 0; submitInfo.pWaitSemaphores = nullptr; submitInfo.pWaitDstStageMask = 0; - submitInfo.commandBufferCount = 0; - submitInfo.pCommandBuffers = 0; + submitInfo.commandBufferCount = 1; + submitInfo.pCommandBuffers = &pendingCommands.commandBuffer; submitInfo.signalSemaphoreCount = 0; submitInfo.pSignalSemaphores = 0; @@ -268,10 +300,20 @@ namespace vulkan { ASSERT(false); } + commandsInFlight.Enqueue(pendingCommands, nextSerial); + pendingCommands = CommandPoolAndBuffer(); fencesInFlight.emplace(fence, nextSerial); nextSerial++; } + VkInstance Device::GetInstance() const { + return instance; + } + + VkDevice Device::GetVkDevice() const { + return vkDevice; + } + bool Device::CreateInstance(VulkanGlobalKnobs* usedKnobs) { std::vector layersToRequest; std::vector extensionsToRequest; @@ -456,6 +498,59 @@ namespace vulkan { } } + Device::CommandPoolAndBuffer Device::GetUnusedCommands() { + if (!unusedCommands.empty()) { + CommandPoolAndBuffer commands = unusedCommands.back(); + unusedCommands.pop_back(); + return commands; + } + + CommandPoolAndBuffer commands; + + VkCommandPoolCreateInfo createInfo; + createInfo.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO; + createInfo.pNext = nullptr; + createInfo.flags = VK_COMMAND_POOL_CREATE_TRANSIENT_BIT; + createInfo.queueFamilyIndex = queueFamily; + + if (fn.CreateCommandPool(vkDevice, &createInfo, nullptr, &commands.pool) != VK_SUCCESS) { + ASSERT(false); + } + + VkCommandBufferAllocateInfo allocateInfo; + allocateInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; + allocateInfo.pNext = nullptr; + allocateInfo.commandPool = commands.pool; + allocateInfo.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + allocateInfo.commandBufferCount = 1; + + if (fn.AllocateCommandBuffers(vkDevice, &allocateInfo, &commands.commandBuffer) != VK_SUCCESS) { + ASSERT(false); + } + + return commands; + } + + void Device::RecycleCompletedCommands() { + for (auto& commands : commandsInFlight.IterateUpTo(completedSerial)) { + if (fn.ResetCommandPool(vkDevice, commands.pool, 0) != VK_SUCCESS) { + ASSERT(false); + } + unusedCommands.push_back(commands); + } + commandsInFlight.ClearUpTo(completedSerial); + } + + void Device::FreeCommands(CommandPoolAndBuffer* commands) { + if (commands->pool != VK_NULL_HANDLE) { + fn.DestroyCommandPool(vkDevice, commands->pool, nullptr); + commands->pool = VK_NULL_HANDLE; + } + + // Command buffers are implicitly destroyed when the command pool is. + commands->commandBuffer = VK_NULL_HANDLE; + } + // Queue Queue::Queue(QueueBuilder* builder) diff --git a/src/backend/vulkan/VulkanBackend.h b/src/backend/vulkan/VulkanBackend.h index dca43613a0..c0e41ccc0c 100644 --- a/src/backend/vulkan/VulkanBackend.h +++ b/src/backend/vulkan/VulkanBackend.h @@ -39,6 +39,7 @@ #include "backend/ToBackend.h" #include "common/DynamicLib.h" #include "common/Serial.h" +#include "common/SerialQueue.h" #include @@ -127,8 +128,12 @@ namespace vulkan { const VulkanDeviceInfo& GetDeviceInfo() const; MapReadRequestTracker* GetMapReadRequestTracker() const; MemoryAllocator* GetMemoryAllocator() const; + Serial GetSerial() const; + VkCommandBuffer GetPendingCommandBuffer(); + void SubmitPendingCommands(); + // Contains all the Vulkan entry points, vkDoFoo is called via device->fn.DoFoo. const VulkanFunctions fn; @@ -182,6 +187,19 @@ namespace vulkan { std::vector unusedFences; Serial nextSerial = 1; Serial completedSerial = 0; + + struct CommandPoolAndBuffer { + VkCommandPool pool = VK_NULL_HANDLE; + VkCommandBuffer commandBuffer = VK_NULL_HANDLE; + }; + + CommandPoolAndBuffer GetUnusedCommands(); + void RecycleCompletedCommands(); + void FreeCommands(CommandPoolAndBuffer* commands); + + SerialQueue commandsInFlight; + std::vector unusedCommands; + CommandPoolAndBuffer pendingCommands; }; class Queue : public QueueBase {