Vulkan: Keep track of finished operations with fences

src/backend/vulkan/BufferVk.cpp

@@ -103,6 +103,8 @@ namespace vulkan {
         ASSERT(memory != nullptr);
 
         memcpy(memory + start * sizeof(uint32_t), data, count * sizeof(uint32_t));
+
+        ToBackend(GetDevice())->FakeSubmit();
     }
 
     void Buffer::MapReadAsyncImpl(uint32_t serial, uint32_t start, uint32_t /*count*/) {
@@ -111,6 +113,8 @@ namespace vulkan {
 
         MapReadRequestTracker* tracker = ToBackend(GetDevice())->GetMapReadRequestTracker();
         tracker->Track(this, serial, memory + start);
+
+        ToBackend(GetDevice())->FakeSubmit();
     }
 
     void Buffer::UnmapImpl() {

src/backend/vulkan/VulkanBackend.cpp

@@ -112,9 +112,23 @@ namespace vulkan {
     }
 
     Device::~Device() {
-        // TODO(cwallez@chromium.org): properly wait on everything to be finished
+        if (fn.QueueWaitIdle(queue) != VK_SUCCESS) {
+            ASSERT(false);
+        }
+        CheckPassedFences();
+        ASSERT(fencesInFlight.empty());
+
+        // Some operations might have been started since the last submit and waiting
+        // on a serial that doesn't have a corresponding fence enqueued. Force all
+        // operations to look as if they were completed (because they were).
+        completedSerial = nextSerial;
         Tick();
 
+        for (VkFence fence : unusedFences) {
+            fn.DestroyFence(vkDevice, fence, nullptr);
+        }
+        unusedFences.clear();
+
         if (memoryAllocator) {
             delete memoryAllocator;
             memoryAllocator = nullptr;
@@ -207,10 +221,7 @@ namespace vulkan {
     }
 
     void Device::TickImpl() {
-        // TODO(cwallez@chromium.org): Correctly track the serial with Semaphores
-        fn.QueueWaitIdle(queue);
-        completedSerial = nextSerial;
-        nextSerial++;
+        CheckPassedFences();
 
         mapReadRequestTracker->Tick(completedSerial);
         memoryAllocator->Tick(completedSerial);
@@ -240,6 +251,27 @@ namespace vulkan {
         return vkDevice;
     }
 
+    void Device::FakeSubmit() {
+        VkSubmitInfo submitInfo;
+        submitInfo.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
+        submitInfo.pNext = nullptr;
+        submitInfo.waitSemaphoreCount = 0;
+        submitInfo.pWaitSemaphores = nullptr;
+        submitInfo.pWaitDstStageMask = 0;
+        submitInfo.commandBufferCount = 0;
+        submitInfo.pCommandBuffers = 0;
+        submitInfo.signalSemaphoreCount = 0;
+        submitInfo.pSignalSemaphores = 0;
+
+        VkFence fence = GetUnusedFence();
+        if (fn.QueueSubmit(queue, 1, &submitInfo, fence) != VK_SUCCESS) {
+            ASSERT(false);
+        }
+
+        fencesInFlight.emplace(fence, nextSerial);
+        nextSerial++;
+    }
+
     bool Device::CreateInstance(VulkanGlobalKnobs* usedKnobs) {
         std::vector<const char*> layersToRequest;
         std::vector<const char*> extensionsToRequest;
@@ -378,6 +410,52 @@ namespace vulkan {
         return const_cast<VulkanFunctions*>(&fn);
     }
 
+    VkFence Device::GetUnusedFence() {
+        if (!unusedFences.empty()) {
+            VkFence fence = unusedFences.back();
+            unusedFences.pop_back();
+            return fence;
+        }
+
+        VkFenceCreateInfo createInfo;
+        createInfo.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
+        createInfo.pNext = nullptr;
+        createInfo.flags = 0;
+
+        VkFence fence = VK_NULL_HANDLE;
+        if (fn.CreateFence(vkDevice, &createInfo, nullptr, &fence) != VK_SUCCESS) {
+            ASSERT(false);
+        }
+
+        return fence;
+    }
+
+    void Device::CheckPassedFences() {
+        while (!fencesInFlight.empty()) {
+            VkFence fence = fencesInFlight.front().first;
+            Serial fenceSerial = fencesInFlight.front().second;
+
+            VkResult result = fn.GetFenceStatus(vkDevice, fence);
+            ASSERT(result == VK_SUCCESS || result == VK_NOT_READY);
+
+            // Fence are added in order, so we can stop searching as soon
+            // as we see one that's not ready.
+            if (result == VK_NOT_READY) {
+                return;
+            }
+
+            if (fn.ResetFences(vkDevice, 1, &fence) != VK_SUCCESS) {
+                ASSERT(false);
+            }
+            unusedFences.push_back(fence);
+
+            fencesInFlight.pop();
+
+            ASSERT(fenceSerial > completedSerial);
+            completedSerial = fenceSerial;
+        }
+    }
+
     // Queue
 
     Queue::Queue(QueueBuilder* builder)

src/backend/vulkan/VulkanBackend.h

@@ -40,6 +40,8 @@
 #include "common/DynamicLib.h"
 #include "common/Serial.h"
 
+#include <queue>
+
 namespace backend {
 namespace vulkan {
 
@@ -133,6 +135,8 @@ namespace vulkan {
             VkInstance GetInstance() const;
             VkDevice GetVkDevice() const;
 
+            void FakeSubmit();
+
         private:
             bool CreateInstance(VulkanGlobalKnobs* usedKnobs);
             bool CreateDevice(VulkanDeviceKnobs* usedKnobs);
@@ -164,10 +168,20 @@ namespace vulkan {
             VkQueue queue = VK_NULL_HANDLE;
             VkDebugReportCallbackEXT debugReportCallback = VK_NULL_HANDLE;
 
-            Serial nextSerial = 1;
-            Serial completedSerial = 0;
             MapReadRequestTracker* mapReadRequestTracker = nullptr;
             MemoryAllocator* memoryAllocator = nullptr;
+
+            VkFence GetUnusedFence();
+            void CheckPassedFences();
+
+            // We track which operations are in flight on the GPU with an increasing serial.
+            // This works only because we have a single queue. Each submit to a queue is associated
+            // to a serial and a fence, such that when the fence is "ready" we know the operations
+            // have finished.
+            std::queue<std::pair<VkFence, Serial>> fencesInFlight;
+            std::vector<VkFence> unusedFences;
+            Serial nextSerial = 1;
+            Serial completedSerial = 0;
     };
 
     class Queue : public QueueBase {