Residency 6: Enable D3D12 Residency and Add Tests

Toggles on D3D12 residency management by default. Adds basic residency
tests.

Bug: dawn:193
Change-Id: Idafa4a6d0f8f4052fb3428ac3d5f6be018db68cf
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/16385
Reviewed-by: Rafael Cintron <rafael.cintron@microsoft.com>
Reviewed-by: Austin Eng <enga@chromium.org>
Commit-Queue: Brandon Jones <brandon1.jones@intel.com>

BUILD.gn

@@ -375,6 +375,7 @@ source_set("dawn_white_box_tests_sources") {
   if (dawn_enable_d3d12) {
     sources += [
       "src/tests/white_box/D3D12DescriptorHeapTests.cpp",
+      "src/tests/white_box/D3D12ResidencyTests.cpp",
       "src/tests/white_box/D3D12SmallTextureTests.cpp",
     ]
   }

src/dawn_native/d3d12/BufferD3D12.cpp

@@ -314,6 +314,15 @@ namespace dawn_native { namespace d3d12 {
         ToBackend(GetDevice())->DeallocateMemory(mResourceAllocation);
     }
 
+    bool Buffer::CheckIsResidentForTesting() const {
+        Heap* heap = ToBackend(mResourceAllocation.GetResourceHeap());
+        return heap->IsInList() || heap->IsResidencyLocked();
+    }
+
+    bool Buffer::CheckAllocationMethodForTesting(AllocationMethod allocationMethod) const {
+        return mResourceAllocation.GetInfo().mMethod == allocationMethod;
+    }
+
     MapRequestTracker::MapRequestTracker(Device* device) : mDevice(device) {
     }
 

src/dawn_native/d3d12/BufferD3D12.h

@@ -42,6 +42,9 @@ namespace dawn_native { namespace d3d12 {
         void TrackUsageAndTransitionNow(CommandRecordingContext* commandContext,
                                         wgpu::BufferUsage newUsage);
 
+        bool CheckAllocationMethodForTesting(AllocationMethod allocationMethod) const;
+        bool CheckIsResidentForTesting() const;
+
       private:
         ~Buffer() override;
         // Dawn API

src/dawn_native/d3d12/DeviceD3D12.cpp

@@ -414,7 +414,7 @@ namespace dawn_native { namespace d3d12 {
         const bool useResourceHeapTier2 = (GetDeviceInfo().resourceHeapTier >= 2);
         SetToggle(Toggle::UseD3D12ResourceHeapTier2, useResourceHeapTier2);
         SetToggle(Toggle::UseD3D12RenderPass, GetDeviceInfo().supportsRenderPass);
-        SetToggle(Toggle::UseD3D12ResidencyManagement, false);
+        SetToggle(Toggle::UseD3D12ResidencyManagement, true);
 
         // By default use the maximum shader-visible heap size allowed.
         SetToggle(Toggle::UseD3D12SmallShaderVisibleHeapForTesting, false);

src/dawn_native/d3d12/ResidencyManagerD3D12.cpp

@@ -110,7 +110,15 @@ namespace dawn_native { namespace d3d12 {
         // continue to make forward progress. Note the choice to halve memory is arbitrarily chosen
         // and subject to future experimentation.
         mVideoMemoryInfo.externalReservation =
-            std::min(queryVideoMemoryInfo.Budget / 2, mVideoMemoryInfo.externalReservation);
+            std::min(queryVideoMemoryInfo.Budget / 2, mVideoMemoryInfo.externalRequest);
+
+        mVideoMemoryInfo.dawnUsage =
+            queryVideoMemoryInfo.CurrentUsage - mVideoMemoryInfo.externalReservation;
+
+        // If we're restricting the budget for testing, leave the budget as is.
+        if (mRestrictBudgetForTesting) {
+            return;
+        }
 
         // We cap Dawn's budget to 95% of the provided budget. Leaving some budget unused
         // decreases fluctuations in the operating-system-defined budget, which improves stability
@@ -119,8 +127,6 @@ namespace dawn_native { namespace d3d12 {
         static constexpr float kBudgetCap = 0.95;
         mVideoMemoryInfo.dawnBudget =
             (queryVideoMemoryInfo.Budget - mVideoMemoryInfo.externalReservation) * kBudgetCap;
-        mVideoMemoryInfo.dawnUsage =
-            queryVideoMemoryInfo.CurrentUsage - mVideoMemoryInfo.externalReservation;
     }
 
     // Removes from the LRU and returns the least recently used heap when possible. Returns nullptr
@@ -280,4 +286,21 @@ namespace dawn_native { namespace d3d12 {
 
         mLRUCache.Append(heap);
     }
+
+    // Places an artifical cap on Dawn's budget so we can test in a predictable manner. If used,
+    // this function must be called before any resources have been created.
+    void ResidencyManager::RestrictBudgetForTesting(uint64_t artificialBudgetCap) {
+        ASSERT(mLRUCache.empty());
+        ASSERT(!mRestrictBudgetForTesting);
+
+        mRestrictBudgetForTesting = true;
+        UpdateVideoMemoryInfo();
+
+        // Dawn has a non-zero memory usage even before any resources have been created, and this
+        // value can vary depending on the environment Dawn is running in. By adding this in
+        // addition to the artificial budget cap, we can create a predictable and reproducible
+        // budget for testing.
+        mVideoMemoryInfo.dawnBudget = mVideoMemoryInfo.dawnUsage + artificialBudgetCap;
+    }
+
 }}  // namespace dawn_native::d3d12

src/dawn_native/d3d12/ResidencyManagerD3D12.h

@@ -38,6 +38,8 @@ namespace dawn_native { namespace d3d12 {
 
         void TrackResidentAllocation(Heap* heap);
 
+        void RestrictBudgetForTesting(uint64_t);
+
       private:
         struct VideoMemoryInfo {
             uint64_t dawnBudget;
@@ -52,6 +54,7 @@ namespace dawn_native { namespace d3d12 {
         Device* mDevice;
         LinkedList<Heap> mLRUCache;
         bool mResidencyManagementEnabled = false;
+        bool mRestrictBudgetForTesting = false;
         VideoMemoryInfo mVideoMemoryInfo = {};
     };
 

src/tests/white_box/D3D12ResidencyTests.cpp

@@ -0,0 +1,279 @@
+// Copyright 2020 The Dawn Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "dawn_native/ResourceMemoryAllocation.h"
+#include "dawn_native/d3d12/BufferD3D12.h"
+#include "dawn_native/d3d12/DeviceD3D12.h"
+#include "dawn_native/d3d12/ResidencyManagerD3D12.h"
+#include "dawn_native/d3d12/ResourceHeapAllocationD3D12.h"
+#include "tests/DawnTest.h"
+#include "utils/WGPUHelpers.h"
+
+#include <vector>
+
+constexpr uint32_t kRestrictedBudgetSize = 100000000;         // 100MB
+constexpr uint32_t kDirectlyAllocatedResourceSize = 5000000;  // 5MB
+constexpr uint32_t kSuballocatedResourceSize = 1000000;       // 1MB
+constexpr uint32_t kSourceBufferSize = 4;                     // 4B
+
+class D3D12ResidencyTests : public DawnTest {
+  protected:
+    void TestSetUp() override {
+        DAWN_SKIP_TEST_IF(UsesWire());
+
+        // Restrict Dawn's budget to create an artificial budget.
+        dawn_native::d3d12::Device* d3dDevice =
+            reinterpret_cast<dawn_native::d3d12::Device*>(device.Get());
+        d3dDevice->GetResidencyManager()->RestrictBudgetForTesting(kRestrictedBudgetSize);
+
+        // Initialize a source buffer on the GPU to serve as a source to quickly copy data to other
+        // buffers.
+        constexpr uint32_t one = 1;
+        mSourceBuffer =
+            utils::CreateBufferFromData(device, &one, sizeof(one), wgpu::BufferUsage::CopySrc);
+    }
+
+    std::vector<wgpu::Buffer> AllocateBuffers(uint32_t bufferSize, uint32_t numberOfBuffers) {
+        std::vector<wgpu::Buffer> buffers;
+
+        for (uint64_t i = 0; i < numberOfBuffers; i++) {
+            buffers.push_back(CreateBuffer(bufferSize, wgpu::BufferUsage::CopyDst));
+        }
+
+        return buffers;
+    }
+
+    bool CheckIfBufferIsResident(wgpu::Buffer buffer) const {
+        dawn_native::d3d12::Buffer* d3dBuffer =
+            reinterpret_cast<dawn_native::d3d12::Buffer*>(buffer.Get());
+        return d3dBuffer->CheckIsResidentForTesting();
+    }
+
+    bool CheckAllocationMethod(wgpu::Buffer buffer,
+                               dawn_native::AllocationMethod allocationMethod) const {
+        dawn_native::d3d12::Buffer* d3dBuffer =
+            reinterpret_cast<dawn_native::d3d12::Buffer*>(buffer.Get());
+        return d3dBuffer->CheckAllocationMethodForTesting(allocationMethod);
+    }
+
+    bool IsUMA() const {
+        return reinterpret_cast<dawn_native::d3d12::Device*>(device.Get())->GetDeviceInfo().isUMA;
+    }
+
+    wgpu::Buffer CreateBuffer(uint32_t bufferSize, wgpu::BufferUsage usage) {
+        wgpu::BufferDescriptor descriptor;
+
+        descriptor.size = bufferSize;
+        descriptor.usage = usage;
+
+        return device.CreateBuffer(&descriptor);
+    }
+
+    static void MapReadCallback(WGPUBufferMapAsyncStatus status,
+                                const void* data,
+                                uint64_t,
+                                void* userdata) {
+        ASSERT_EQ(WGPUBufferMapAsyncStatus_Success, status);
+        ASSERT_NE(nullptr, data);
+
+        static_cast<D3D12ResidencyTests*>(userdata)->mMappedReadData = data;
+    }
+
+    static void MapWriteCallback(WGPUBufferMapAsyncStatus status,
+                                 void* data,
+                                 uint64_t,
+                                 void* userdata) {
+        ASSERT_EQ(WGPUBufferMapAsyncStatus_Success, status);
+        ASSERT_NE(nullptr, data);
+
+        static_cast<D3D12ResidencyTests*>(userdata)->mMappedWriteData = data;
+    }
+
+    void TouchBuffers(uint32_t beginIndex,
+                      uint32_t numBuffers,
+                      const std::vector<wgpu::Buffer>& bufferSet) {
+        wgpu::CommandEncoder encoder = device.CreateCommandEncoder();
+
+        // Perform a copy on the range of buffers to ensure the are moved to dedicated GPU memory.
+        for (uint32_t i = beginIndex; i < beginIndex + numBuffers; i++) {
+            encoder.CopyBufferToBuffer(mSourceBuffer, 0, bufferSet[i], 0, kSourceBufferSize);
+        }
+        wgpu::CommandBuffer copy = encoder.Finish();
+        queue.Submit(1, &copy);
+    }
+
+    wgpu::Buffer mSourceBuffer;
+    void* mMappedWriteData = nullptr;
+    const void* mMappedReadData = nullptr;
+};
+
+// Check that resources existing on suballocated heaps are made resident and evicted correctly.
+TEST_P(D3D12ResidencyTests, OvercommitSmallResources) {
+    // Create suballocated buffers to fill half the budget.
+    std::vector<wgpu::Buffer> bufferSet1 = AllocateBuffers(
+        kSuballocatedResourceSize, ((kRestrictedBudgetSize / 2) / kSuballocatedResourceSize));
+
+    // Check that all the buffers allocated are resident. Also make sure they were suballocated
+    // internally.
+    for (uint32_t i = 0; i < bufferSet1.size(); i++) {
+        EXPECT_TRUE(CheckIfBufferIsResident(bufferSet1[i]));
+        EXPECT_TRUE(
+            CheckAllocationMethod(bufferSet1[i], dawn_native::AllocationMethod::kSubAllocated));
+    }
+
+    // Create enough directly-allocated buffers to use the entire budget.
+    std::vector<wgpu::Buffer> bufferSet2 = AllocateBuffers(
+        kDirectlyAllocatedResourceSize, kRestrictedBudgetSize / kDirectlyAllocatedResourceSize);
+
+    // Check that everything in bufferSet1 is now evicted.
+    for (uint32_t i = 0; i < bufferSet1.size(); i++) {
+        EXPECT_FALSE(CheckIfBufferIsResident(bufferSet1[i]));
+    }
+
+    // Touch one of the non-resident buffers. This should cause the buffer to become resident.
+    constexpr uint32_t indexOfBufferInSet1 = 5;
+    TouchBuffers(indexOfBufferInSet1, 1, bufferSet1);
+    // Check that this buffer is now resident.
+    EXPECT_TRUE(CheckIfBufferIsResident(bufferSet1[indexOfBufferInSet1]));
+
+    // Touch everything in bufferSet2 again to evict the buffer made resident in the previous
+    // operation.
+    TouchBuffers(0, bufferSet2.size(), bufferSet2);
+    // Check that indexOfBufferInSet1 was evicted.
+    EXPECT_FALSE(CheckIfBufferIsResident(bufferSet1[indexOfBufferInSet1]));
+}
+
+// Check that resources existing on directly allocated heaps are made resident and evicted
+// correctly.
+TEST_P(D3D12ResidencyTests, OvercommitLargeResources) {
+    // Create directly-allocated buffers to fill half the budget.
+    std::vector<wgpu::Buffer> bufferSet1 =
+        AllocateBuffers(kDirectlyAllocatedResourceSize,
+                        ((kRestrictedBudgetSize / 2) / kDirectlyAllocatedResourceSize));
+
+    // Check that all the allocated buffers are resident. Also make sure they were directly
+    // allocated internally.
+    for (uint32_t i = 0; i < bufferSet1.size(); i++) {
+        EXPECT_TRUE(CheckIfBufferIsResident(bufferSet1[i]));
+        EXPECT_TRUE(CheckAllocationMethod(bufferSet1[i], dawn_native::AllocationMethod::kDirect));
+    }
+
+    // Create enough directly-allocated buffers to use the entire budget.
+    std::vector<wgpu::Buffer> bufferSet2 = AllocateBuffers(
+        kDirectlyAllocatedResourceSize, kRestrictedBudgetSize / kDirectlyAllocatedResourceSize);
+
+    // Check that everything in bufferSet1 is now evicted.
+    for (uint32_t i = 0; i < bufferSet1.size(); i++) {
+        EXPECT_FALSE(CheckIfBufferIsResident(bufferSet1[i]));
+    }
+
+    // Touch one of the non-resident buffers. This should cause the buffer to become resident.
+    constexpr uint32_t indexOfBufferInSet1 = 1;
+    TouchBuffers(indexOfBufferInSet1, 1, bufferSet1);
+    EXPECT_TRUE(CheckIfBufferIsResident(bufferSet1[indexOfBufferInSet1]));
+
+    // Touch everything in bufferSet2 again to evict the buffer made resident in the previous
+    // operation.
+    TouchBuffers(0, bufferSet2.size(), bufferSet2);
+    // Check that indexOfBufferInSet1 was evicted.
+    EXPECT_FALSE(CheckIfBufferIsResident(bufferSet1[indexOfBufferInSet1]));
+}
+
+// Check that calling MapReadAsync makes the buffer resident and keeps it locked resident.
+TEST_P(D3D12ResidencyTests, AsyncMappedBufferRead) {
+    // Dawn currently only manages LOCAL_MEMORY. Mappable buffers exist in NON_LOCAL_MEMORY on
+    // discrete devices.
+    DAWN_SKIP_TEST_IF(!IsUMA())
+
+    // Create a mappable buffer.
+    wgpu::Buffer buffer = CreateBuffer(4, wgpu::BufferUsage::MapRead | wgpu::BufferUsage::CopyDst);
+
+    uint32_t data = 12345;
+    buffer.SetSubData(0, sizeof(uint32_t), &data);
+
+    // The mappable buffer should be resident.
+    EXPECT_TRUE(CheckIfBufferIsResident(buffer));
+
+    // Create and touch enough buffers to use the entire budget.
+    std::vector<wgpu::Buffer> bufferSet = AllocateBuffers(
+        kDirectlyAllocatedResourceSize, kRestrictedBudgetSize / kDirectlyAllocatedResourceSize);
+    TouchBuffers(0, bufferSet.size(), bufferSet);
+
+    // The mappable buffer should have been evicted.
+    EXPECT_FALSE(CheckIfBufferIsResident(buffer));
+
+    // Calling MapReadAsync should make the buffer resident.
+    buffer.MapReadAsync(MapReadCallback, this);
+    EXPECT_TRUE(CheckIfBufferIsResident(buffer));
+
+    while (mMappedReadData == nullptr) {
+        WaitABit();
+    }
+
+    // Touch enough resources such that the entire budget is used. The mappable buffer should remain
+    // locked resident.
+    TouchBuffers(0, bufferSet.size(), bufferSet);
+    EXPECT_TRUE(CheckIfBufferIsResident(buffer));
+
+    // Unmap the buffer, allocate and touch enough resources such that the entire budget is used.
+    // This should evict the mappable buffer.
+    buffer.Unmap();
+    std::vector<wgpu::Buffer> bufferSet2 = AllocateBuffers(
+        kDirectlyAllocatedResourceSize, kRestrictedBudgetSize / kDirectlyAllocatedResourceSize);
+    TouchBuffers(0, bufferSet2.size(), bufferSet2);
+    EXPECT_FALSE(CheckIfBufferIsResident(buffer));
+}
+
+// Check that calling MapWriteAsync makes the buffer resident and keeps it locked resident.
+TEST_P(D3D12ResidencyTests, AsyncMappedBufferWrite) {
+    // Dawn currently only manages LOCAL_MEMORY. Mappable buffers exist in NON_LOCAL_MEMORY on
+    // discrete devices.
+    DAWN_SKIP_TEST_IF(!IsUMA())
+
+    // Create a mappable buffer.
+    wgpu::Buffer buffer = CreateBuffer(4, wgpu::BufferUsage::MapWrite | wgpu::BufferUsage::CopySrc);
+    // The mappable buffer should be resident.
+    EXPECT_TRUE(CheckIfBufferIsResident(buffer));
+
+    // Create and touch enough buffers to use the entire budget.
+    std::vector<wgpu::Buffer> bufferSet1 = AllocateBuffers(
+        kDirectlyAllocatedResourceSize, kRestrictedBudgetSize / kDirectlyAllocatedResourceSize);
+    TouchBuffers(0, bufferSet1.size(), bufferSet1);
+
+    // The mappable buffer should have been evicted.
+    EXPECT_FALSE(CheckIfBufferIsResident(buffer));
+
+    // Calling MapWriteAsync should make the buffer resident.
+    buffer.MapWriteAsync(MapWriteCallback, this);
+    EXPECT_TRUE(CheckIfBufferIsResident(buffer));
+
+    while (mMappedWriteData == nullptr) {
+        WaitABit();
+    }
+
+    // Touch enough resources such that the entire budget is used. The mappable buffer should remain
+    // locked resident.
+    TouchBuffers(0, bufferSet1.size(), bufferSet1);
+    EXPECT_TRUE(CheckIfBufferIsResident(buffer));
+
+    // Unmap the buffer, allocate and touch enough resources such that the entire budget is used.
+    // This should evict the mappable buffer.
+    buffer.Unmap();
+    std::vector<wgpu::Buffer> bufferSet2 = AllocateBuffers(
+        kDirectlyAllocatedResourceSize, kRestrictedBudgetSize / kDirectlyAllocatedResourceSize);
+    TouchBuffers(0, bufferSet2.size(), bufferSet2);
+    EXPECT_FALSE(CheckIfBufferIsResident(buffer));
+}
+
+DAWN_INSTANTIATE_TEST(D3D12ResidencyTests, D3D12Backend());