Residency 5: Implement and Integrate Residency Management

Adds all D3D12 residency management logic.

Bug: dawn:193
Change-Id: Ibc160289c29cc85817dcfaaef1b92d04599aa802
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/16384
Reviewed-by: Austin Eng <enga@chromium.org>
Reviewed-by: Corentin Wallez <cwallez@chromium.org>
Commit-Queue: Brandon Jones <brandon1.jones@intel.com>

src/common/LinkedList.h

@@ -7,6 +7,8 @@
 #ifndef COMMON_LINKED_LIST_H
 #define COMMON_LINKED_LIST_H
 
+#include "common/Assert.h"
+
 // Simple LinkedList type. (See the Q&A section to understand how this
 // differs from std::list).
 //
@@ -117,6 +119,12 @@ class LinkNode {
         e->next_ = this;
     }
 
+    // Check if |this| is in a list.
+    bool IsInList() const {
+        ASSERT((this->previous_ == nullptr) == (this->next_ == nullptr));
+        return this->next_ != nullptr;
+    }
+
     // Remove |this| from the linked list.
     void RemoveFromList() {
         this->previous_->next_ = this->next_;

src/dawn_native/Buffer.cpp

@@ -463,4 +463,8 @@ namespace dawn_native {
         mState = BufferState::Destroyed;
     }
 
+    bool BufferBase::IsMapped() const {
+        return mState == BufferState::Mapped;
+    }
+
 }  // namespace dawn_native

src/dawn_native/Buffer.h

@@ -82,6 +82,8 @@ namespace dawn_native {
 
         void DestroyInternal();
 
+        bool IsMapped() const;
+
       private:
         virtual MaybeError MapAtCreationImpl(uint8_t** mappedPointer) = 0;
         virtual MaybeError SetSubDataImpl(uint32_t start, uint32_t count, const void* data);

src/dawn_native/d3d12/BufferD3D12.cpp

@@ -21,6 +21,7 @@
 #include "dawn_native/d3d12/D3D12Error.h"
 #include "dawn_native/d3d12/DeviceD3D12.h"
 #include "dawn_native/d3d12/HeapD3D12.h"
+#include "dawn_native/d3d12/ResidencyManagerD3D12.h"
 
 namespace dawn_native { namespace d3d12 {
 
@@ -244,6 +245,11 @@ namespace dawn_native { namespace d3d12 {
     }
 
     MaybeError Buffer::MapAtCreationImpl(uint8_t** mappedPointer) {
+        // The mapped buffer can be accessed at any time, so it must be locked to ensure it is never
+        // evicted. This buffer should already have been made resident when it was created.
+        Heap* heap = ToBackend(mResourceAllocation.GetResourceHeap());
+        DAWN_TRY(ToBackend(GetDevice())->GetResidencyManager()->LockMappableHeap(heap));
+
         mWrittenMappedRange = {0, static_cast<size_t>(GetSize())};
         DAWN_TRY(CheckHRESULT(GetD3D12Resource()->Map(0, &mWrittenMappedRange,
                                                       reinterpret_cast<void**>(mappedPointer)),
@@ -252,6 +258,11 @@ namespace dawn_native { namespace d3d12 {
     }
 
     MaybeError Buffer::MapReadAsyncImpl(uint32_t serial) {
+        // The mapped buffer can be accessed at any time, so we must make the buffer resident and
+        // lock it to ensure it is never evicted.
+        Heap* heap = ToBackend(mResourceAllocation.GetResourceHeap());
+        DAWN_TRY(ToBackend(GetDevice())->GetResidencyManager()->LockMappableHeap(heap));
+
         mWrittenMappedRange = {};
         D3D12_RANGE readRange = {0, static_cast<size_t>(GetSize())};
         char* data = nullptr;
@@ -266,6 +277,11 @@ namespace dawn_native { namespace d3d12 {
     }
 
     MaybeError Buffer::MapWriteAsyncImpl(uint32_t serial) {
+        // The mapped buffer can be accessed at any time, so we must make the buffer resident and
+        // lock it to ensure it is never evicted.
+        Heap* heap = ToBackend(mResourceAllocation.GetResourceHeap());
+        DAWN_TRY(ToBackend(GetDevice())->GetResidencyManager()->LockMappableHeap(heap));
+
         mWrittenMappedRange = {0, static_cast<size_t>(GetSize())};
         char* data = nullptr;
         DAWN_TRY(CheckHRESULT(
@@ -280,10 +296,21 @@ namespace dawn_native { namespace d3d12 {
 
     void Buffer::UnmapImpl() {
         GetD3D12Resource()->Unmap(0, &mWrittenMappedRange);
+        // When buffers are mapped, they are locked to keep them in resident memory. We must unlock
+        // them when they are unmapped.
+        Heap* heap = ToBackend(mResourceAllocation.GetResourceHeap());
+        ToBackend(GetDevice())->GetResidencyManager()->UnlockMappableHeap(heap);
         mWrittenMappedRange = {};
     }
 
     void Buffer::DestroyImpl() {
+        // We must ensure that if a mapped buffer is destroyed, it does not leave a dangling lock
+        // reference on its heap.
+        if (IsMapped()) {
+            Heap* heap = ToBackend(mResourceAllocation.GetResourceHeap());
+            ToBackend(GetDevice())->GetResidencyManager()->UnlockMappableHeap(heap);
+        }
+
         ToBackend(GetDevice())->DeallocateMemory(mResourceAllocation);
     }
 

src/dawn_native/d3d12/CommandRecordingContext.cpp

@@ -14,7 +14,9 @@
 #include "dawn_native/d3d12/CommandRecordingContext.h"
 #include "dawn_native/d3d12/CommandAllocatorManager.h"
 #include "dawn_native/d3d12/D3D12Error.h"
+#include "dawn_native/d3d12/DeviceD3D12.h"
 #include "dawn_native/d3d12/HeapD3D12.h"
+#include "dawn_native/d3d12/ResidencyManagerD3D12.h"
 
 namespace dawn_native { namespace d3d12 {
 
@@ -52,7 +54,7 @@ namespace dawn_native { namespace d3d12 {
         return {};
     }
 
-    MaybeError CommandRecordingContext::ExecuteCommandList(ID3D12CommandQueue* d3d12CommandQueue) {
+    MaybeError CommandRecordingContext::ExecuteCommandList(Device* device) {
         if (IsOpen()) {
             // Shared textures must be transitioned to common state after the last usage in order
             // for them to be used by other APIs like D3D11. We ensure this by transitioning to the
@@ -68,9 +70,11 @@ namespace dawn_native { namespace d3d12 {
                 Release();
                 DAWN_TRY(std::move(error));
             }
+            DAWN_TRY(device->GetResidencyManager()->EnsureHeapsAreResident(
+                mHeapsPendingUsage.data(), mHeapsPendingUsage.size()));
 
             ID3D12CommandList* d3d12CommandList = GetCommandList();
-            d3d12CommandQueue->ExecuteCommandLists(1, &d3d12CommandList);
+            device->GetCommandQueue()->ExecuteCommandLists(1, &d3d12CommandList);
 
             mIsOpen = false;
             mSharedTextures.clear();

src/dawn_native/d3d12/CommandRecordingContext.h

@@ -35,7 +35,7 @@ namespace dawn_native { namespace d3d12 {
         void Release();
         bool IsOpen() const;
 
-        MaybeError ExecuteCommandList(ID3D12CommandQueue* d3d12CommandQueue);
+        MaybeError ExecuteCommandList(Device* device);
 
         void TrackHeapUsage(Heap* heap, Serial serial);
 

src/dawn_native/d3d12/DeviceD3D12.cpp

@@ -228,7 +228,7 @@ namespace dawn_native { namespace d3d12 {
     }
 
     MaybeError Device::ExecutePendingCommandContext() {
-        return mPendingCommands.ExecuteCommandList(mCommandQueue.Get());
+        return mPendingCommands.ExecuteCommandList(this);
     }
 
     ResultOrError<BindGroupBase*> Device::CreateBindGroupImpl(
@@ -307,7 +307,6 @@ namespace dawn_native { namespace d3d12 {
         Buffer* dstBuffer = ToBackend(destination);
         StagingBuffer* srcBuffer = ToBackend(source);
         dstBuffer->TrackUsageAndTransitionNow(commandRecordingContext, wgpu::BufferUsage::CopyDst);
-        srcBuffer->TrackUsage(commandRecordingContext);
 
         commandRecordingContext->GetCommandList()->CopyBufferRegion(
             dstBuffer->GetD3D12Resource().Get(), destinationOffset, srcBuffer->GetResource(),

src/dawn_native/d3d12/HeapAllocatorD3D12.cpp

@@ -16,6 +16,7 @@
 #include "dawn_native/d3d12/D3D12Error.h"
 #include "dawn_native/d3d12/DeviceD3D12.h"
 #include "dawn_native/d3d12/HeapD3D12.h"
+#include "dawn_native/d3d12/ResidencyManagerD3D12.h"
 
 namespace dawn_native { namespace d3d12 {
 
@@ -41,12 +42,22 @@ namespace dawn_native { namespace d3d12 {
         heapDesc.Alignment = D3D12_DEFAULT_MSAA_RESOURCE_PLACEMENT_ALIGNMENT;
         heapDesc.Flags = mHeapFlags;
 
-        ComPtr<ID3D12Heap> heap;
+        // CreateHeap will implicitly make the created heap resident. We must ensure enough free
+        // memory exists before allocating to avoid an out-of-memory error when overcommitted.
+        DAWN_TRY(mDevice->GetResidencyManager()->EnsureCanMakeResident(size));
+
+        ComPtr<ID3D12Heap> d3d12Heap;
         DAWN_TRY(CheckOutOfMemoryHRESULT(
-            mDevice->GetD3D12Device()->CreateHeap(&heapDesc, IID_PPV_ARGS(&heap)),
+            mDevice->GetD3D12Device()->CreateHeap(&heapDesc, IID_PPV_ARGS(&d3d12Heap)),
             "ID3D12Device::CreateHeap"));
 
-        return {std::make_unique<Heap>(std::move(heap), size)};
+        std::unique_ptr<ResourceHeapBase> heapBase =
+            std::make_unique<Heap>(std::move(d3d12Heap), heapDesc.Properties.Type, size);
+
+        // Calling CreateHeap implicitly calls MakeResident on the new heap. We must track this to
+        // avoid calling MakeResident a second time.
+        mDevice->GetResidencyManager()->TrackResidentAllocation(ToBackend(heapBase.get()));
+        return heapBase;
     }
 
     void HeapAllocator::DeallocateResourceHeap(std::unique_ptr<ResourceHeapBase> heap) {

src/dawn_native/d3d12/HeapD3D12.cpp

@@ -15,8 +15,17 @@
 #include "dawn_native/d3d12/HeapD3D12.h"
 
 namespace dawn_native { namespace d3d12 {
-    Heap::Heap(ComPtr<ID3D12Pageable> d3d12Pageable, uint64_t size)
+    Heap::Heap(ComPtr<ID3D12Pageable> d3d12Pageable, D3D12_HEAP_TYPE d3d12HeapType, uint64_t size)
-        : mD3d12Pageable(std::move(d3d12Pageable)), mSize(size) {
+        : mD3d12Pageable(std::move(d3d12Pageable)), mD3d12HeapType(d3d12HeapType), mSize(size) {
+    }
+
+    Heap::~Heap() {
+        // When a heap is destroyed, it no longer resides in resident memory, so we must evict it
+        // from the LRU cache. If this heap is not manually removed from the LRU-cache, the
+        // ResidencyManager will attempt to use it after it has been deallocated.
+        if (IsInResidencyLRUCache()) {
+            RemoveFromList();
+        }
     }
 
     // This function should only be used when mD3D12Pageable was initialized from a ID3D12Pageable
@@ -33,6 +42,10 @@ namespace dawn_native { namespace d3d12 {
         return mD3d12Pageable;
     }
 
+    D3D12_HEAP_TYPE Heap::GetD3D12HeapType() const {
+        return mD3d12HeapType;
+    }
+
     Serial Heap::GetLastUsage() const {
         return mLastUsage;
     }
@@ -41,7 +54,38 @@ namespace dawn_native { namespace d3d12 {
         mLastUsage = serial;
     }
 
+    uint64_t Heap::GetLastSubmission() const {
+        return mLastSubmission;
+    }
+
+    void Heap::SetLastSubmission(Serial serial) {
+        mLastSubmission = serial;
+    }
+
     uint64_t Heap::GetSize() const {
         return mSize;
     }
+
+    bool Heap::IsInResidencyLRUCache() const {
+        return IsInList();
+    }
+
+    void Heap::IncrementResidencyLock() {
+        ASSERT(mD3d12HeapType != D3D12_HEAP_TYPE_DEFAULT);
+        mResidencyLockRefCount++;
+    }
+
+    void Heap::DecrementResidencyLock() {
+        ASSERT(mD3d12HeapType != D3D12_HEAP_TYPE_DEFAULT);
+        mResidencyLockRefCount--;
+    }
+
+    bool Heap::IsResidencyLocked() const {
+        if (mResidencyLockRefCount == 0) {
+            return false;
+        }
+
+        return true;
+    }
+
 }}  // namespace dawn_native::d3d12

src/dawn_native/d3d12/HeapD3D12.h

@@ -15,19 +15,26 @@
 #ifndef DAWNNATIVE_D3D12_HEAPD3D12_H_
 #define DAWNNATIVE_D3D12_HEAPD3D12_H_
 
+#include "common/LinkedList.h"
 #include "common/Serial.h"
 #include "dawn_native/ResourceHeap.h"
 #include "dawn_native/d3d12/d3d12_platform.h"
 
 namespace dawn_native { namespace d3d12 {
 
-    class Heap : public ResourceHeapBase {
+    // This class is used to represent heap allocations, but also serves as a node within the
+    // ResidencyManager's LRU cache. This node is inserted into the LRU-cache when it is first
+    // allocated, and any time it is scheduled to be used by the GPU. This node is removed from the
+    // LRU cache when it is evicted from resident memory due to budget constraints, or when the heap
+    // is destroyed.
+    class Heap : public ResourceHeapBase, public LinkNode<Heap> {
       public:
-        Heap(ComPtr<ID3D12Pageable> d3d12Pageable, uint64_t size);
+        Heap(ComPtr<ID3D12Pageable> d3d12Pageable, D3D12_HEAP_TYPE heapType, uint64_t size);
-        ~Heap() = default;
+        ~Heap();
 
         ComPtr<ID3D12Heap> GetD3D12Heap() const;
         ComPtr<ID3D12Pageable> GetD3D12Pageable() const;
+        D3D12_HEAP_TYPE GetD3D12HeapType() const;
 
         // We set mLastRecordingSerial to denote the serial this heap was last recorded to be used.
         // We must check this serial against the current serial when recording heap usages to ensure
@@ -35,11 +42,31 @@ namespace dawn_native { namespace d3d12 {
         Serial GetLastUsage() const;
         void SetLastUsage(Serial serial);
 
+        // The residency manager must know the last serial that any portion of the heap was
+        // submitted to be used so that we can ensure this heap stays resident in memory at least
+        // until that serial has completed.
+        uint64_t GetLastSubmission() const;
+        void SetLastSubmission(Serial serial);
+
         uint64_t GetSize() const;
 
+        bool IsInResidencyLRUCache() const;
+
+        // In some scenarios, such as async buffer mapping, we must lock residency to ensure the
+        // heap cannot be evicted. Because multiple buffers may be mapped in a single heap, we must
+        // track the number of resources currently locked.
+        void IncrementResidencyLock();
+        void DecrementResidencyLock();
+        bool IsResidencyLocked() const;
+
       private:
         ComPtr<ID3D12Pageable> mD3d12Pageable;
+        D3D12_HEAP_TYPE mD3d12HeapType;
+        // mLastUsage denotes the last time this heap was recorded for use.
         Serial mLastUsage = 0;
+        // mLastSubmission denotes the last time this heap was submitted to the GPU.
+        Serial mLastSubmission = 0;
+        uint32_t mResidencyLockRefCount = 0;
         uint64_t mSize = 0;
     };
 }}  // namespace dawn_native::d3d12

src/dawn_native/d3d12/ResidencyManagerD3D12.cpp

@@ -15,8 +15,11 @@
 #include "dawn_native/d3d12/ResidencyManagerD3D12.h"
 
 #include "dawn_native/d3d12/AdapterD3D12.h"
+#include "dawn_native/d3d12/D3D12Error.h"
 #include "dawn_native/d3d12/DeviceD3D12.h"
 #include "dawn_native/d3d12/Forward.h"
+#include "dawn_native/d3d12/HeapD3D12.h"
+
 #include "dawn_native/d3d12/d3d12_platform.h"
 
 namespace dawn_native { namespace d3d12 {
@@ -28,6 +31,59 @@ namespace dawn_native { namespace d3d12 {
         UpdateVideoMemoryInfo();
     }
 
+    // Increments number of locks on a heap to ensure the heap remains resident.
+    MaybeError ResidencyManager::LockMappableHeap(Heap* heap) {
+        if (!mResidencyManagementEnabled) {
+            return {};
+        }
+
+        // Depending on device architecture, the heap may not need tracked.
+        if (!ShouldTrackHeap(heap)) {
+            return {};
+        }
+
+        // If the heap isn't already resident, make it resident.
+        if (!heap->IsInResidencyLRUCache() && !heap->IsResidencyLocked()) {
+            DAWN_TRY(EnsureCanMakeResident(heap->GetSize()));
+            ID3D12Pageable* pageable = heap->GetD3D12Pageable().Get();
+            DAWN_TRY(
+                CheckHRESULT(mDevice->GetD3D12Device()->MakeResident(1, &pageable),
+                             "Making a heap resident due to an underlying resource being mapped."));
+        }
+
+        // Since we can't evict the heap, it's unnecessary to track the heap in the LRU Cache.
+        if (heap->IsInResidencyLRUCache()) {
+            heap->RemoveFromList();
+        }
+
+        heap->IncrementResidencyLock();
+
+        return {};
+    }
+
+    // Decrements number of locks on a heap. When the number of locks becomes zero, the heap is
+    // inserted into the LRU cache and becomes eligible for eviction.
+    void ResidencyManager::UnlockMappableHeap(Heap* heap) {
+        if (!mResidencyManagementEnabled) {
+            return;
+        }
+
+        // Depending on device architecture, the heap may not need tracked.
+        if (!ShouldTrackHeap(heap)) {
+            return;
+        }
+
+        ASSERT(heap->IsResidencyLocked());
+        ASSERT(!heap->IsInResidencyLRUCache());
+        heap->DecrementResidencyLock();
+
+        // When all locks have been removed, the resource remains resident and becomes tracked in
+        // the LRU.
+        if (!heap->IsResidencyLocked()) {
+            mLRUCache.Append(heap);
+        }
+    }
+
     // Allows an application component external to Dawn to cap Dawn's residency budget to prevent
     // competition for device local memory. Returns the amount of memory reserved, which may be less
     // that the requested reservation when under pressure.
@@ -66,4 +122,157 @@ namespace dawn_native { namespace d3d12 {
         mVideoMemoryInfo.dawnUsage =
             queryVideoMemoryInfo.CurrentUsage - mVideoMemoryInfo.externalReservation;
     }
+
+    // Removes from the LRU and returns the least recently used heap when possible. Returns nullptr
+    // when nothing further can be evicted.
+    ResultOrError<Heap*> ResidencyManager::RemoveSingleEntryFromLRU() {
+        ASSERT(!mLRUCache.empty());
+        Heap* heap = mLRUCache.head()->value();
+        Serial lastSubmissionSerial = heap->GetLastSubmission();
+
+        // If the next candidate for eviction was inserted into the LRU during the current serial,
+        // it is because more memory is being used in a single command list than is available.
+        // In this scenario, we cannot make any more resources resident and thrashing must occur.
+        if (lastSubmissionSerial == mDevice->GetPendingCommandSerial()) {
+            return nullptr;
+        }
+
+        // We must ensure that any previous use of a resource has completed before the resource can
+        // be evicted.
+        if (lastSubmissionSerial > mDevice->GetCompletedCommandSerial()) {
+            DAWN_TRY(mDevice->WaitForSerial(lastSubmissionSerial));
+        }
+
+        heap->RemoveFromList();
+        return heap;
+    }
+
+    // Any time we need to make something resident in local memory, we must check that we have
+    // enough free memory to make the new object resident while also staying within our budget.
+    // If there isn't enough memory, we should evict until there is.
+    MaybeError ResidencyManager::EnsureCanMakeResident(uint64_t sizeToMakeResident) {
+        if (!mResidencyManagementEnabled) {
+            return {};
+        }
+
+        UpdateVideoMemoryInfo();
+
+        uint64_t memoryUsageAfterMakeResident = sizeToMakeResident + mVideoMemoryInfo.dawnUsage;
+
+        // Return when we can call MakeResident and remain under budget.
+        if (memoryUsageAfterMakeResident < mVideoMemoryInfo.dawnBudget) {
+            return {};
+        }
+
+        std::vector<ID3D12Pageable*> resourcesToEvict;
+
+        uint64_t sizeEvicted = 0;
+        while (sizeEvicted < sizeToMakeResident) {
+            Heap* heap;
+            DAWN_TRY_ASSIGN(heap, RemoveSingleEntryFromLRU());
+
+            // If no heap was returned, then nothing more can be evicted.
+            if (heap == nullptr) {
+                break;
+            }
+
+            sizeEvicted += heap->GetSize();
+            resourcesToEvict.push_back(heap->GetD3D12Pageable().Get());
+        }
+
+        if (resourcesToEvict.size() > 0) {
+            DAWN_TRY(CheckHRESULT(
+                mDevice->GetD3D12Device()->Evict(resourcesToEvict.size(), resourcesToEvict.data()),
+                "Evicting resident heaps to free device local memory"));
+        }
+
+        return {};
+    }
+
+    // Ensure that we are only tracking heaps that exist in DXGI_MEMORY_SEGMENT_LOCAL.
+    bool ResidencyManager::ShouldTrackHeap(Heap* heap) const {
+        D3D12_HEAP_PROPERTIES heapProperties =
+            mDevice->GetD3D12Device()->GetCustomHeapProperties(0, heap->GetD3D12HeapType());
+
+        if (mDevice->GetDeviceInfo().isUMA) {
+            // On UMA devices, MEMORY_POOL_L0 corresponds to MEMORY_SEGMENT_LOCAL, so we must track
+            // heaps in MEMORY_POOL_L0. For UMA, all heaps types exist in MEMORY_POOL_L0.
+            return heapProperties.MemoryPoolPreference == D3D12_MEMORY_POOL_L0;
+        }
+
+        // On non-UMA devices, MEMORY_POOL_L1 corresponds to MEMORY_SEGMENT_LOCAL, so only track the
+        // heap if it is in MEMORY_POOL_L1. For non-UMA, DEFAULT heaps exist in MEMORY_POOL_L1,
+        // while READBACK and UPLOAD heaps exist in MEMORY_POOL_L0.
+        return heapProperties.MemoryPoolPreference == D3D12_MEMORY_POOL_L1;
+    }
+
+    // Given a list of heaps that are pending usage, this function will estimate memory needed,
+    // evict resources until enough space is available, then make resident any heaps scheduled for
+    // usage.
+    MaybeError ResidencyManager::EnsureHeapsAreResident(Heap** heaps, size_t heapCount) {
+        if (!mResidencyManagementEnabled) {
+            return {};
+        }
+
+        std::vector<ID3D12Pageable*> heapsToMakeResident;
+        uint64_t sizeToMakeResident = 0;
+
+        Serial pendingCommandSerial = mDevice->GetPendingCommandSerial();
+        for (size_t i = 0; i < heapCount; i++) {
+            Heap* heap = heaps[i];
+
+            // Depending on device architecture, the heap may not need tracked.
+            if (!ShouldTrackHeap(heap)) {
+                continue;
+            }
+
+            // Heaps that are locked resident are not tracked in the LRU cache.
+            if (heap->IsResidencyLocked()) {
+                continue;
+            }
+
+            if (heap->IsInResidencyLRUCache()) {
+                // If the heap is already in the LRU, we must remove it and append again below to
+                // update its position in the LRU.
+                heap->RemoveFromList();
+            } else {
+                heapsToMakeResident.push_back(heap->GetD3D12Pageable().Get());
+                sizeToMakeResident += heap->GetSize();
+            }
+
+            mLRUCache.Append(heap);
+            heap->SetLastSubmission(pendingCommandSerial);
+        }
+
+        if (heapsToMakeResident.size() != 0) {
+            DAWN_TRY(EnsureCanMakeResident(sizeToMakeResident));
+
+            // Note that MakeResident is a synchronous function and can add a significant
+            // overhead to command recording. In the future, it may be possible to decrease this
+            // overhead by using MakeResident on a secondary thread, or by instead making use of
+            // the EnqueueMakeResident function (which is not available on all Windows 10
+            // platforms).
+            DAWN_TRY(CheckHRESULT(mDevice->GetD3D12Device()->MakeResident(
+                                      heapsToMakeResident.size(), heapsToMakeResident.data()),
+                                  "Making scheduled-to-be-used resources resident in "
+                                  "device local memory"));
+        }
+
+        return {};
+    }
+
+    // When a new heap is allocated, the heap will be made resident upon creation. We must track
+    // when this happens to avoid calling MakeResident a second time.
+    void ResidencyManager::TrackResidentAllocation(Heap* heap) {
+        if (!mResidencyManagementEnabled) {
+            return;
+        }
+
+        // Depending on device architecture and heap type, the heap may not need tracked.
+        if (!ShouldTrackHeap(heap)) {
+            return;
+        }
+
+        mLRUCache.Append(heap);
+    }
 }}  // namespace dawn_native::d3d12

src/dawn_native/d3d12/ResidencyManagerD3D12.h

@@ -15,17 +15,29 @@
 #ifndef DAWNNATIVE_D3D12_RESIDENCYMANAGERD3D12_H_
 #define DAWNNATIVE_D3D12_RESIDENCYMANAGERD3D12_H_
 
+#include "common/LinkedList.h"
+#include "common/Serial.h"
+#include "dawn_native/Error.h"
 #include "dawn_native/dawn_platform.h"
 
 namespace dawn_native { namespace d3d12 {
 
     class Device;
+    class Heap;
 
     class ResidencyManager {
       public:
         ResidencyManager(Device* device);
+
+        MaybeError LockMappableHeap(Heap* heap);
+        void UnlockMappableHeap(Heap* heap);
+        MaybeError EnsureCanMakeResident(uint64_t allocationSize);
+        MaybeError EnsureHeapsAreResident(Heap** heaps, size_t heapCount);
+
         uint64_t SetExternalMemoryReservation(uint64_t requestedReservationSize);
 
+        void TrackResidentAllocation(Heap* heap);
+
       private:
         struct VideoMemoryInfo {
             uint64_t dawnBudget;
@@ -33,9 +45,12 @@ namespace dawn_native { namespace d3d12 {
             uint64_t externalReservation;
             uint64_t externalRequest;
         };
+        ResultOrError<Heap*> RemoveSingleEntryFromLRU();
+        bool ShouldTrackHeap(Heap* heap) const;
         void UpdateVideoMemoryInfo();
 
-        Device* mDevice = nullptr;
+        Device* mDevice;
+        LinkedList<Heap> mLRUCache;
         bool mResidencyManagementEnabled = false;
         VideoMemoryInfo mVideoMemoryInfo = {};
     };

src/dawn_native/d3d12/ResourceAllocatorManagerD3D12.cpp

@@ -18,6 +18,7 @@
 #include "dawn_native/d3d12/DeviceD3D12.h"
 #include "dawn_native/d3d12/HeapAllocatorD3D12.h"
 #include "dawn_native/d3d12/HeapD3D12.h"
+#include "dawn_native/d3d12/ResidencyManagerD3D12.h"
 
 namespace dawn_native { namespace d3d12 {
     namespace {
@@ -255,6 +256,10 @@ namespace dawn_native { namespace d3d12 {
 
         Heap* heap = ToBackend(allocation.GetResourceHeap());
 
+        // Before calling CreatePlacedResource, we must ensure the target heap is resident.
+        // CreatePlacedResource will fail if it is not.
+        DAWN_TRY(mDevice->GetResidencyManager()->EnsureHeapsAreResident(&heap, 1));
+
         // With placed resources, a single heap can be reused.
         // The resource placed at an offset is only reclaimed
         // upon Tick or after the last command list using the resource has completed
@@ -296,6 +301,11 @@ namespace dawn_native { namespace d3d12 {
             return ResourceHeapAllocation{};  // Invalid
         }
 
+        // CreateCommittedResource will implicitly make the created resource resident. We must
+        // ensure enough free memory exists before allocating to avoid an out-of-memory error when
+        // overcommitted.
+        DAWN_TRY(mDevice->GetResidencyManager()->EnsureCanMakeResident(resourceInfo.SizeInBytes));
+
         // Note: Heap flags are inferred by the resource descriptor and do not need to be explicitly
         // provided to CreateCommittedResource.
         ComPtr<ID3D12Resource> committedResource;
@@ -310,7 +320,11 @@ namespace dawn_native { namespace d3d12 {
         // heap granularity, every directly allocated ResourceHeapAllocation also stores a Heap
         // object. This object is created manually, and must be deleted manually upon deallocation
         // of the committed resource.
-        Heap* heap = new Heap(committedResource, resourceInfo.SizeInBytes);
+        Heap* heap = new Heap(committedResource, heapType, resourceInfo.SizeInBytes);
+
+        // Calling CreateCommittedResource implicitly calls MakeResident on the resource. We must
+        // track this to avoid calling MakeResident a second time.
+        mDevice->GetResidencyManager()->TrackResidentAllocation(heap);
 
         AllocationInfo info;
         info.mMethod = AllocationMethod::kDirect;

src/dawn_native/d3d12/StagingBufferD3D12.cpp

@@ -16,6 +16,7 @@
 #include "dawn_native/d3d12/D3D12Error.h"
 #include "dawn_native/d3d12/DeviceD3D12.h"
 #include "dawn_native/d3d12/HeapD3D12.h"
+#include "dawn_native/d3d12/ResidencyManagerD3D12.h"
 
 namespace dawn_native { namespace d3d12 {
 
@@ -41,6 +42,11 @@ namespace dawn_native { namespace d3d12 {
                         mDevice->AllocateMemory(D3D12_HEAP_TYPE_UPLOAD, resourceDescriptor,
                                                 D3D12_RESOURCE_STATE_GENERIC_READ));
 
+        // The mapped buffer can be accessed at any time, so it must be locked to ensure it is never
+        // evicted. This buffer should already have been made resident when it was created.
+        DAWN_TRY(mDevice->GetResidencyManager()->LockMappableHeap(
+            ToBackend(mUploadHeap.GetResourceHeap())));
+
         return CheckHRESULT(GetResource()->Map(0, nullptr, &mMappedPointer), "ID3D12Resource::Map");
     }
 
@@ -50,6 +56,12 @@ namespace dawn_native { namespace d3d12 {
         if (mUploadHeap.GetInfo().mMethod == AllocationMethod::kInvalid) {
             return;
         }
+
+        // The underlying heap was locked in residency upon creation. We must unlock it when this
+        // buffer becomes unmapped.
+        mDevice->GetResidencyManager()->UnlockMappableHeap(
+            ToBackend(mUploadHeap.GetResourceHeap()));
+
         // Invalidate the CPU virtual address & flush cache (if needed).
         GetResource()->Unmap(0, nullptr);
         mMappedPointer = nullptr;
@@ -60,10 +72,4 @@ namespace dawn_native { namespace d3d12 {
     ID3D12Resource* StagingBuffer::GetResource() const {
         return mUploadHeap.GetD3D12Resource().Get();
     }
-
-    void StagingBuffer::TrackUsage(CommandRecordingContext* commandContext) {
-        // Track the underlying heap to ensure residency.
-        Heap* heap = ToBackend(mUploadHeap.GetResourceHeap());
-        commandContext->TrackHeapUsage(heap, mDevice->GetPendingCommandSerial());
-    }
 }}  // namespace dawn_native::d3d12

src/dawn_native/d3d12/StagingBufferD3D12.h

@@ -33,8 +33,6 @@ namespace dawn_native { namespace d3d12 {
 
         MaybeError Initialize() override;
 
-        void TrackUsage(CommandRecordingContext* commandContext);
-
       private:
         Device* mDevice;
         ResourceHeapAllocation mUploadHeap;

src/tests/unittests/LinkedListTests.cpp

@@ -351,3 +351,15 @@ TEST(LinkedList, NodeMoveConstructor) {
     EXPECT_EQ(&n2_new, n3.previous());
     EXPECT_EQ(2, n2_new.id());
 }
+
+TEST(LinkedList, IsInList) {
+    LinkedList<Node> list;
+
+    Node n(1);
+
+    EXPECT_FALSE(n.IsInList());
+    list.Append(&n);
+    EXPECT_TRUE(n.IsInList());
+    n.RemoveFromList();
+    EXPECT_FALSE(n.IsInList());
+}