From 7982cc0527630f123bd844cf5bb97ad0c3e0041c Mon Sep 17 00:00:00 2001 From: Brandon Jones Date: Tue, 31 Mar 2020 15:31:56 +0000 Subject: [PATCH] Residency 5: Implement and Integrate Residency Management Adds all D3D12 residency management logic. Bug: dawn:193 Change-Id: Ibc160289c29cc85817dcfaaef1b92d04599aa802 Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/16384 Reviewed-by: Austin Eng Reviewed-by: Corentin Wallez Commit-Queue: Brandon Jones --- src/common/LinkedList.h | 8 + src/dawn_native/Buffer.cpp | 4 + src/dawn_native/Buffer.h | 2 + src/dawn_native/d3d12/BufferD3D12.cpp | 27 +++ .../d3d12/CommandRecordingContext.cpp | 8 +- .../d3d12/CommandRecordingContext.h | 2 +- src/dawn_native/d3d12/DeviceD3D12.cpp | 3 +- src/dawn_native/d3d12/HeapAllocatorD3D12.cpp | 17 +- src/dawn_native/d3d12/HeapD3D12.cpp | 48 +++- src/dawn_native/d3d12/HeapD3D12.h | 33 ++- .../d3d12/ResidencyManagerD3D12.cpp | 209 ++++++++++++++++++ src/dawn_native/d3d12/ResidencyManagerD3D12.h | 17 +- .../d3d12/ResourceAllocatorManagerD3D12.cpp | 16 +- src/dawn_native/d3d12/StagingBufferD3D12.cpp | 18 +- src/dawn_native/d3d12/StagingBufferD3D12.h | 2 - src/tests/unittests/LinkedListTests.cpp | 12 + 16 files changed, 403 insertions(+), 23 deletions(-) diff --git a/src/common/LinkedList.h b/src/common/LinkedList.h index c67c986458..7c0a413966 100644 --- a/src/common/LinkedList.h +++ b/src/common/LinkedList.h @@ -7,6 +7,8 @@ #ifndef COMMON_LINKED_LIST_H #define COMMON_LINKED_LIST_H +#include "common/Assert.h" + // Simple LinkedList type. (See the Q&A section to understand how this // differs from std::list). // @@ -117,6 +119,12 @@ class LinkNode { e->next_ = this; } + // Check if |this| is in a list. + bool IsInList() const { + ASSERT((this->previous_ == nullptr) == (this->next_ == nullptr)); + return this->next_ != nullptr; + } + // Remove |this| from the linked list. void RemoveFromList() { this->previous_->next_ = this->next_; diff --git a/src/dawn_native/Buffer.cpp b/src/dawn_native/Buffer.cpp index 5e173f375a..6502475b16 100644 --- a/src/dawn_native/Buffer.cpp +++ b/src/dawn_native/Buffer.cpp @@ -463,4 +463,8 @@ namespace dawn_native { mState = BufferState::Destroyed; } + bool BufferBase::IsMapped() const { + return mState == BufferState::Mapped; + } + } // namespace dawn_native diff --git a/src/dawn_native/Buffer.h b/src/dawn_native/Buffer.h index 6a0a0e1be8..387ef5c371 100644 --- a/src/dawn_native/Buffer.h +++ b/src/dawn_native/Buffer.h @@ -82,6 +82,8 @@ namespace dawn_native { void DestroyInternal(); + bool IsMapped() const; + private: virtual MaybeError MapAtCreationImpl(uint8_t** mappedPointer) = 0; virtual MaybeError SetSubDataImpl(uint32_t start, uint32_t count, const void* data); diff --git a/src/dawn_native/d3d12/BufferD3D12.cpp b/src/dawn_native/d3d12/BufferD3D12.cpp index 32365daef4..4fb6f20a69 100644 --- a/src/dawn_native/d3d12/BufferD3D12.cpp +++ b/src/dawn_native/d3d12/BufferD3D12.cpp @@ -21,6 +21,7 @@ #include "dawn_native/d3d12/D3D12Error.h" #include "dawn_native/d3d12/DeviceD3D12.h" #include "dawn_native/d3d12/HeapD3D12.h" +#include "dawn_native/d3d12/ResidencyManagerD3D12.h" namespace dawn_native { namespace d3d12 { @@ -244,6 +245,11 @@ namespace dawn_native { namespace d3d12 { } MaybeError Buffer::MapAtCreationImpl(uint8_t** mappedPointer) { + // The mapped buffer can be accessed at any time, so it must be locked to ensure it is never + // evicted. This buffer should already have been made resident when it was created. + Heap* heap = ToBackend(mResourceAllocation.GetResourceHeap()); + DAWN_TRY(ToBackend(GetDevice())->GetResidencyManager()->LockMappableHeap(heap)); + mWrittenMappedRange = {0, static_cast(GetSize())}; DAWN_TRY(CheckHRESULT(GetD3D12Resource()->Map(0, &mWrittenMappedRange, reinterpret_cast(mappedPointer)), @@ -252,6 +258,11 @@ namespace dawn_native { namespace d3d12 { } MaybeError Buffer::MapReadAsyncImpl(uint32_t serial) { + // The mapped buffer can be accessed at any time, so we must make the buffer resident and + // lock it to ensure it is never evicted. + Heap* heap = ToBackend(mResourceAllocation.GetResourceHeap()); + DAWN_TRY(ToBackend(GetDevice())->GetResidencyManager()->LockMappableHeap(heap)); + mWrittenMappedRange = {}; D3D12_RANGE readRange = {0, static_cast(GetSize())}; char* data = nullptr; @@ -266,6 +277,11 @@ namespace dawn_native { namespace d3d12 { } MaybeError Buffer::MapWriteAsyncImpl(uint32_t serial) { + // The mapped buffer can be accessed at any time, so we must make the buffer resident and + // lock it to ensure it is never evicted. + Heap* heap = ToBackend(mResourceAllocation.GetResourceHeap()); + DAWN_TRY(ToBackend(GetDevice())->GetResidencyManager()->LockMappableHeap(heap)); + mWrittenMappedRange = {0, static_cast(GetSize())}; char* data = nullptr; DAWN_TRY(CheckHRESULT( @@ -280,10 +296,21 @@ namespace dawn_native { namespace d3d12 { void Buffer::UnmapImpl() { GetD3D12Resource()->Unmap(0, &mWrittenMappedRange); + // When buffers are mapped, they are locked to keep them in resident memory. We must unlock + // them when they are unmapped. + Heap* heap = ToBackend(mResourceAllocation.GetResourceHeap()); + ToBackend(GetDevice())->GetResidencyManager()->UnlockMappableHeap(heap); mWrittenMappedRange = {}; } void Buffer::DestroyImpl() { + // We must ensure that if a mapped buffer is destroyed, it does not leave a dangling lock + // reference on its heap. + if (IsMapped()) { + Heap* heap = ToBackend(mResourceAllocation.GetResourceHeap()); + ToBackend(GetDevice())->GetResidencyManager()->UnlockMappableHeap(heap); + } + ToBackend(GetDevice())->DeallocateMemory(mResourceAllocation); } diff --git a/src/dawn_native/d3d12/CommandRecordingContext.cpp b/src/dawn_native/d3d12/CommandRecordingContext.cpp index bd18782e37..81dad3c138 100644 --- a/src/dawn_native/d3d12/CommandRecordingContext.cpp +++ b/src/dawn_native/d3d12/CommandRecordingContext.cpp @@ -14,7 +14,9 @@ #include "dawn_native/d3d12/CommandRecordingContext.h" #include "dawn_native/d3d12/CommandAllocatorManager.h" #include "dawn_native/d3d12/D3D12Error.h" +#include "dawn_native/d3d12/DeviceD3D12.h" #include "dawn_native/d3d12/HeapD3D12.h" +#include "dawn_native/d3d12/ResidencyManagerD3D12.h" namespace dawn_native { namespace d3d12 { @@ -52,7 +54,7 @@ namespace dawn_native { namespace d3d12 { return {}; } - MaybeError CommandRecordingContext::ExecuteCommandList(ID3D12CommandQueue* d3d12CommandQueue) { + MaybeError CommandRecordingContext::ExecuteCommandList(Device* device) { if (IsOpen()) { // Shared textures must be transitioned to common state after the last usage in order // for them to be used by other APIs like D3D11. We ensure this by transitioning to the @@ -68,9 +70,11 @@ namespace dawn_native { namespace d3d12 { Release(); DAWN_TRY(std::move(error)); } + DAWN_TRY(device->GetResidencyManager()->EnsureHeapsAreResident( + mHeapsPendingUsage.data(), mHeapsPendingUsage.size())); ID3D12CommandList* d3d12CommandList = GetCommandList(); - d3d12CommandQueue->ExecuteCommandLists(1, &d3d12CommandList); + device->GetCommandQueue()->ExecuteCommandLists(1, &d3d12CommandList); mIsOpen = false; mSharedTextures.clear(); diff --git a/src/dawn_native/d3d12/CommandRecordingContext.h b/src/dawn_native/d3d12/CommandRecordingContext.h index 0414abd7c2..932fa4d7bf 100644 --- a/src/dawn_native/d3d12/CommandRecordingContext.h +++ b/src/dawn_native/d3d12/CommandRecordingContext.h @@ -35,7 +35,7 @@ namespace dawn_native { namespace d3d12 { void Release(); bool IsOpen() const; - MaybeError ExecuteCommandList(ID3D12CommandQueue* d3d12CommandQueue); + MaybeError ExecuteCommandList(Device* device); void TrackHeapUsage(Heap* heap, Serial serial); diff --git a/src/dawn_native/d3d12/DeviceD3D12.cpp b/src/dawn_native/d3d12/DeviceD3D12.cpp index 7dcf8c807d..f0e8b46919 100644 --- a/src/dawn_native/d3d12/DeviceD3D12.cpp +++ b/src/dawn_native/d3d12/DeviceD3D12.cpp @@ -228,7 +228,7 @@ namespace dawn_native { namespace d3d12 { } MaybeError Device::ExecutePendingCommandContext() { - return mPendingCommands.ExecuteCommandList(mCommandQueue.Get()); + return mPendingCommands.ExecuteCommandList(this); } ResultOrError Device::CreateBindGroupImpl( @@ -307,7 +307,6 @@ namespace dawn_native { namespace d3d12 { Buffer* dstBuffer = ToBackend(destination); StagingBuffer* srcBuffer = ToBackend(source); dstBuffer->TrackUsageAndTransitionNow(commandRecordingContext, wgpu::BufferUsage::CopyDst); - srcBuffer->TrackUsage(commandRecordingContext); commandRecordingContext->GetCommandList()->CopyBufferRegion( dstBuffer->GetD3D12Resource().Get(), destinationOffset, srcBuffer->GetResource(), diff --git a/src/dawn_native/d3d12/HeapAllocatorD3D12.cpp b/src/dawn_native/d3d12/HeapAllocatorD3D12.cpp index dd6c641545..ced2dd1c5b 100644 --- a/src/dawn_native/d3d12/HeapAllocatorD3D12.cpp +++ b/src/dawn_native/d3d12/HeapAllocatorD3D12.cpp @@ -16,6 +16,7 @@ #include "dawn_native/d3d12/D3D12Error.h" #include "dawn_native/d3d12/DeviceD3D12.h" #include "dawn_native/d3d12/HeapD3D12.h" +#include "dawn_native/d3d12/ResidencyManagerD3D12.h" namespace dawn_native { namespace d3d12 { @@ -41,12 +42,22 @@ namespace dawn_native { namespace d3d12 { heapDesc.Alignment = D3D12_DEFAULT_MSAA_RESOURCE_PLACEMENT_ALIGNMENT; heapDesc.Flags = mHeapFlags; - ComPtr heap; + // CreateHeap will implicitly make the created heap resident. We must ensure enough free + // memory exists before allocating to avoid an out-of-memory error when overcommitted. + DAWN_TRY(mDevice->GetResidencyManager()->EnsureCanMakeResident(size)); + + ComPtr d3d12Heap; DAWN_TRY(CheckOutOfMemoryHRESULT( - mDevice->GetD3D12Device()->CreateHeap(&heapDesc, IID_PPV_ARGS(&heap)), + mDevice->GetD3D12Device()->CreateHeap(&heapDesc, IID_PPV_ARGS(&d3d12Heap)), "ID3D12Device::CreateHeap")); - return {std::make_unique(std::move(heap), size)}; + std::unique_ptr heapBase = + std::make_unique(std::move(d3d12Heap), heapDesc.Properties.Type, size); + + // Calling CreateHeap implicitly calls MakeResident on the new heap. We must track this to + // avoid calling MakeResident a second time. + mDevice->GetResidencyManager()->TrackResidentAllocation(ToBackend(heapBase.get())); + return heapBase; } void HeapAllocator::DeallocateResourceHeap(std::unique_ptr heap) { diff --git a/src/dawn_native/d3d12/HeapD3D12.cpp b/src/dawn_native/d3d12/HeapD3D12.cpp index f8c7e2adcc..112702b87f 100644 --- a/src/dawn_native/d3d12/HeapD3D12.cpp +++ b/src/dawn_native/d3d12/HeapD3D12.cpp @@ -15,8 +15,17 @@ #include "dawn_native/d3d12/HeapD3D12.h" namespace dawn_native { namespace d3d12 { - Heap::Heap(ComPtr d3d12Pageable, uint64_t size) - : mD3d12Pageable(std::move(d3d12Pageable)), mSize(size) { + Heap::Heap(ComPtr d3d12Pageable, D3D12_HEAP_TYPE d3d12HeapType, uint64_t size) + : mD3d12Pageable(std::move(d3d12Pageable)), mD3d12HeapType(d3d12HeapType), mSize(size) { + } + + Heap::~Heap() { + // When a heap is destroyed, it no longer resides in resident memory, so we must evict it + // from the LRU cache. If this heap is not manually removed from the LRU-cache, the + // ResidencyManager will attempt to use it after it has been deallocated. + if (IsInResidencyLRUCache()) { + RemoveFromList(); + } } // This function should only be used when mD3D12Pageable was initialized from a ID3D12Pageable @@ -33,6 +42,10 @@ namespace dawn_native { namespace d3d12 { return mD3d12Pageable; } + D3D12_HEAP_TYPE Heap::GetD3D12HeapType() const { + return mD3d12HeapType; + } + Serial Heap::GetLastUsage() const { return mLastUsage; } @@ -41,7 +54,38 @@ namespace dawn_native { namespace d3d12 { mLastUsage = serial; } + uint64_t Heap::GetLastSubmission() const { + return mLastSubmission; + } + + void Heap::SetLastSubmission(Serial serial) { + mLastSubmission = serial; + } + uint64_t Heap::GetSize() const { return mSize; } + + bool Heap::IsInResidencyLRUCache() const { + return IsInList(); + } + + void Heap::IncrementResidencyLock() { + ASSERT(mD3d12HeapType != D3D12_HEAP_TYPE_DEFAULT); + mResidencyLockRefCount++; + } + + void Heap::DecrementResidencyLock() { + ASSERT(mD3d12HeapType != D3D12_HEAP_TYPE_DEFAULT); + mResidencyLockRefCount--; + } + + bool Heap::IsResidencyLocked() const { + if (mResidencyLockRefCount == 0) { + return false; + } + + return true; + } + }} // namespace dawn_native::d3d12 \ No newline at end of file diff --git a/src/dawn_native/d3d12/HeapD3D12.h b/src/dawn_native/d3d12/HeapD3D12.h index 05320f6088..de1e205907 100644 --- a/src/dawn_native/d3d12/HeapD3D12.h +++ b/src/dawn_native/d3d12/HeapD3D12.h @@ -15,19 +15,26 @@ #ifndef DAWNNATIVE_D3D12_HEAPD3D12_H_ #define DAWNNATIVE_D3D12_HEAPD3D12_H_ +#include "common/LinkedList.h" #include "common/Serial.h" #include "dawn_native/ResourceHeap.h" #include "dawn_native/d3d12/d3d12_platform.h" namespace dawn_native { namespace d3d12 { - class Heap : public ResourceHeapBase { + // This class is used to represent heap allocations, but also serves as a node within the + // ResidencyManager's LRU cache. This node is inserted into the LRU-cache when it is first + // allocated, and any time it is scheduled to be used by the GPU. This node is removed from the + // LRU cache when it is evicted from resident memory due to budget constraints, or when the heap + // is destroyed. + class Heap : public ResourceHeapBase, public LinkNode { public: - Heap(ComPtr d3d12Pageable, uint64_t size); - ~Heap() = default; + Heap(ComPtr d3d12Pageable, D3D12_HEAP_TYPE heapType, uint64_t size); + ~Heap(); ComPtr GetD3D12Heap() const; ComPtr GetD3D12Pageable() const; + D3D12_HEAP_TYPE GetD3D12HeapType() const; // We set mLastRecordingSerial to denote the serial this heap was last recorded to be used. // We must check this serial against the current serial when recording heap usages to ensure @@ -35,11 +42,31 @@ namespace dawn_native { namespace d3d12 { Serial GetLastUsage() const; void SetLastUsage(Serial serial); + // The residency manager must know the last serial that any portion of the heap was + // submitted to be used so that we can ensure this heap stays resident in memory at least + // until that serial has completed. + uint64_t GetLastSubmission() const; + void SetLastSubmission(Serial serial); + uint64_t GetSize() const; + bool IsInResidencyLRUCache() const; + + // In some scenarios, such as async buffer mapping, we must lock residency to ensure the + // heap cannot be evicted. Because multiple buffers may be mapped in a single heap, we must + // track the number of resources currently locked. + void IncrementResidencyLock(); + void DecrementResidencyLock(); + bool IsResidencyLocked() const; + private: ComPtr mD3d12Pageable; + D3D12_HEAP_TYPE mD3d12HeapType; + // mLastUsage denotes the last time this heap was recorded for use. Serial mLastUsage = 0; + // mLastSubmission denotes the last time this heap was submitted to the GPU. + Serial mLastSubmission = 0; + uint32_t mResidencyLockRefCount = 0; uint64_t mSize = 0; }; }} // namespace dawn_native::d3d12 diff --git a/src/dawn_native/d3d12/ResidencyManagerD3D12.cpp b/src/dawn_native/d3d12/ResidencyManagerD3D12.cpp index c1057482be..3970fdc589 100644 --- a/src/dawn_native/d3d12/ResidencyManagerD3D12.cpp +++ b/src/dawn_native/d3d12/ResidencyManagerD3D12.cpp @@ -15,8 +15,11 @@ #include "dawn_native/d3d12/ResidencyManagerD3D12.h" #include "dawn_native/d3d12/AdapterD3D12.h" +#include "dawn_native/d3d12/D3D12Error.h" #include "dawn_native/d3d12/DeviceD3D12.h" #include "dawn_native/d3d12/Forward.h" +#include "dawn_native/d3d12/HeapD3D12.h" + #include "dawn_native/d3d12/d3d12_platform.h" namespace dawn_native { namespace d3d12 { @@ -28,6 +31,59 @@ namespace dawn_native { namespace d3d12 { UpdateVideoMemoryInfo(); } + // Increments number of locks on a heap to ensure the heap remains resident. + MaybeError ResidencyManager::LockMappableHeap(Heap* heap) { + if (!mResidencyManagementEnabled) { + return {}; + } + + // Depending on device architecture, the heap may not need tracked. + if (!ShouldTrackHeap(heap)) { + return {}; + } + + // If the heap isn't already resident, make it resident. + if (!heap->IsInResidencyLRUCache() && !heap->IsResidencyLocked()) { + DAWN_TRY(EnsureCanMakeResident(heap->GetSize())); + ID3D12Pageable* pageable = heap->GetD3D12Pageable().Get(); + DAWN_TRY( + CheckHRESULT(mDevice->GetD3D12Device()->MakeResident(1, &pageable), + "Making a heap resident due to an underlying resource being mapped.")); + } + + // Since we can't evict the heap, it's unnecessary to track the heap in the LRU Cache. + if (heap->IsInResidencyLRUCache()) { + heap->RemoveFromList(); + } + + heap->IncrementResidencyLock(); + + return {}; + } + + // Decrements number of locks on a heap. When the number of locks becomes zero, the heap is + // inserted into the LRU cache and becomes eligible for eviction. + void ResidencyManager::UnlockMappableHeap(Heap* heap) { + if (!mResidencyManagementEnabled) { + return; + } + + // Depending on device architecture, the heap may not need tracked. + if (!ShouldTrackHeap(heap)) { + return; + } + + ASSERT(heap->IsResidencyLocked()); + ASSERT(!heap->IsInResidencyLRUCache()); + heap->DecrementResidencyLock(); + + // When all locks have been removed, the resource remains resident and becomes tracked in + // the LRU. + if (!heap->IsResidencyLocked()) { + mLRUCache.Append(heap); + } + } + // Allows an application component external to Dawn to cap Dawn's residency budget to prevent // competition for device local memory. Returns the amount of memory reserved, which may be less // that the requested reservation when under pressure. @@ -66,4 +122,157 @@ namespace dawn_native { namespace d3d12 { mVideoMemoryInfo.dawnUsage = queryVideoMemoryInfo.CurrentUsage - mVideoMemoryInfo.externalReservation; } + + // Removes from the LRU and returns the least recently used heap when possible. Returns nullptr + // when nothing further can be evicted. + ResultOrError ResidencyManager::RemoveSingleEntryFromLRU() { + ASSERT(!mLRUCache.empty()); + Heap* heap = mLRUCache.head()->value(); + Serial lastSubmissionSerial = heap->GetLastSubmission(); + + // If the next candidate for eviction was inserted into the LRU during the current serial, + // it is because more memory is being used in a single command list than is available. + // In this scenario, we cannot make any more resources resident and thrashing must occur. + if (lastSubmissionSerial == mDevice->GetPendingCommandSerial()) { + return nullptr; + } + + // We must ensure that any previous use of a resource has completed before the resource can + // be evicted. + if (lastSubmissionSerial > mDevice->GetCompletedCommandSerial()) { + DAWN_TRY(mDevice->WaitForSerial(lastSubmissionSerial)); + } + + heap->RemoveFromList(); + return heap; + } + + // Any time we need to make something resident in local memory, we must check that we have + // enough free memory to make the new object resident while also staying within our budget. + // If there isn't enough memory, we should evict until there is. + MaybeError ResidencyManager::EnsureCanMakeResident(uint64_t sizeToMakeResident) { + if (!mResidencyManagementEnabled) { + return {}; + } + + UpdateVideoMemoryInfo(); + + uint64_t memoryUsageAfterMakeResident = sizeToMakeResident + mVideoMemoryInfo.dawnUsage; + + // Return when we can call MakeResident and remain under budget. + if (memoryUsageAfterMakeResident < mVideoMemoryInfo.dawnBudget) { + return {}; + } + + std::vector resourcesToEvict; + + uint64_t sizeEvicted = 0; + while (sizeEvicted < sizeToMakeResident) { + Heap* heap; + DAWN_TRY_ASSIGN(heap, RemoveSingleEntryFromLRU()); + + // If no heap was returned, then nothing more can be evicted. + if (heap == nullptr) { + break; + } + + sizeEvicted += heap->GetSize(); + resourcesToEvict.push_back(heap->GetD3D12Pageable().Get()); + } + + if (resourcesToEvict.size() > 0) { + DAWN_TRY(CheckHRESULT( + mDevice->GetD3D12Device()->Evict(resourcesToEvict.size(), resourcesToEvict.data()), + "Evicting resident heaps to free device local memory")); + } + + return {}; + } + + // Ensure that we are only tracking heaps that exist in DXGI_MEMORY_SEGMENT_LOCAL. + bool ResidencyManager::ShouldTrackHeap(Heap* heap) const { + D3D12_HEAP_PROPERTIES heapProperties = + mDevice->GetD3D12Device()->GetCustomHeapProperties(0, heap->GetD3D12HeapType()); + + if (mDevice->GetDeviceInfo().isUMA) { + // On UMA devices, MEMORY_POOL_L0 corresponds to MEMORY_SEGMENT_LOCAL, so we must track + // heaps in MEMORY_POOL_L0. For UMA, all heaps types exist in MEMORY_POOL_L0. + return heapProperties.MemoryPoolPreference == D3D12_MEMORY_POOL_L0; + } + + // On non-UMA devices, MEMORY_POOL_L1 corresponds to MEMORY_SEGMENT_LOCAL, so only track the + // heap if it is in MEMORY_POOL_L1. For non-UMA, DEFAULT heaps exist in MEMORY_POOL_L1, + // while READBACK and UPLOAD heaps exist in MEMORY_POOL_L0. + return heapProperties.MemoryPoolPreference == D3D12_MEMORY_POOL_L1; + } + + // Given a list of heaps that are pending usage, this function will estimate memory needed, + // evict resources until enough space is available, then make resident any heaps scheduled for + // usage. + MaybeError ResidencyManager::EnsureHeapsAreResident(Heap** heaps, size_t heapCount) { + if (!mResidencyManagementEnabled) { + return {}; + } + + std::vector heapsToMakeResident; + uint64_t sizeToMakeResident = 0; + + Serial pendingCommandSerial = mDevice->GetPendingCommandSerial(); + for (size_t i = 0; i < heapCount; i++) { + Heap* heap = heaps[i]; + + // Depending on device architecture, the heap may not need tracked. + if (!ShouldTrackHeap(heap)) { + continue; + } + + // Heaps that are locked resident are not tracked in the LRU cache. + if (heap->IsResidencyLocked()) { + continue; + } + + if (heap->IsInResidencyLRUCache()) { + // If the heap is already in the LRU, we must remove it and append again below to + // update its position in the LRU. + heap->RemoveFromList(); + } else { + heapsToMakeResident.push_back(heap->GetD3D12Pageable().Get()); + sizeToMakeResident += heap->GetSize(); + } + + mLRUCache.Append(heap); + heap->SetLastSubmission(pendingCommandSerial); + } + + if (heapsToMakeResident.size() != 0) { + DAWN_TRY(EnsureCanMakeResident(sizeToMakeResident)); + + // Note that MakeResident is a synchronous function and can add a significant + // overhead to command recording. In the future, it may be possible to decrease this + // overhead by using MakeResident on a secondary thread, or by instead making use of + // the EnqueueMakeResident function (which is not available on all Windows 10 + // platforms). + DAWN_TRY(CheckHRESULT(mDevice->GetD3D12Device()->MakeResident( + heapsToMakeResident.size(), heapsToMakeResident.data()), + "Making scheduled-to-be-used resources resident in " + "device local memory")); + } + + return {}; + } + + // When a new heap is allocated, the heap will be made resident upon creation. We must track + // when this happens to avoid calling MakeResident a second time. + void ResidencyManager::TrackResidentAllocation(Heap* heap) { + if (!mResidencyManagementEnabled) { + return; + } + + // Depending on device architecture and heap type, the heap may not need tracked. + if (!ShouldTrackHeap(heap)) { + return; + } + + mLRUCache.Append(heap); + } }} // namespace dawn_native::d3d12 \ No newline at end of file diff --git a/src/dawn_native/d3d12/ResidencyManagerD3D12.h b/src/dawn_native/d3d12/ResidencyManagerD3D12.h index 575515aaa8..71c7fae589 100644 --- a/src/dawn_native/d3d12/ResidencyManagerD3D12.h +++ b/src/dawn_native/d3d12/ResidencyManagerD3D12.h @@ -15,17 +15,29 @@ #ifndef DAWNNATIVE_D3D12_RESIDENCYMANAGERD3D12_H_ #define DAWNNATIVE_D3D12_RESIDENCYMANAGERD3D12_H_ +#include "common/LinkedList.h" +#include "common/Serial.h" +#include "dawn_native/Error.h" #include "dawn_native/dawn_platform.h" namespace dawn_native { namespace d3d12 { class Device; + class Heap; class ResidencyManager { public: ResidencyManager(Device* device); + + MaybeError LockMappableHeap(Heap* heap); + void UnlockMappableHeap(Heap* heap); + MaybeError EnsureCanMakeResident(uint64_t allocationSize); + MaybeError EnsureHeapsAreResident(Heap** heaps, size_t heapCount); + uint64_t SetExternalMemoryReservation(uint64_t requestedReservationSize); + void TrackResidentAllocation(Heap* heap); + private: struct VideoMemoryInfo { uint64_t dawnBudget; @@ -33,9 +45,12 @@ namespace dawn_native { namespace d3d12 { uint64_t externalReservation; uint64_t externalRequest; }; + ResultOrError RemoveSingleEntryFromLRU(); + bool ShouldTrackHeap(Heap* heap) const; void UpdateVideoMemoryInfo(); - Device* mDevice = nullptr; + Device* mDevice; + LinkedList mLRUCache; bool mResidencyManagementEnabled = false; VideoMemoryInfo mVideoMemoryInfo = {}; }; diff --git a/src/dawn_native/d3d12/ResourceAllocatorManagerD3D12.cpp b/src/dawn_native/d3d12/ResourceAllocatorManagerD3D12.cpp index 257b00378d..0fd4a6404a 100644 --- a/src/dawn_native/d3d12/ResourceAllocatorManagerD3D12.cpp +++ b/src/dawn_native/d3d12/ResourceAllocatorManagerD3D12.cpp @@ -18,6 +18,7 @@ #include "dawn_native/d3d12/DeviceD3D12.h" #include "dawn_native/d3d12/HeapAllocatorD3D12.h" #include "dawn_native/d3d12/HeapD3D12.h" +#include "dawn_native/d3d12/ResidencyManagerD3D12.h" namespace dawn_native { namespace d3d12 { namespace { @@ -255,6 +256,10 @@ namespace dawn_native { namespace d3d12 { Heap* heap = ToBackend(allocation.GetResourceHeap()); + // Before calling CreatePlacedResource, we must ensure the target heap is resident. + // CreatePlacedResource will fail if it is not. + DAWN_TRY(mDevice->GetResidencyManager()->EnsureHeapsAreResident(&heap, 1)); + // With placed resources, a single heap can be reused. // The resource placed at an offset is only reclaimed // upon Tick or after the last command list using the resource has completed @@ -296,6 +301,11 @@ namespace dawn_native { namespace d3d12 { return ResourceHeapAllocation{}; // Invalid } + // CreateCommittedResource will implicitly make the created resource resident. We must + // ensure enough free memory exists before allocating to avoid an out-of-memory error when + // overcommitted. + DAWN_TRY(mDevice->GetResidencyManager()->EnsureCanMakeResident(resourceInfo.SizeInBytes)); + // Note: Heap flags are inferred by the resource descriptor and do not need to be explicitly // provided to CreateCommittedResource. ComPtr committedResource; @@ -310,7 +320,11 @@ namespace dawn_native { namespace d3d12 { // heap granularity, every directly allocated ResourceHeapAllocation also stores a Heap // object. This object is created manually, and must be deleted manually upon deallocation // of the committed resource. - Heap* heap = new Heap(committedResource, resourceInfo.SizeInBytes); + Heap* heap = new Heap(committedResource, heapType, resourceInfo.SizeInBytes); + + // Calling CreateCommittedResource implicitly calls MakeResident on the resource. We must + // track this to avoid calling MakeResident a second time. + mDevice->GetResidencyManager()->TrackResidentAllocation(heap); AllocationInfo info; info.mMethod = AllocationMethod::kDirect; diff --git a/src/dawn_native/d3d12/StagingBufferD3D12.cpp b/src/dawn_native/d3d12/StagingBufferD3D12.cpp index 0035c967d7..2d6b5af1d0 100644 --- a/src/dawn_native/d3d12/StagingBufferD3D12.cpp +++ b/src/dawn_native/d3d12/StagingBufferD3D12.cpp @@ -16,6 +16,7 @@ #include "dawn_native/d3d12/D3D12Error.h" #include "dawn_native/d3d12/DeviceD3D12.h" #include "dawn_native/d3d12/HeapD3D12.h" +#include "dawn_native/d3d12/ResidencyManagerD3D12.h" namespace dawn_native { namespace d3d12 { @@ -41,6 +42,11 @@ namespace dawn_native { namespace d3d12 { mDevice->AllocateMemory(D3D12_HEAP_TYPE_UPLOAD, resourceDescriptor, D3D12_RESOURCE_STATE_GENERIC_READ)); + // The mapped buffer can be accessed at any time, so it must be locked to ensure it is never + // evicted. This buffer should already have been made resident when it was created. + DAWN_TRY(mDevice->GetResidencyManager()->LockMappableHeap( + ToBackend(mUploadHeap.GetResourceHeap()))); + return CheckHRESULT(GetResource()->Map(0, nullptr, &mMappedPointer), "ID3D12Resource::Map"); } @@ -50,6 +56,12 @@ namespace dawn_native { namespace d3d12 { if (mUploadHeap.GetInfo().mMethod == AllocationMethod::kInvalid) { return; } + + // The underlying heap was locked in residency upon creation. We must unlock it when this + // buffer becomes unmapped. + mDevice->GetResidencyManager()->UnlockMappableHeap( + ToBackend(mUploadHeap.GetResourceHeap())); + // Invalidate the CPU virtual address & flush cache (if needed). GetResource()->Unmap(0, nullptr); mMappedPointer = nullptr; @@ -60,10 +72,4 @@ namespace dawn_native { namespace d3d12 { ID3D12Resource* StagingBuffer::GetResource() const { return mUploadHeap.GetD3D12Resource().Get(); } - - void StagingBuffer::TrackUsage(CommandRecordingContext* commandContext) { - // Track the underlying heap to ensure residency. - Heap* heap = ToBackend(mUploadHeap.GetResourceHeap()); - commandContext->TrackHeapUsage(heap, mDevice->GetPendingCommandSerial()); - } }} // namespace dawn_native::d3d12 diff --git a/src/dawn_native/d3d12/StagingBufferD3D12.h b/src/dawn_native/d3d12/StagingBufferD3D12.h index 5cc7cb26c7..5bc32a26e2 100644 --- a/src/dawn_native/d3d12/StagingBufferD3D12.h +++ b/src/dawn_native/d3d12/StagingBufferD3D12.h @@ -33,8 +33,6 @@ namespace dawn_native { namespace d3d12 { MaybeError Initialize() override; - void TrackUsage(CommandRecordingContext* commandContext); - private: Device* mDevice; ResourceHeapAllocation mUploadHeap; diff --git a/src/tests/unittests/LinkedListTests.cpp b/src/tests/unittests/LinkedListTests.cpp index ee298b0a0f..94f2ecdc6a 100644 --- a/src/tests/unittests/LinkedListTests.cpp +++ b/src/tests/unittests/LinkedListTests.cpp @@ -350,4 +350,16 @@ TEST(LinkedList, NodeMoveConstructor) { EXPECT_EQ(&n3, n2_new.next()); EXPECT_EQ(&n2_new, n3.previous()); EXPECT_EQ(2, n2_new.id()); +} + +TEST(LinkedList, IsInList) { + LinkedList list; + + Node n(1); + + EXPECT_FALSE(n.IsInList()); + list.Append(&n); + EXPECT_TRUE(n.IsInList()); + n.RemoveFromList(); + EXPECT_FALSE(n.IsInList()); } \ No newline at end of file