From 154badfe2f4c70f6b48bb7269d1dee6e685898a7 Mon Sep 17 00:00:00 2001 From: Bryan Bernhart Date: Thu, 17 Oct 2019 17:25:28 +0000 Subject: [PATCH] Resource Management 8: placed resource sub-allocation. - Adds d3d allocators (placed resource + heap). - Support for heap tier 1 but only buffers. - Suballocation optimization is enabled for allocations under 4MB. BUG=dawn:27 Change-Id: I79177830670d1f322bbadf45f797415a3e9208d9 Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/5680 Reviewed-by: Corentin Wallez Commit-Queue: Bryan Bernhart --- BUILD.gn | 6 +- src/dawn_native/BuddyMemoryAllocator.cpp | 14 +- src/dawn_native/BuddyMemoryAllocator.h | 3 +- src/dawn_native/MemoryAllocator.h | 3 +- src/dawn_native/d3d12/BufferD3D12.cpp | 3 +- .../d3d12/CommittedResourceAllocatorD3D12.cpp | 52 ----- .../d3d12/CommittedResourceAllocatorD3D12.h | 47 ---- src/dawn_native/d3d12/DeviceD3D12.cpp | 8 +- src/dawn_native/d3d12/DeviceD3D12.h | 3 +- src/dawn_native/d3d12/HeapAllocatorD3D12.cpp | 52 +++++ src/dawn_native/d3d12/HeapAllocatorD3D12.h | 42 ++++ src/dawn_native/d3d12/HeapD3D12.cpp | 25 ++ src/dawn_native/d3d12/HeapD3D12.h | 35 +++ .../d3d12/ResourceAllocatorManagerD3D12.cpp | 218 +++++++++++++++--- .../d3d12/ResourceAllocatorManagerD3D12.h | 57 +++-- src/dawn_native/d3d12/StagingBufferD3D12.cpp | 11 +- .../unittests/BuddyMemoryAllocatorTests.cpp | 3 +- 17 files changed, 415 insertions(+), 167 deletions(-) delete mode 100644 src/dawn_native/d3d12/CommittedResourceAllocatorD3D12.cpp delete mode 100644 src/dawn_native/d3d12/CommittedResourceAllocatorD3D12.h create mode 100644 src/dawn_native/d3d12/HeapAllocatorD3D12.cpp create mode 100644 src/dawn_native/d3d12/HeapAllocatorD3D12.h create mode 100644 src/dawn_native/d3d12/HeapD3D12.cpp create mode 100644 src/dawn_native/d3d12/HeapD3D12.h diff --git a/BUILD.gn b/BUILD.gn index 24c21ca956..921168006e 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -256,8 +256,6 @@ source_set("libdawn_native_sources") { "src/dawn_native/d3d12/CommandBufferD3D12.h", "src/dawn_native/d3d12/CommandRecordingContext.cpp", "src/dawn_native/d3d12/CommandRecordingContext.h", - "src/dawn_native/d3d12/CommittedResourceAllocatorD3D12.cpp", - "src/dawn_native/d3d12/CommittedResourceAllocatorD3D12.h", "src/dawn_native/d3d12/ComputePipelineD3D12.cpp", "src/dawn_native/d3d12/ComputePipelineD3D12.h", "src/dawn_native/d3d12/D3D12Error.cpp", @@ -269,6 +267,10 @@ source_set("libdawn_native_sources") { "src/dawn_native/d3d12/DeviceD3D12.cpp", "src/dawn_native/d3d12/DeviceD3D12.h", "src/dawn_native/d3d12/Forward.h", + "src/dawn_native/d3d12/HeapAllocatorD3D12.cpp", + "src/dawn_native/d3d12/HeapAllocatorD3D12.h", + "src/dawn_native/d3d12/HeapD3D12.cpp", + "src/dawn_native/d3d12/HeapD3D12.h", "src/dawn_native/d3d12/NativeSwapChainImplD3D12.cpp", "src/dawn_native/d3d12/NativeSwapChainImplD3D12.h", "src/dawn_native/d3d12/PipelineLayoutD3D12.cpp", diff --git a/src/dawn_native/BuddyMemoryAllocator.cpp b/src/dawn_native/BuddyMemoryAllocator.cpp index 87f4743e1f..8e8272c1a8 100644 --- a/src/dawn_native/BuddyMemoryAllocator.cpp +++ b/src/dawn_native/BuddyMemoryAllocator.cpp @@ -34,12 +34,18 @@ namespace dawn_native { } ResultOrError BuddyMemoryAllocator::Allocate(uint64_t allocationSize, - uint64_t alignment, - int memoryFlags) { + uint64_t alignment) { ResourceMemoryAllocation invalidAllocation = ResourceMemoryAllocation{}; + if (allocationSize == 0) { + return invalidAllocation; + } + + // Round allocation size to nearest power-of-two. + allocationSize = NextPowerOfTwo(allocationSize); + // Allocation cannot exceed the memory size. - if (allocationSize == 0 || allocationSize > mMemorySize) { + if (allocationSize > mMemorySize) { return invalidAllocation; } @@ -53,7 +59,7 @@ namespace dawn_native { if (mTrackedSubAllocations[memoryIndex].refcount == 0) { // Transfer ownership to this allocator std::unique_ptr memory; - DAWN_TRY_ASSIGN(memory, mClient->Allocate(mMemorySize, memoryFlags)); + DAWN_TRY_ASSIGN(memory, mClient->Allocate(mMemorySize)); mTrackedSubAllocations[memoryIndex] = {/*refcount*/ 0, std::move(memory)}; } diff --git a/src/dawn_native/BuddyMemoryAllocator.h b/src/dawn_native/BuddyMemoryAllocator.h index b31b40074b..b8821139b4 100644 --- a/src/dawn_native/BuddyMemoryAllocator.h +++ b/src/dawn_native/BuddyMemoryAllocator.h @@ -44,8 +44,7 @@ namespace dawn_native { ~BuddyMemoryAllocator() = default; ResultOrError Allocate(uint64_t allocationSize, - uint64_t alignment, - int memoryFlags = 0); + uint64_t alignment); void Deallocate(const ResourceMemoryAllocation& allocation); uint64_t GetMemorySize() const; diff --git a/src/dawn_native/MemoryAllocator.h b/src/dawn_native/MemoryAllocator.h index e932c91cd5..dfb299328b 100644 --- a/src/dawn_native/MemoryAllocator.h +++ b/src/dawn_native/MemoryAllocator.h @@ -24,8 +24,7 @@ namespace dawn_native { public: virtual ~MemoryAllocator() = default; - virtual ResultOrError> Allocate(uint64_t size, - int memoryFlags) = 0; + virtual ResultOrError> Allocate(uint64_t size) = 0; virtual void Deallocate(std::unique_ptr allocation) = 0; }; } // namespace dawn_native diff --git a/src/dawn_native/d3d12/BufferD3D12.cpp b/src/dawn_native/d3d12/BufferD3D12.cpp index a0a208305f..6c0f912f83 100644 --- a/src/dawn_native/d3d12/BufferD3D12.cpp +++ b/src/dawn_native/d3d12/BufferD3D12.cpp @@ -111,8 +111,7 @@ namespace dawn_native { namespace d3d12 { DAWN_TRY_ASSIGN( mResourceAllocation, - ToBackend(GetDevice()) - ->AllocateMemory(heapType, resourceDescriptor, bufferUsage, D3D12_HEAP_FLAG_NONE)); + ToBackend(GetDevice())->AllocateMemory(heapType, resourceDescriptor, bufferUsage)); return {}; } diff --git a/src/dawn_native/d3d12/CommittedResourceAllocatorD3D12.cpp b/src/dawn_native/d3d12/CommittedResourceAllocatorD3D12.cpp deleted file mode 100644 index 9a55e690b2..0000000000 --- a/src/dawn_native/d3d12/CommittedResourceAllocatorD3D12.cpp +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2019 The Dawn Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "dawn_native/d3d12/CommittedResourceAllocatorD3D12.h" -#include "dawn_native/d3d12/DeviceD3D12.h" - -namespace dawn_native { namespace d3d12 { - - CommittedResourceAllocator::CommittedResourceAllocator(Device* device, D3D12_HEAP_TYPE heapType) - : mDevice(device), mHeapType(heapType) { - } - - ResultOrError CommittedResourceAllocator::Allocate( - const D3D12_RESOURCE_DESC& resourceDescriptor, - D3D12_RESOURCE_STATES initialUsage, - D3D12_HEAP_FLAGS heapFlags) { - D3D12_HEAP_PROPERTIES heapProperties; - heapProperties.Type = mHeapType; - heapProperties.CPUPageProperty = D3D12_CPU_PAGE_PROPERTY_UNKNOWN; - heapProperties.MemoryPoolPreference = D3D12_MEMORY_POOL_UNKNOWN; - heapProperties.CreationNodeMask = 0; - heapProperties.VisibleNodeMask = 0; - - ComPtr committedResource; - if (FAILED(mDevice->GetD3D12Device()->CreateCommittedResource( - &heapProperties, heapFlags, &resourceDescriptor, initialUsage, nullptr, - IID_PPV_ARGS(&committedResource)))) { - return DAWN_OUT_OF_MEMORY_ERROR("Unable to allocate resource"); - } - - AllocationInfo info; - info.mMethod = AllocationMethod::kDirect; - - return ResourceHeapAllocation{info, - /*offset*/ 0, std::move(committedResource)}; - } - - void CommittedResourceAllocator::Deallocate(ResourceHeapAllocation& allocation) { - mDevice->ReferenceUntilUnused(allocation.GetD3D12Resource()); - } -}} // namespace dawn_native::d3d12 diff --git a/src/dawn_native/d3d12/CommittedResourceAllocatorD3D12.h b/src/dawn_native/d3d12/CommittedResourceAllocatorD3D12.h deleted file mode 100644 index 7bfb9d8b42..0000000000 --- a/src/dawn_native/d3d12/CommittedResourceAllocatorD3D12.h +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2019 The Dawn Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef DAWNNATIVE_D3D12_COMMITTEDRESOURCEALLOCATORD3D12_H_ -#define DAWNNATIVE_D3D12_COMMITTEDRESOURCEALLOCATORD3D12_H_ - -#include "common/SerialQueue.h" -#include "dawn_native/Error.h" -#include "dawn_native/d3d12/ResourceHeapAllocationD3D12.h" -#include "dawn_native/d3d12/d3d12_platform.h" - -namespace dawn_native { namespace d3d12 { - - class Device; - - // Wrapper to allocate D3D12 committed resource. - // Committed resources are implicitly backed by a D3D12 heap. - class CommittedResourceAllocator { - public: - CommittedResourceAllocator(Device* device, D3D12_HEAP_TYPE heapType); - ~CommittedResourceAllocator() = default; - - ResultOrError Allocate( - const D3D12_RESOURCE_DESC& resourceDescriptor, - D3D12_RESOURCE_STATES initialUsage, - D3D12_HEAP_FLAGS heapFlags); - void Deallocate(ResourceHeapAllocation& allocation); - - private: - Device* mDevice; - D3D12_HEAP_TYPE mHeapType; - }; - -}} // namespace dawn_native::d3d12 - -#endif // DAWNNATIVE_D3D12_COMMITTEDRESOURCEALLOCATORD3D12_H_ diff --git a/src/dawn_native/d3d12/DeviceD3D12.cpp b/src/dawn_native/d3d12/DeviceD3D12.cpp index f1b2e3120f..4a2e9814ee 100644 --- a/src/dawn_native/d3d12/DeviceD3D12.cpp +++ b/src/dawn_native/d3d12/DeviceD3D12.cpp @@ -213,6 +213,7 @@ namespace dawn_native { namespace d3d12 { mDynamicUploader->Deallocate(mCompletedSerial); mResourceAllocator->Tick(mCompletedSerial); + mResourceAllocatorManager->Tick(mCompletedSerial); DAWN_TRY(mCommandAllocatorManager->Tick(mCompletedSerial)); mDescriptorHeapAllocator->Deallocate(mCompletedSerial); mMapRequestTracker->Tick(mCompletedSerial); @@ -348,10 +349,9 @@ namespace dawn_native { namespace d3d12 { ResultOrError Device::AllocateMemory( D3D12_HEAP_TYPE heapType, const D3D12_RESOURCE_DESC& resourceDescriptor, - D3D12_RESOURCE_STATES initialUsage, - D3D12_HEAP_FLAGS heapFlags) { - return mResourceAllocatorManager->AllocateMemory(heapType, resourceDescriptor, initialUsage, - heapFlags); + D3D12_RESOURCE_STATES initialUsage) { + return mResourceAllocatorManager->AllocateMemory(heapType, resourceDescriptor, + initialUsage); } TextureBase* Device::WrapSharedHandle(const TextureDescriptor* descriptor, diff --git a/src/dawn_native/d3d12/DeviceD3D12.h b/src/dawn_native/d3d12/DeviceD3D12.h index 0a00da293b..4981c1821c 100644 --- a/src/dawn_native/d3d12/DeviceD3D12.h +++ b/src/dawn_native/d3d12/DeviceD3D12.h @@ -90,8 +90,7 @@ namespace dawn_native { namespace d3d12 { ResultOrError AllocateMemory( D3D12_HEAP_TYPE heapType, const D3D12_RESOURCE_DESC& resourceDescriptor, - D3D12_RESOURCE_STATES initialUsage, - D3D12_HEAP_FLAGS heapFlags); + D3D12_RESOURCE_STATES initialUsage); void DeallocateMemory(ResourceHeapAllocation& allocation); diff --git a/src/dawn_native/d3d12/HeapAllocatorD3D12.cpp b/src/dawn_native/d3d12/HeapAllocatorD3D12.cpp new file mode 100644 index 0000000000..8cd56402ae --- /dev/null +++ b/src/dawn_native/d3d12/HeapAllocatorD3D12.cpp @@ -0,0 +1,52 @@ +// Copyright 2019 The Dawn Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "dawn_native/d3d12/HeapAllocatorD3D12.h" +#include "dawn_native/d3d12/DeviceD3D12.h" +#include "dawn_native/d3d12/HeapD3D12.h" + +namespace dawn_native { namespace d3d12 { + + HeapAllocator::HeapAllocator(Device* device, + D3D12_HEAP_TYPE heapType, + D3D12_HEAP_FLAGS heapFlags) + : mDevice(device), mHeapType(heapType), mHeapFlags(heapFlags) { + } + + ResultOrError> HeapAllocator::Allocate(uint64_t size) { + D3D12_HEAP_DESC heapDesc; + heapDesc.SizeInBytes = size; + heapDesc.Properties.Type = mHeapType; + heapDesc.Properties.CPUPageProperty = D3D12_CPU_PAGE_PROPERTY_UNKNOWN; + heapDesc.Properties.MemoryPoolPreference = D3D12_MEMORY_POOL_UNKNOWN; + heapDesc.Properties.CreationNodeMask = 0; + heapDesc.Properties.VisibleNodeMask = 0; + // MSAA vs non-MSAA resources have separate heap alignments. + // TODO(bryan.bernhart@intel.com): Support heap creation containing MSAA resources. + heapDesc.Alignment = D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT; + heapDesc.Flags = mHeapFlags; + + ComPtr heap; + if (FAILED(mDevice->GetD3D12Device()->CreateHeap(&heapDesc, IID_PPV_ARGS(&heap)))) { + return DAWN_OUT_OF_MEMORY_ERROR("Unable to allocate heap"); + } + + return {std::make_unique(std::move(heap))}; + } + + void HeapAllocator::Deallocate(std::unique_ptr heap) { + mDevice->ReferenceUntilUnused(static_cast(heap.get())->GetD3D12Heap()); + } + +}} // namespace dawn_native::d3d12 \ No newline at end of file diff --git a/src/dawn_native/d3d12/HeapAllocatorD3D12.h b/src/dawn_native/d3d12/HeapAllocatorD3D12.h new file mode 100644 index 0000000000..2c91333354 --- /dev/null +++ b/src/dawn_native/d3d12/HeapAllocatorD3D12.h @@ -0,0 +1,42 @@ +// Copyright 2019 The Dawn Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DAWNNATIVE_D3D12_HEAPALLOCATORD3D12_H_ +#define DAWNNATIVE_D3D12_HEAPALLOCATORD3D12_H_ + +#include "dawn_native/MemoryAllocator.h" +#include "dawn_native/d3d12/d3d12_platform.h" + +namespace dawn_native { namespace d3d12 { + + class Device; + + // Wrapper to allocate a D3D12 heap. + class HeapAllocator : public MemoryAllocator { + public: + HeapAllocator(Device* device, D3D12_HEAP_TYPE heapType, D3D12_HEAP_FLAGS heapFlags); + ~HeapAllocator() override = default; + + ResultOrError> Allocate(uint64_t size) override; + void Deallocate(std::unique_ptr allocation) override; + + private: + Device* mDevice; + D3D12_HEAP_TYPE mHeapType; + D3D12_HEAP_FLAGS mHeapFlags; + }; + +}} // namespace dawn_native::d3d12 + +#endif // DAWNNATIVE_D3D12_HEAPALLOCATORD3D12_H_ \ No newline at end of file diff --git a/src/dawn_native/d3d12/HeapD3D12.cpp b/src/dawn_native/d3d12/HeapD3D12.cpp new file mode 100644 index 0000000000..2e35bdf7cf --- /dev/null +++ b/src/dawn_native/d3d12/HeapD3D12.cpp @@ -0,0 +1,25 @@ +// Copyright 2019 The Dawn Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "dawn_native/d3d12/HeapD3D12.h" + +namespace dawn_native { namespace d3d12 { + + Heap::Heap(ComPtr heap) : mHeap(std::move(heap)) { + } + + ComPtr Heap::GetD3D12Heap() const { + return mHeap; + } +}} // namespace dawn_native::d3d12 \ No newline at end of file diff --git a/src/dawn_native/d3d12/HeapD3D12.h b/src/dawn_native/d3d12/HeapD3D12.h new file mode 100644 index 0000000000..834e42ac9f --- /dev/null +++ b/src/dawn_native/d3d12/HeapD3D12.h @@ -0,0 +1,35 @@ +// Copyright 2019 The Dawn Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DAWNNATIVE_D3D12_HEAPD3D12_H_ +#define DAWNNATIVE_D3D12_HEAPD3D12_H_ + +#include "dawn_native/ResourceHeap.h" +#include "dawn_native/d3d12/d3d12_platform.h" + +namespace dawn_native { namespace d3d12 { + + class Heap : public ResourceHeapBase { + public: + Heap(ComPtr heap); + ~Heap() = default; + + ComPtr GetD3D12Heap() const; + + private: + ComPtr mHeap; + }; +}} // namespace dawn_native::d3d12 + +#endif // DAWNNATIVE_D3D12_HEAPD3D12_H_ \ No newline at end of file diff --git a/src/dawn_native/d3d12/ResourceAllocatorManagerD3D12.cpp b/src/dawn_native/d3d12/ResourceAllocatorManagerD3D12.cpp index b18c998fb8..afa1bd4c0d 100644 --- a/src/dawn_native/d3d12/ResourceAllocatorManagerD3D12.cpp +++ b/src/dawn_native/d3d12/ResourceAllocatorManagerD3D12.cpp @@ -13,56 +13,218 @@ // limitations under the License. #include "dawn_native/d3d12/ResourceAllocatorManagerD3D12.h" -#include "dawn_native/d3d12/Forward.h" + +#include "dawn_native/d3d12/DeviceD3D12.h" +#include "dawn_native/d3d12/HeapAllocatorD3D12.h" +#include "dawn_native/d3d12/HeapD3D12.h" namespace dawn_native { namespace d3d12 { + namespace { + D3D12_HEAP_TYPE GetD3D12HeapType(ResourceHeapKind resourceHeapKind) { + switch (resourceHeapKind) { + case Readback_OnlyBuffers: + return D3D12_HEAP_TYPE_READBACK; + case Default_OnlyBuffers: + case Default_OnlyNonRenderableOrDepthTextures: + case Default_OnlyRenderableOrDepthTextures: + return D3D12_HEAP_TYPE_DEFAULT; + case Upload_OnlyBuffers: + return D3D12_HEAP_TYPE_UPLOAD; + default: + UNREACHABLE(); + } + } + + D3D12_HEAP_FLAGS GetD3D12HeapFlags(ResourceHeapKind resourceHeapKind) { + switch (resourceHeapKind) { + case Default_OnlyBuffers: + case Readback_OnlyBuffers: + case Upload_OnlyBuffers: + return D3D12_HEAP_FLAG_ALLOW_ONLY_BUFFERS; + case Default_OnlyNonRenderableOrDepthTextures: + return D3D12_HEAP_FLAG_ALLOW_ONLY_NON_RT_DS_TEXTURES; + case Default_OnlyRenderableOrDepthTextures: + return D3D12_HEAP_FLAG_ALLOW_ONLY_RT_DS_TEXTURES; + default: + UNREACHABLE(); + } + } + + ResourceHeapKind GetResourceHeapKind(D3D12_RESOURCE_DIMENSION dimension, + D3D12_HEAP_TYPE heapType, + D3D12_RESOURCE_FLAGS flags) { + switch (dimension) { + case D3D12_RESOURCE_DIMENSION_BUFFER: { + switch (heapType) { + case D3D12_HEAP_TYPE_UPLOAD: + return Upload_OnlyBuffers; + case D3D12_HEAP_TYPE_DEFAULT: + return Default_OnlyBuffers; + case D3D12_HEAP_TYPE_READBACK: + return Readback_OnlyBuffers; + default: + UNREACHABLE(); + } + } break; + case D3D12_RESOURCE_DIMENSION_TEXTURE1D: + case D3D12_RESOURCE_DIMENSION_TEXTURE2D: + case D3D12_RESOURCE_DIMENSION_TEXTURE3D: { + switch (heapType) { + case D3D12_HEAP_TYPE_DEFAULT: { + if ((flags & D3D12_RESOURCE_FLAG_ALLOW_DEPTH_STENCIL) || + (flags & D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET)) { + return Default_OnlyRenderableOrDepthTextures; + } else { + return Default_OnlyNonRenderableOrDepthTextures; + } + } break; + default: + UNREACHABLE(); + } + } break; + default: + UNREACHABLE(); + } + } + } // namespace ResourceAllocatorManager::ResourceAllocatorManager(Device* device) : mDevice(device) { + for (uint32_t i = 0; i < ResourceHeapKind::EnumCount; i++) { + const ResourceHeapKind resourceHeapKind = static_cast(i); + mSubAllocatedResourceAllocators[i] = std::make_unique( + kMaxHeapSize, kMinHeapSize, + std::make_unique(mDevice, GetD3D12HeapType(resourceHeapKind), + GetD3D12HeapFlags(resourceHeapKind))); + } } ResultOrError ResourceAllocatorManager::AllocateMemory( D3D12_HEAP_TYPE heapType, const D3D12_RESOURCE_DESC& resourceDescriptor, - D3D12_RESOURCE_STATES initialUsage, - D3D12_HEAP_FLAGS heapFlags) { - const size_t heapTypeIndex = GetD3D12HeapTypeToIndex(heapType); - ASSERT(heapTypeIndex < kNumHeapTypes); - - // Get the direct allocator using a tightly sized heap (aka CreateCommittedResource). - CommittedResourceAllocator* allocator = mDirectResourceAllocators[heapTypeIndex].get(); - if (allocator == nullptr) { - mDirectResourceAllocators[heapTypeIndex] = - std::make_unique(mDevice, heapType); - allocator = mDirectResourceAllocators[heapTypeIndex].get(); + D3D12_RESOURCE_STATES initialUsage) { + // TODO(bryan.bernhart@intel.com): Conditionally disable sub-allocation. + // For very large resources, there is no benefit to suballocate. + // For very small resources, it is inefficent to suballocate given the min. heap + // size could be much larger then the resource allocation. + // Attempt to satisfy the request using sub-allocation (placed resource in a heap). + ResourceHeapAllocation subAllocation; + DAWN_TRY_ASSIGN(subAllocation, + CreatePlacedResource(heapType, resourceDescriptor, initialUsage)); + if (subAllocation.GetInfo().mMethod != AllocationMethod::kInvalid) { + return subAllocation; } - ResourceHeapAllocation allocation; - DAWN_TRY_ASSIGN(allocation, - allocator->Allocate(resourceDescriptor, initialUsage, heapFlags)); + // If sub-allocation fails, fall-back to direct allocation (committed resource). + ResourceHeapAllocation directAllocation; + DAWN_TRY_ASSIGN(directAllocation, + CreateCommittedResource(heapType, resourceDescriptor, initialUsage)); - return allocation; + return directAllocation; } - size_t ResourceAllocatorManager::GetD3D12HeapTypeToIndex(D3D12_HEAP_TYPE heapType) const { - ASSERT(heapType > 0); - ASSERT(static_cast(heapType) <= kNumHeapTypes); - return heapType - 1; + void ResourceAllocatorManager::Tick(Serial completedSerial) { + for (ResourceHeapAllocation& allocation : + mAllocationsToDelete.IterateUpTo(completedSerial)) { + if (allocation.GetInfo().mMethod == AllocationMethod::kSubAllocated) { + FreeMemory(allocation); + } + } + mAllocationsToDelete.ClearUpTo(completedSerial); } void ResourceAllocatorManager::DeallocateMemory(ResourceHeapAllocation& allocation) { if (allocation.GetInfo().mMethod == AllocationMethod::kInvalid) { return; } - CommittedResourceAllocator* allocator = nullptr; - D3D12_HEAP_PROPERTIES heapProp; - allocation.GetD3D12Resource()->GetHeapProperties(&heapProp, nullptr); - const size_t heapTypeIndex = GetD3D12HeapTypeToIndex(heapProp.Type); - ASSERT(heapTypeIndex < kNumHeapTypes); - allocator = mDirectResourceAllocators[heapTypeIndex].get(); - allocator->Deallocate(allocation); - // Invalidate the underlying resource heap in case the client accidentally + mAllocationsToDelete.Enqueue(allocation, mDevice->GetPendingCommandSerial()); + + // Invalidate the allocation immediately in case one accidentally // calls DeallocateMemory again using the same allocation. allocation.Invalidate(); } + + void ResourceAllocatorManager::FreeMemory(ResourceHeapAllocation& allocation) { + ASSERT(allocation.GetInfo().mMethod == AllocationMethod::kSubAllocated); + + D3D12_HEAP_PROPERTIES heapProp; + allocation.GetD3D12Resource()->GetHeapProperties(&heapProp, nullptr); + + const D3D12_RESOURCE_DESC resourceDescriptor = allocation.GetD3D12Resource()->GetDesc(); + + const size_t resourceHeapKindIndex = GetResourceHeapKind( + resourceDescriptor.Dimension, heapProp.Type, resourceDescriptor.Flags); + + mSubAllocatedResourceAllocators[resourceHeapKindIndex]->Deallocate(allocation); + } + + ResultOrError ResourceAllocatorManager::CreatePlacedResource( + D3D12_HEAP_TYPE heapType, + const D3D12_RESOURCE_DESC& resourceDescriptor, + D3D12_RESOURCE_STATES initialUsage) { + const size_t resourceHeapKindIndex = + GetResourceHeapKind(resourceDescriptor.Dimension, heapType, resourceDescriptor.Flags); + + BuddyMemoryAllocator* allocator = + mSubAllocatedResourceAllocators[resourceHeapKindIndex].get(); + + const D3D12_RESOURCE_ALLOCATION_INFO resourceInfo = + mDevice->GetD3D12Device()->GetResourceAllocationInfo(0, 1, &resourceDescriptor); + + ResourceMemoryAllocation allocation; + DAWN_TRY_ASSIGN(allocation, + allocator->Allocate(resourceInfo.SizeInBytes, resourceInfo.Alignment)); + if (allocation.GetInfo().mMethod == AllocationMethod::kInvalid) { + return ResourceHeapAllocation{}; // invalid + } + + ID3D12Heap* heap = static_cast(allocation.GetResourceHeap())->GetD3D12Heap().Get(); + + // With placed resources, a single heap can be reused. + // The resource placed at an offset is only reclaimed + // upon Tick or after the last command list using the resource has completed + // on the GPU. This means the same physical memory is not reused + // within the same command-list and does not require additional synchronization (aliasing + // barrier). + // https://docs.microsoft.com/en-us/windows/win32/api/d3d12/nf-d3d12-id3d12device-createplacedresource + ComPtr placedResource; + if (FAILED(mDevice->GetD3D12Device()->CreatePlacedResource( + heap, allocation.GetOffset(), &resourceDescriptor, initialUsage, nullptr, + IID_PPV_ARGS(&placedResource)))) { + // Note: Heap must already exist before the resource is created. If CreatePlacedResource + // fails, it's unlikely to be OOM. + return DAWN_DEVICE_LOST_ERROR("Unable to allocate resource"); + } + + return ResourceHeapAllocation{allocation.GetInfo(), allocation.GetOffset(), + std::move(placedResource)}; + } + + ResultOrError ResourceAllocatorManager::CreateCommittedResource( + D3D12_HEAP_TYPE heapType, + const D3D12_RESOURCE_DESC& resourceDescriptor, + D3D12_RESOURCE_STATES initialUsage) { + D3D12_HEAP_PROPERTIES heapProperties; + heapProperties.Type = heapType; + heapProperties.CPUPageProperty = D3D12_CPU_PAGE_PROPERTY_UNKNOWN; + heapProperties.MemoryPoolPreference = D3D12_MEMORY_POOL_UNKNOWN; + heapProperties.CreationNodeMask = 0; + heapProperties.VisibleNodeMask = 0; + + // Note: Heap flags are inferred by the resource descriptor and do not need to be explicitly + // provided to CreateCommittedResource. + ComPtr committedResource; + if (FAILED(mDevice->GetD3D12Device()->CreateCommittedResource( + &heapProperties, D3D12_HEAP_FLAG_NONE, &resourceDescriptor, initialUsage, nullptr, + IID_PPV_ARGS(&committedResource)))) { + return DAWN_OUT_OF_MEMORY_ERROR("Unable to allocate resource"); + } + + AllocationInfo info; + info.mMethod = AllocationMethod::kDirect; + + return ResourceHeapAllocation{info, + /*offset*/ 0, std::move(committedResource)}; + } + }} // namespace dawn_native::d3d12 diff --git a/src/dawn_native/d3d12/ResourceAllocatorManagerD3D12.h b/src/dawn_native/d3d12/ResourceAllocatorManagerD3D12.h index d8f1cdb30e..c8794252a7 100644 --- a/src/dawn_native/d3d12/ResourceAllocatorManagerD3D12.h +++ b/src/dawn_native/d3d12/ResourceAllocatorManagerD3D12.h @@ -15,7 +15,10 @@ #ifndef DAWNNATIVE_D3D12_RESOURCEALLOCATORMANAGERD3D12_H_ #define DAWNNATIVE_D3D12_RESOURCEALLOCATORMANAGERD3D12_H_ -#include "dawn_native/d3d12/CommittedResourceAllocatorD3D12.h" +#include "common/SerialQueue.h" + +#include "dawn_native/BuddyMemoryAllocator.h" +#include "dawn_native/d3d12/ResourceHeapAllocationD3D12.h" #include @@ -23,8 +26,23 @@ namespace dawn_native { namespace d3d12 { class Device; - // Manages a list of resource allocators used by the device to create resources using multiple - // allocation methods. + // Heap types + flags combinations are named after the D3D constants. + // https://docs.microsoft.com/en-us/windows/win32/api/d3d12/ne-d3d12-d3d12_heap_flags + // https://docs.microsoft.com/en-us/windows/win32/api/d3d12/ne-d3d12-d3d12_heap_type + enum ResourceHeapKind { + Readback_OnlyBuffers, + Upload_OnlyBuffers, + Default_OnlyBuffers, + + Default_OnlyNonRenderableOrDepthTextures, + Default_OnlyRenderableOrDepthTextures, + + EnumCount, + InvalidEnum = EnumCount, + }; + + // Manages a list of resource allocators used by the device to create resources using + // multiple allocation methods. class ResourceAllocatorManager { public: ResourceAllocatorManager(Device* device); @@ -32,29 +50,34 @@ namespace dawn_native { namespace d3d12 { ResultOrError AllocateMemory( D3D12_HEAP_TYPE heapType, const D3D12_RESOURCE_DESC& resourceDescriptor, - D3D12_RESOURCE_STATES initialUsage, - D3D12_HEAP_FLAGS heapFlags); + D3D12_RESOURCE_STATES initialUsage); void DeallocateMemory(ResourceHeapAllocation& allocation); + void Tick(Serial lastCompletedSerial); + private: - size_t GetD3D12HeapTypeToIndex(D3D12_HEAP_TYPE heapType) const; + void FreeMemory(ResourceHeapAllocation& allocation); + + ResultOrError CreatePlacedResource( + D3D12_HEAP_TYPE heapType, + const D3D12_RESOURCE_DESC& resourceDescriptor, + D3D12_RESOURCE_STATES initialUsage); + + ResultOrError CreateCommittedResource( + D3D12_HEAP_TYPE heapType, + const D3D12_RESOURCE_DESC& resourceDescriptor, + D3D12_RESOURCE_STATES initialUsage); Device* mDevice; - static constexpr uint32_t kNumHeapTypes = 4u; // Number of D3D12_HEAP_TYPE + static constexpr uint64_t kMaxHeapSize = 32ll * 1024ll * 1024ll * 1024ll; // 32GB + static constexpr uint64_t kMinHeapSize = 4ll * 1024ll * 1024ll; // 4MB - static_assert(D3D12_HEAP_TYPE_READBACK <= kNumHeapTypes, - "Readback heap type enum exceeds max heap types"); - static_assert(D3D12_HEAP_TYPE_UPLOAD <= kNumHeapTypes, - "Upload heap type enum exceeds max heap types"); - static_assert(D3D12_HEAP_TYPE_DEFAULT <= kNumHeapTypes, - "Default heap type enum exceeds max heap types"); - static_assert(D3D12_HEAP_TYPE_CUSTOM <= kNumHeapTypes, - "Custom heap type enum exceeds max heap types"); + std::array, ResourceHeapKind::EnumCount> + mSubAllocatedResourceAllocators; - std::array, kNumHeapTypes> - mDirectResourceAllocators; + SerialQueue mAllocationsToDelete; }; }} // namespace dawn_native::d3d12 diff --git a/src/dawn_native/d3d12/StagingBufferD3D12.cpp b/src/dawn_native/d3d12/StagingBufferD3D12.cpp index 9e6c2bd7c6..8d21484012 100644 --- a/src/dawn_native/d3d12/StagingBufferD3D12.cpp +++ b/src/dawn_native/d3d12/StagingBufferD3D12.cpp @@ -35,9 +35,9 @@ namespace dawn_native { namespace d3d12 { resourceDescriptor.Layout = D3D12_TEXTURE_LAYOUT_ROW_MAJOR; resourceDescriptor.Flags = D3D12_RESOURCE_FLAG_NONE; - DAWN_TRY_ASSIGN(mUploadHeap, mDevice->AllocateMemory( - D3D12_HEAP_TYPE_UPLOAD, resourceDescriptor, - D3D12_RESOURCE_STATE_GENERIC_READ, D3D12_HEAP_FLAG_NONE)); + DAWN_TRY_ASSIGN(mUploadHeap, + mDevice->AllocateMemory(D3D12_HEAP_TYPE_UPLOAD, resourceDescriptor, + D3D12_RESOURCE_STATE_GENERIC_READ)); if (FAILED(GetResource()->Map(0, nullptr, &mMappedPointer))) { return DAWN_DEVICE_LOST_ERROR("Unable to map staging buffer."); @@ -47,6 +47,11 @@ namespace dawn_native { namespace d3d12 { } StagingBuffer::~StagingBuffer() { + // Always check if the allocation is valid before Unmap. + // The resource would not exist had it failed to allocate. + if (mUploadHeap.GetInfo().mMethod == AllocationMethod::kInvalid) { + return; + } // Invalidate the CPU virtual address & flush cache (if needed). GetResource()->Unmap(0, nullptr); mMappedPointer = nullptr; diff --git a/src/tests/unittests/BuddyMemoryAllocatorTests.cpp b/src/tests/unittests/BuddyMemoryAllocatorTests.cpp index 0f85afbd41..72f6bbeb32 100644 --- a/src/tests/unittests/BuddyMemoryAllocatorTests.cpp +++ b/src/tests/unittests/BuddyMemoryAllocatorTests.cpp @@ -20,8 +20,7 @@ using namespace dawn_native; class DummyMemoryAllocator : public MemoryAllocator { public: - ResultOrError> Allocate(uint64_t size, - int memoryFlags = 0) override { + ResultOrError> Allocate(uint64_t size) override { return std::make_unique(); } void Deallocate(std::unique_ptr allocation) override {