From 450e212cf5f68a04faca9a1f995444e00287f9d5 Mon Sep 17 00:00:00 2001 From: Bryan Bernhart Date: Wed, 18 Sep 2019 22:06:41 +0000 Subject: [PATCH] Remove device dependencies from ringbuffer. Allows ringbuffer sub-allocator to be used for non-staging memory. BUG=dawn:155 Change-Id: Id0021907f520909aaebaf79e992124a47797d38d Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/9760 Reviewed-by: Corentin Wallez Reviewed-by: Austin Eng Commit-Queue: Bryan Bernhart --- BUILD.gn | 6 +- src/dawn_native/Buffer.cpp | 13 +- src/dawn_native/Device.cpp | 5 +- src/dawn_native/Device.h | 2 +- src/dawn_native/DynamicUploader.cpp | 83 ++++---- src/dawn_native/DynamicUploader.h | 29 +-- ...RingBuffer.cpp => RingBufferAllocator.cpp} | 102 +++------ .../{RingBuffer.h => RingBufferAllocator.h} | 40 ++-- src/dawn_native/d3d12/DeviceD3D12.cpp | 5 +- src/dawn_native/d3d12/TextureD3D12.cpp | 6 +- src/dawn_native/metal/DeviceMTL.mm | 3 +- src/dawn_native/null/DeviceNull.cpp | 1 + src/dawn_native/null/DeviceNull.h | 2 +- src/dawn_native/vulkan/DeviceVk.cpp | 3 +- src/dawn_native/vulkan/TextureVk.cpp | 6 +- .../unittests/RingBufferAllocatorTests.cpp | 161 ++++++++++++++ src/tests/unittests/RingBufferTests.cpp | 201 ------------------ 17 files changed, 290 insertions(+), 378 deletions(-) rename src/dawn_native/{RingBuffer.cpp => RingBufferAllocator.cpp} (57%) rename src/dawn_native/{RingBuffer.h => RingBufferAllocator.h} (62%) create mode 100644 src/tests/unittests/RingBufferAllocatorTests.cpp delete mode 100644 src/tests/unittests/RingBufferTests.cpp diff --git a/BUILD.gn b/BUILD.gn index d02dfa2809..ca4f82aa3b 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -216,8 +216,8 @@ source_set("libdawn_native_sources") { "src/dawn_native/ResourceHeap.h", "src/dawn_native/ResourceMemoryAllocation.cpp", "src/dawn_native/ResourceMemoryAllocation.h", - "src/dawn_native/RingBuffer.cpp", - "src/dawn_native/RingBuffer.h", + "src/dawn_native/RingBufferAllocator.cpp", + "src/dawn_native/RingBufferAllocator.h", "src/dawn_native/Sampler.cpp", "src/dawn_native/Sampler.h", "src/dawn_native/ShaderModule.cpp", @@ -769,7 +769,7 @@ test("dawn_unittests") { "src/tests/unittests/PerStageTests.cpp", "src/tests/unittests/RefCountedTests.cpp", "src/tests/unittests/ResultTests.cpp", - "src/tests/unittests/RingBufferTests.cpp", + "src/tests/unittests/RingBufferAllocatorTests.cpp", "src/tests/unittests/SerialMapTests.cpp", "src/tests/unittests/SerialQueueTests.cpp", "src/tests/unittests/ToBackendTests.cpp", diff --git a/src/dawn_native/Buffer.cpp b/src/dawn_native/Buffer.cpp index bd5b2fb161..21bc305fa1 100644 --- a/src/dawn_native/Buffer.cpp +++ b/src/dawn_native/Buffer.cpp @@ -167,9 +167,7 @@ namespace dawn_native { // error buffer. // TODO(enga): Suballocate and reuse memory from a larger staging buffer so we don't create // many small buffers. - DynamicUploader* uploader = nullptr; - DAWN_TRY_ASSIGN(uploader, GetDevice()->GetDynamicUploader()); - DAWN_TRY_ASSIGN(mStagingBuffer, uploader->CreateStagingBuffer(GetSize())); + DAWN_TRY_ASSIGN(mStagingBuffer, GetDevice()->CreateStagingBuffer(GetSize())); ASSERT(mStagingBuffer->GetMappedPointer() != nullptr); *mappedPointer = reinterpret_cast(mStagingBuffer->GetMappedPointer()); @@ -252,11 +250,11 @@ namespace dawn_native { } MaybeError BufferBase::SetSubDataImpl(uint32_t start, uint32_t count, const void* data) { - DynamicUploader* uploader = nullptr; - DAWN_TRY_ASSIGN(uploader, GetDevice()->GetDynamicUploader()); + DynamicUploader* uploader = GetDevice()->GetDynamicUploader(); UploadHandle uploadHandle; - DAWN_TRY_ASSIGN(uploadHandle, uploader->Allocate(count)); + DAWN_TRY_ASSIGN(uploadHandle, + uploader->Allocate(count, GetDevice()->GetPendingCommandSerial())); ASSERT(uploadHandle.mappedBuffer != nullptr); memcpy(uploadHandle.mappedBuffer, data, count); @@ -311,8 +309,7 @@ namespace dawn_native { ASSERT(mStagingBuffer); DAWN_TRY(GetDevice()->CopyFromStagingToBuffer(mStagingBuffer.get(), 0, this, 0, GetSize())); - DynamicUploader* uploader = nullptr; - DAWN_TRY_ASSIGN(uploader, GetDevice()->GetDynamicUploader()); + DynamicUploader* uploader = GetDevice()->GetDynamicUploader(); uploader->ReleaseStagingBuffer(std::move(mStagingBuffer)); return {}; diff --git a/src/dawn_native/Device.cpp b/src/dawn_native/Device.cpp index 7bc8011c02..1bdc792322 100644 --- a/src/dawn_native/Device.cpp +++ b/src/dawn_native/Device.cpp @@ -704,10 +704,7 @@ namespace dawn_native { // Other implementation details - ResultOrError DeviceBase::GetDynamicUploader() const { - if (mDynamicUploader->IsEmpty()) { - DAWN_TRY(mDynamicUploader->CreateAndAppendBuffer()); - } + DynamicUploader* DeviceBase::GetDynamicUploader() const { return mDynamicUploader.get(); } diff --git a/src/dawn_native/Device.h b/src/dawn_native/Device.h index cf69282376..c66826af4e 100644 --- a/src/dawn_native/Device.h +++ b/src/dawn_native/Device.h @@ -167,7 +167,7 @@ namespace dawn_native { uint64_t destinationOffset, uint64_t size) = 0; - ResultOrError GetDynamicUploader() const; + DynamicUploader* GetDynamicUploader() const; std::vector GetEnabledExtensions() const; std::vector GetTogglesUsed() const; diff --git a/src/dawn_native/DynamicUploader.cpp b/src/dawn_native/DynamicUploader.cpp index c716332429..b77cb94204 100644 --- a/src/dawn_native/DynamicUploader.cpp +++ b/src/dawn_native/DynamicUploader.cpp @@ -18,15 +18,9 @@ namespace dawn_native { - DynamicUploader::DynamicUploader(DeviceBase* device) : mDevice(device) { - } - - ResultOrError> DynamicUploader::CreateStagingBuffer( - size_t size) { - std::unique_ptr stagingBuffer; - DAWN_TRY_ASSIGN(stagingBuffer, mDevice->CreateStagingBuffer(size)); - DAWN_TRY(stagingBuffer->Initialize()); - return stagingBuffer; + DynamicUploader::DynamicUploader(DeviceBase* device, size_t size) : mDevice(device) { + mRingBuffers.emplace_back( + std::unique_ptr(new RingBuffer{nullptr, RingBufferAllocator(size)})); } void DynamicUploader::ReleaseStagingBuffer(std::unique_ptr stagingBuffer) { @@ -34,73 +28,78 @@ namespace dawn_native { mDevice->GetPendingCommandSerial()); } - MaybeError DynamicUploader::CreateAndAppendBuffer(size_t size) { - std::unique_ptr ringBuffer = std::make_unique(mDevice, size); - DAWN_TRY(ringBuffer->Initialize()); - mRingBuffers.emplace_back(std::move(ringBuffer)); - return {}; - } - - ResultOrError DynamicUploader::Allocate(uint32_t size) { + ResultOrError DynamicUploader::Allocate(size_t allocationSize, Serial serial) { // Note: Validation ensures size is already aligned. // First-fit: find next smallest buffer large enough to satisfy the allocation request. - RingBuffer* targetRingBuffer = GetLargestBuffer(); + RingBuffer* targetRingBuffer = mRingBuffers.back().get(); for (auto& ringBuffer : mRingBuffers) { + const RingBufferAllocator& ringBufferAllocator = ringBuffer->mAllocator; // Prevent overflow. - ASSERT(ringBuffer->GetSize() >= ringBuffer->GetUsedSize()); - const size_t remainingSize = ringBuffer->GetSize() - ringBuffer->GetUsedSize(); - if (size <= remainingSize) { + ASSERT(ringBufferAllocator.GetSize() >= ringBufferAllocator.GetUsedSize()); + const size_t remainingSize = + ringBufferAllocator.GetSize() - ringBufferAllocator.GetUsedSize(); + if (allocationSize <= remainingSize) { targetRingBuffer = ringBuffer.get(); break; } } - UploadHandle uploadHandle = UploadHandle{}; + size_t startOffset = kInvalidOffset; if (targetRingBuffer != nullptr) { - uploadHandle = targetRingBuffer->SubAllocate(size); + startOffset = targetRingBuffer->mAllocator.Allocate(allocationSize, serial); } // Upon failure, append a newly created (and much larger) ring buffer to fulfill the // request. - if (uploadHandle.mappedBuffer == nullptr) { + if (startOffset == kInvalidOffset) { // Compute the new max size (in powers of two to preserve alignment). - size_t newMaxSize = targetRingBuffer->GetSize() * 2; - while (newMaxSize < size) { + size_t newMaxSize = targetRingBuffer->mAllocator.GetSize() * 2; + while (newMaxSize < allocationSize) { newMaxSize *= 2; } // TODO(bryan.bernhart@intel.com): Fall-back to no sub-allocations should this fail. - DAWN_TRY(CreateAndAppendBuffer(newMaxSize)); - targetRingBuffer = GetLargestBuffer(); - uploadHandle = targetRingBuffer->SubAllocate(size); + mRingBuffers.emplace_back(std::unique_ptr( + new RingBuffer{nullptr, RingBufferAllocator(newMaxSize)})); + + targetRingBuffer = mRingBuffers.back().get(); + startOffset = targetRingBuffer->mAllocator.Allocate(allocationSize, serial); } - uploadHandle.stagingBuffer = targetRingBuffer->GetStagingBuffer(); + ASSERT(startOffset != kInvalidOffset); + + // Allocate the staging buffer backing the ringbuffer. + // Note: the first ringbuffer will be lazily created. + if (targetRingBuffer->mStagingBuffer == nullptr) { + std::unique_ptr stagingBuffer; + DAWN_TRY_ASSIGN(stagingBuffer, + mDevice->CreateStagingBuffer(targetRingBuffer->mAllocator.GetSize())); + targetRingBuffer->mStagingBuffer = std::move(stagingBuffer); + } + + ASSERT(targetRingBuffer->mStagingBuffer != nullptr); + + UploadHandle uploadHandle; + uploadHandle.stagingBuffer = targetRingBuffer->mStagingBuffer.get(); + uploadHandle.mappedBuffer = + static_cast(uploadHandle.stagingBuffer->GetMappedPointer()) + startOffset; + uploadHandle.startOffset = startOffset; return uploadHandle; } - void DynamicUploader::Tick(Serial lastCompletedSerial) { + void DynamicUploader::Deallocate(Serial lastCompletedSerial) { // Reclaim memory within the ring buffers by ticking (or removing requests no longer // in-flight). for (size_t i = 0; i < mRingBuffers.size(); ++i) { - mRingBuffers[i]->Tick(lastCompletedSerial); + mRingBuffers[i]->mAllocator.Deallocate(lastCompletedSerial); // Never erase the last buffer as to prevent re-creating smaller buffers // again. The last buffer is the largest. - if (mRingBuffers[i]->Empty() && i < mRingBuffers.size() - 1) { + if (mRingBuffers[i]->mAllocator.Empty() && i < mRingBuffers.size() - 1) { mRingBuffers.erase(mRingBuffers.begin() + i); } } mReleasedStagingBuffers.ClearUpTo(lastCompletedSerial); } - - RingBuffer* DynamicUploader::GetLargestBuffer() { - ASSERT(!mRingBuffers.empty()); - return mRingBuffers.back().get(); - } - - bool DynamicUploader::IsEmpty() const { - return mRingBuffers.empty(); - } } // namespace dawn_native \ No newline at end of file diff --git a/src/dawn_native/DynamicUploader.h b/src/dawn_native/DynamicUploader.h index 0caecb9919..f0d4510f15 100644 --- a/src/dawn_native/DynamicUploader.h +++ b/src/dawn_native/DynamicUploader.h @@ -16,37 +16,42 @@ #define DAWNNATIVE_DYNAMICUPLOADER_H_ #include "dawn_native/Forward.h" -#include "dawn_native/RingBuffer.h" +#include "dawn_native/RingBufferAllocator.h" +#include "dawn_native/StagingBuffer.h" // DynamicUploader is the front-end implementation used to manage multiple ring buffers for upload // usage. namespace dawn_native { + struct UploadHandle { + uint8_t* mappedBuffer = nullptr; + size_t startOffset = 0; + StagingBufferBase* stagingBuffer = nullptr; + }; + class DynamicUploader { public: - DynamicUploader(DeviceBase* device); + DynamicUploader(DeviceBase* device, size_t size = kBaseUploadBufferSize); ~DynamicUploader() = default; - // We add functions to Create/Release StagingBuffers to the DynamicUploader as there's + // We add functions to Release StagingBuffers to the DynamicUploader as there's // currently no place to track the allocated staging buffers such that they're freed after // pending commands are finished. This should be changed when better resource allocation is // implemented. - ResultOrError> CreateStagingBuffer(size_t size); void ReleaseStagingBuffer(std::unique_ptr stagingBuffer); - ResultOrError Allocate(uint32_t size); - void Tick(Serial lastCompletedSerial); - - RingBuffer* GetLargestBuffer(); - - MaybeError CreateAndAppendBuffer(size_t size = kBaseUploadBufferSize); - - bool IsEmpty() const; + ResultOrError Allocate(size_t allocationSize, Serial serial); + void Deallocate(Serial lastCompletedSerial); private: // TODO(bryan.bernhart@intel.com): Figure out this value. static constexpr size_t kBaseUploadBufferSize = 64000; + struct RingBuffer { + std::unique_ptr mStagingBuffer; + RingBufferAllocator mAllocator; + }; + std::vector> mRingBuffers; SerialQueue> mReleasedStagingBuffers; DeviceBase* mDevice; diff --git a/src/dawn_native/RingBuffer.cpp b/src/dawn_native/RingBufferAllocator.cpp similarity index 57% rename from src/dawn_native/RingBuffer.cpp rename to src/dawn_native/RingBufferAllocator.cpp index 90b92138bd..6cb94b7048 100644 --- a/src/dawn_native/RingBuffer.cpp +++ b/src/dawn_native/RingBufferAllocator.cpp @@ -12,13 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "dawn_native/RingBuffer.h" -#include "dawn_native/Device.h" +#include "dawn_native/RingBufferAllocator.h" -#include - -// Note: Current RingBuffer implementation uses two indices (start and end) to implement a circular -// queue. However, this approach defines a full queue when one element is still unused. +// Note: Current RingBufferAllocator implementation uses two indices (start and end) to implement a +// circular queue. However, this approach defines a full queue when one element is still unused. // // For example, [E,E,E,E] would be equivelent to [U,U,U,U]. // ^ ^ @@ -32,36 +29,10 @@ // TODO(bryan.bernhart@intel.com): Follow-up with ringbuffer optimization. namespace dawn_native { - static constexpr size_t INVALID_OFFSET = std::numeric_limits::max(); - - RingBuffer::RingBuffer(DeviceBase* device, size_t size) : mBufferSize(size), mDevice(device) { + RingBufferAllocator::RingBufferAllocator(size_t maxSize) : mMaxBlockSize(maxSize) { } - MaybeError RingBuffer::Initialize() { - DAWN_TRY_ASSIGN(mStagingBuffer, mDevice->CreateStagingBuffer(mBufferSize)); - DAWN_TRY(mStagingBuffer->Initialize()); - return {}; - } - - // Record allocations in a request when serial advances. - // This method has been split from Tick() for testing. - void RingBuffer::Track() { - if (mCurrentRequestSize == 0) - return; - const Serial currentSerial = mDevice->GetPendingCommandSerial(); - if (mInflightRequests.Empty() || currentSerial > mInflightRequests.LastSerial()) { - Request request; - request.endOffset = mUsedEndOffset; - request.size = mCurrentRequestSize; - - mInflightRequests.Enqueue(std::move(request), currentSerial); - mCurrentRequestSize = 0; // reset - } - } - - void RingBuffer::Tick(Serial lastCompletedSerial) { - Track(); - + void RingBufferAllocator::Deallocate(Serial lastCompletedSerial) { // Reclaim memory from previously recorded blocks. for (Request& request : mInflightRequests.IterateUpTo(lastCompletedSerial)) { mUsedStartOffset = request.endOffset; @@ -72,23 +43,18 @@ namespace dawn_native { mInflightRequests.ClearUpTo(lastCompletedSerial); } - size_t RingBuffer::GetSize() const { - return mBufferSize; + size_t RingBufferAllocator::GetSize() const { + return mMaxBlockSize; } - size_t RingBuffer::GetUsedSize() const { + size_t RingBufferAllocator::GetUsedSize() const { return mUsedSize; } - bool RingBuffer::Empty() const { + bool RingBufferAllocator::Empty() const { return mInflightRequests.Empty(); } - StagingBufferBase* RingBuffer::GetStagingBuffer() const { - ASSERT(mStagingBuffer != nullptr); - return mStagingBuffer.get(); - } - // Sub-allocate the ring-buffer by requesting a chunk of the specified size. // This is a serial-based resource scheme, the life-span of resources (and the allocations) get // tracked by GPU progress via serials. Memory can be reused by determining if the GPU has @@ -96,55 +62,55 @@ namespace dawn_native { // queue, which identifies an existing (or new) frames-worth of resources. Internally, the // ring-buffer maintains offsets of 3 "memory" states: Free, Reclaimed, and Used. This is done // in FIFO order as older frames would free resources before newer ones. - UploadHandle RingBuffer::SubAllocate(size_t allocSize) { - ASSERT(mStagingBuffer != nullptr); - + size_t RingBufferAllocator::Allocate(size_t allocationSize, Serial serial) { // Check if the buffer is full by comparing the used size. // If the buffer is not split where waste occurs (e.g. cannot fit new sub-alloc in front), a // subsequent sub-alloc could fail where the used size was previously adjusted to include // the wasted. - if (mUsedSize >= mBufferSize) - return UploadHandle{}; + if (allocationSize == 0 || mUsedSize >= mMaxBlockSize) { + return kInvalidOffset; + } - size_t startOffset = INVALID_OFFSET; + size_t startOffset = kInvalidOffset; // Check if the buffer is NOT split (i.e sub-alloc on ends) if (mUsedStartOffset <= mUsedEndOffset) { // Order is important (try to sub-alloc at end first). // This is due to FIFO order where sub-allocs are inserted from left-to-right (when not // wrapped). - if (mUsedEndOffset + allocSize <= mBufferSize) { + if (mUsedEndOffset + allocationSize <= mMaxBlockSize) { startOffset = mUsedEndOffset; - mUsedEndOffset += allocSize; - mUsedSize += allocSize; - mCurrentRequestSize += allocSize; - } else if (allocSize <= mUsedStartOffset) { // Try to sub-alloc at front. - // Count the space at front in the request size so that a subsequent + mUsedEndOffset += allocationSize; + mUsedSize += allocationSize; + mCurrentRequestSize += allocationSize; + } else if (allocationSize <= mUsedStartOffset) { // Try to sub-alloc at front. + // Count the space at the end so that a subsequent // sub-alloc cannot not succeed when the buffer is full. - const size_t requestSize = (mBufferSize - mUsedEndOffset) + allocSize; + const size_t requestSize = (mMaxBlockSize - mUsedEndOffset) + allocationSize; startOffset = 0; - mUsedEndOffset = allocSize; + mUsedEndOffset = allocationSize; mUsedSize += requestSize; mCurrentRequestSize += requestSize; } - } else if (mUsedEndOffset + allocSize <= + } else if (mUsedEndOffset + allocationSize <= mUsedStartOffset) { // Otherwise, buffer is split where sub-alloc must be // in-between. startOffset = mUsedEndOffset; - mUsedEndOffset += allocSize; - mUsedSize += allocSize; - mCurrentRequestSize += allocSize; + mUsedEndOffset += allocationSize; + mUsedSize += allocationSize; + mCurrentRequestSize += allocationSize; } - if (startOffset == INVALID_OFFSET) - return UploadHandle{}; + if (startOffset != kInvalidOffset) { + Request request; + request.endOffset = mUsedEndOffset; + request.size = mCurrentRequestSize; - UploadHandle uploadHandle; - uploadHandle.mappedBuffer = - static_cast(mStagingBuffer->GetMappedPointer()) + startOffset; - uploadHandle.startOffset = startOffset; + mInflightRequests.Enqueue(std::move(request), serial); + mCurrentRequestSize = 0; // reset + } - return uploadHandle; + return startOffset; } } // namespace dawn_native diff --git a/src/dawn_native/RingBuffer.h b/src/dawn_native/RingBufferAllocator.h similarity index 62% rename from src/dawn_native/RingBuffer.h rename to src/dawn_native/RingBufferAllocator.h index dbc51bcea9..630da98643 100644 --- a/src/dawn_native/RingBuffer.h +++ b/src/dawn_native/RingBufferAllocator.h @@ -12,46 +12,32 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef DAWNNATIVE_RINGBUFFER_H_ -#define DAWNNATIVE_RINGBUFFER_H_ +#ifndef DAWNNATIVE_RINGBUFFERALLOCATOR_H_ +#define DAWNNATIVE_RINGBUFFERALLOCATOR_H_ #include "common/SerialQueue.h" -#include "dawn_native/StagingBuffer.h" +#include #include -// RingBuffer is the front-end implementation used to manage a ring buffer in GPU memory. +// RingBufferAllocator is the front-end implementation used to manage a ring buffer in GPU memory. namespace dawn_native { - struct UploadHandle { - uint8_t* mappedBuffer = nullptr; - size_t startOffset = 0; - StagingBufferBase* stagingBuffer = nullptr; - }; + static constexpr size_t kInvalidOffset = std::numeric_limits::max(); - class DeviceBase; - - class RingBuffer { + class RingBufferAllocator { public: - RingBuffer(DeviceBase* device, size_t size); - ~RingBuffer() = default; + RingBufferAllocator(size_t maxSize); + ~RingBufferAllocator() = default; - MaybeError Initialize(); + size_t Allocate(size_t allocationSize, Serial serial); + void Deallocate(Serial lastCompletedSerial); - UploadHandle SubAllocate(size_t requestedSize); - - void Tick(Serial lastCompletedSerial); size_t GetSize() const; bool Empty() const; size_t GetUsedSize() const; - StagingBufferBase* GetStagingBuffer() const; - - // Seperated for testing. - void Track(); private: - std::unique_ptr mStagingBuffer; - struct Request { size_t endOffset; size_t size; @@ -62,13 +48,11 @@ namespace dawn_native { size_t mUsedEndOffset = 0; // Tail of used sub-alloc requests (in bytes). size_t mUsedStartOffset = 0; // Head of used sub-alloc requests (in bytes). - size_t mBufferSize = 0; // Max size of the ring buffer (in bytes). + size_t mMaxBlockSize = 0; // Max size of the ring buffer (in bytes). size_t mUsedSize = 0; // Size of the sub-alloc requests (in bytes) of the ring buffer. size_t mCurrentRequestSize = 0; // Size of the sub-alloc requests (in bytes) of the current serial. - - DeviceBase* mDevice; }; } // namespace dawn_native -#endif // DAWNNATIVE_RINGBUFFER_H_ \ No newline at end of file +#endif // DAWNNATIVE_RINGBUFFERALLOCATOR_H_ \ No newline at end of file diff --git a/src/dawn_native/d3d12/DeviceD3D12.cpp b/src/dawn_native/d3d12/DeviceD3D12.cpp index c4e5e96705..d9bb0a805c 100644 --- a/src/dawn_native/d3d12/DeviceD3D12.cpp +++ b/src/dawn_native/d3d12/DeviceD3D12.cpp @@ -216,8 +216,8 @@ namespace dawn_native { namespace d3d12 { mCompletedSerial = mFence->GetCompletedValue(); // Uploader should tick before the resource allocator - // as it enqueues resources to be released. - mDynamicUploader->Tick(mCompletedSerial); + // as it enqueued resources to be released. + mDynamicUploader->Deallocate(mCompletedSerial); mResourceAllocator->Tick(mCompletedSerial); mCommandAllocatorManager->Tick(mCompletedSerial); @@ -319,6 +319,7 @@ namespace dawn_native { namespace d3d12 { ResultOrError> Device::CreateStagingBuffer(size_t size) { std::unique_ptr stagingBuffer = std::make_unique(size, this); + DAWN_TRY(stagingBuffer->Initialize()); return std::move(stagingBuffer); } diff --git a/src/dawn_native/d3d12/TextureD3D12.cpp b/src/dawn_native/d3d12/TextureD3D12.cpp index 97843588a9..e92345cda0 100644 --- a/src/dawn_native/d3d12/TextureD3D12.cpp +++ b/src/dawn_native/d3d12/TextureD3D12.cpp @@ -538,10 +538,10 @@ namespace dawn_native { namespace d3d12 { return DAWN_OUT_OF_MEMORY_ERROR("Unable to allocate buffer."); } uint32_t bufferSize = static_cast(bufferSize64); - DynamicUploader* uploader = nullptr; - DAWN_TRY_ASSIGN(uploader, device->GetDynamicUploader()); + DynamicUploader* uploader = device->GetDynamicUploader(); UploadHandle uploadHandle; - DAWN_TRY_ASSIGN(uploadHandle, uploader->Allocate(bufferSize)); + DAWN_TRY_ASSIGN(uploadHandle, + uploader->Allocate(bufferSize, device->GetPendingCommandSerial())); std::fill(reinterpret_cast(uploadHandle.mappedBuffer), reinterpret_cast(uploadHandle.mappedBuffer + bufferSize), clearColor); diff --git a/src/dawn_native/metal/DeviceMTL.mm b/src/dawn_native/metal/DeviceMTL.mm index 52fd864c1a..046ed13d47 100644 --- a/src/dawn_native/metal/DeviceMTL.mm +++ b/src/dawn_native/metal/DeviceMTL.mm @@ -154,7 +154,7 @@ namespace dawn_native { namespace metal { void Device::TickImpl() { Serial completedSerial = GetCompletedCommandSerial(); - mDynamicUploader->Tick(completedSerial); + mDynamicUploader->Deallocate(completedSerial); mMapTracker->Tick(completedSerial); if (mPendingCommands != nil) { @@ -239,6 +239,7 @@ namespace dawn_native { namespace metal { ResultOrError> Device::CreateStagingBuffer(size_t size) { std::unique_ptr stagingBuffer = std::make_unique(size, this); + DAWN_TRY(stagingBuffer->Initialize()); return std::move(stagingBuffer); } diff --git a/src/dawn_native/null/DeviceNull.cpp b/src/dawn_native/null/DeviceNull.cpp index 44e62db51d..303445a06e 100644 --- a/src/dawn_native/null/DeviceNull.cpp +++ b/src/dawn_native/null/DeviceNull.cpp @@ -150,6 +150,7 @@ namespace dawn_native { namespace null { ResultOrError> Device::CreateStagingBuffer(size_t size) { std::unique_ptr stagingBuffer = std::make_unique(size, this); + DAWN_TRY(stagingBuffer->Initialize()); return std::move(stagingBuffer); } diff --git a/src/dawn_native/null/DeviceNull.h b/src/dawn_native/null/DeviceNull.h index 640ff588c7..232dff82ba 100644 --- a/src/dawn_native/null/DeviceNull.h +++ b/src/dawn_native/null/DeviceNull.h @@ -26,7 +26,7 @@ #include "dawn_native/PipelineLayout.h" #include "dawn_native/Queue.h" #include "dawn_native/RenderPipeline.h" -#include "dawn_native/RingBuffer.h" +#include "dawn_native/RingBufferAllocator.h" #include "dawn_native/Sampler.h" #include "dawn_native/ShaderModule.h" #include "dawn_native/StagingBuffer.h" diff --git a/src/dawn_native/vulkan/DeviceVk.cpp b/src/dawn_native/vulkan/DeviceVk.cpp index 1a66e2b390..bd63b11d63 100644 --- a/src/dawn_native/vulkan/DeviceVk.cpp +++ b/src/dawn_native/vulkan/DeviceVk.cpp @@ -217,7 +217,7 @@ namespace dawn_native { namespace vulkan { // Uploader should tick before the resource allocator // as it enqueues resources to be released. - mDynamicUploader->Tick(mCompletedSerial); + mDynamicUploader->Deallocate(mCompletedSerial); mMemoryAllocator->Tick(mCompletedSerial); @@ -561,6 +561,7 @@ namespace dawn_native { namespace vulkan { ResultOrError> Device::CreateStagingBuffer(size_t size) { std::unique_ptr stagingBuffer = std::make_unique(size, this); + DAWN_TRY(stagingBuffer->Initialize()); return std::move(stagingBuffer); } diff --git a/src/dawn_native/vulkan/TextureVk.cpp b/src/dawn_native/vulkan/TextureVk.cpp index 43a18c5dc7..1f29ea079b 100644 --- a/src/dawn_native/vulkan/TextureVk.cpp +++ b/src/dawn_native/vulkan/TextureVk.cpp @@ -687,10 +687,10 @@ namespace dawn_native { namespace vulkan { return DAWN_OUT_OF_MEMORY_ERROR("Unable to allocate buffer."); } uint32_t bufferSize = static_cast(bufferSize64); - DynamicUploader* uploader = nullptr; - DAWN_TRY_ASSIGN(uploader, device->GetDynamicUploader()); + DynamicUploader* uploader = device->GetDynamicUploader(); UploadHandle uploadHandle; - DAWN_TRY_ASSIGN(uploadHandle, uploader->Allocate(bufferSize)); + DAWN_TRY_ASSIGN(uploadHandle, + uploader->Allocate(bufferSize, device->GetPendingCommandSerial())); std::fill(reinterpret_cast(uploadHandle.mappedBuffer), reinterpret_cast(uploadHandle.mappedBuffer + bufferSize), clearColor); diff --git a/src/tests/unittests/RingBufferAllocatorTests.cpp b/src/tests/unittests/RingBufferAllocatorTests.cpp new file mode 100644 index 0000000000..cc381cc89a --- /dev/null +++ b/src/tests/unittests/RingBufferAllocatorTests.cpp @@ -0,0 +1,161 @@ +// Copyright 2018 The Dawn Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "dawn_native/RingBufferAllocator.h" + +using namespace dawn_native; + +// Number of basic tests for Ringbuffer +TEST(RingBufferAllocatorTests, BasicTest) { + constexpr size_t sizeInBytes = 64000; + RingBufferAllocator allocator(sizeInBytes); + + // Ensure no requests exist on empty buffer. + EXPECT_TRUE(allocator.Empty()); + + ASSERT_EQ(allocator.GetSize(), sizeInBytes); + + // Ensure failure upon sub-allocating an oversized request. + ASSERT_EQ(allocator.Allocate(sizeInBytes + 1, 0), kInvalidOffset); + + // Fill the entire buffer with two requests of equal size. + ASSERT_EQ(allocator.Allocate(sizeInBytes / 2, 1), 0u); + ASSERT_EQ(allocator.Allocate(sizeInBytes / 2, 2), 32000u); + + // Ensure the buffer is full. + ASSERT_EQ(allocator.Allocate(1, 3), kInvalidOffset); +} + +// Tests that several ringbuffer allocations do not fail. +TEST(RingBufferAllocatorTests, RingBufferManyAlloc) { + constexpr size_t maxNumOfFrames = 64000; + constexpr size_t frameSizeInBytes = 4; + + RingBufferAllocator allocator(maxNumOfFrames * frameSizeInBytes); + + size_t offset = 0; + for (size_t i = 0; i < maxNumOfFrames; ++i) { + offset = allocator.Allocate(frameSizeInBytes, i); + ASSERT_EQ(offset, i * frameSizeInBytes); + } +} + +// Tests ringbuffer sub-allocations of the same serial are correctly tracked. +TEST(RingBufferAllocatorTests, AllocInSameFrame) { + constexpr size_t maxNumOfFrames = 3; + constexpr size_t frameSizeInBytes = 4; + + RingBufferAllocator allocator(maxNumOfFrames * frameSizeInBytes); + + // F1 + // [xxxx|--------] + size_t offset = allocator.Allocate(frameSizeInBytes, 1); + + // F1 F2 + // [xxxx|xxxx|----] + + offset = allocator.Allocate(frameSizeInBytes, 2); + + // F1 F2 + // [xxxx|xxxxxxxx] + + offset = allocator.Allocate(frameSizeInBytes, 2); + + ASSERT_EQ(offset, 8u); + ASSERT_EQ(allocator.GetUsedSize(), frameSizeInBytes * 3); + + allocator.Deallocate(2); + + ASSERT_EQ(allocator.GetUsedSize(), 0u); + EXPECT_TRUE(allocator.Empty()); +} + +// Tests ringbuffer sub-allocation at various offsets. +TEST(RingBufferAllocatorTests, RingBufferSubAlloc) { + constexpr size_t maxNumOfFrames = 10; + constexpr size_t frameSizeInBytes = 4; + + RingBufferAllocator allocator(maxNumOfFrames * frameSizeInBytes); + + // Sub-alloc the first eight frames. + Serial serial = 1; + for (size_t i = 0; i < 8; ++i) { + allocator.Allocate(frameSizeInBytes, serial); + serial += 1; + } + + // Each frame corrresponds to the serial number (for simplicity). + // + // F1 F2 F3 F4 F5 F6 F7 F8 + // [xxxx|xxxx|xxxx|xxxx|xxxx|xxxx|xxxx|xxxx|--------] + // + + // Ensure an oversized allocation fails (only 8 bytes left) + ASSERT_EQ(allocator.Allocate(frameSizeInBytes * 3, serial + 1), kInvalidOffset); + ASSERT_EQ(allocator.GetUsedSize(), frameSizeInBytes * 8); + + // Reclaim the first 3 frames. + allocator.Deallocate(3); + + // F4 F5 F6 F7 F8 + // [------------|xxxx|xxxx|xxxx|xxxx|xxxx|--------] + // + ASSERT_EQ(allocator.GetUsedSize(), frameSizeInBytes * 5); + + // Re-try the over-sized allocation. + size_t offset = allocator.Allocate(frameSizeInBytes * 3, serial); + + // F9 F4 F5 F6 F7 F8 + // [xxxxxxxxxxxx|xxxx|xxxx|xxxx|xxxx|xxxx|xxxxxxxx] + // ^^^^^^^^ wasted + + // In this example, Deallocate(8) could not reclaim the wasted bytes. The wasted bytes + // were added to F9's sub-allocation. + // TODO(bryan.bernhart@intel.com): Decide if Deallocate(8) should free these wasted bytes. + + ASSERT_EQ(offset, 0u); + ASSERT_EQ(allocator.GetUsedSize(), frameSizeInBytes * maxNumOfFrames); + + // Ensure we are full. + ASSERT_EQ(allocator.Allocate(frameSizeInBytes, serial + 1), kInvalidOffset); + + // Reclaim the next two frames. + allocator.Deallocate(5); + + // F9 F4 F5 F6 F7 F8 + // [xxxxxxxxxxxx|----|----|xxxx|xxxx|xxxx|xxxxxxxx] + // + ASSERT_EQ(allocator.GetUsedSize(), frameSizeInBytes * 8); + + // Sub-alloc the chunk in the middle. + serial += 1; + offset = allocator.Allocate(frameSizeInBytes * 2, serial); + + ASSERT_EQ(offset, frameSizeInBytes * 3); + ASSERT_EQ(allocator.GetUsedSize(), frameSizeInBytes * maxNumOfFrames); + + // F9 F10 F6 F7 F8 + // [xxxxxxxxxxxx|xxxxxxxxx|xxxx|xxxx|xxxx|xxxxxxxx] + // + + // Ensure we are full. + ASSERT_EQ(allocator.Allocate(frameSizeInBytes, serial + 1), kInvalidOffset); + + // Reclaim all. + allocator.Deallocate(maxNumOfFrames); + + EXPECT_TRUE(allocator.Empty()); +} diff --git a/src/tests/unittests/RingBufferTests.cpp b/src/tests/unittests/RingBufferTests.cpp deleted file mode 100644 index 92ad835d06..0000000000 --- a/src/tests/unittests/RingBufferTests.cpp +++ /dev/null @@ -1,201 +0,0 @@ -// Copyright 2018 The Dawn Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "dawn_native/null/DeviceNull.h" - -using namespace dawn_native; - -namespace { - - size_t ValidateValidUploadHandle(const UploadHandle& uploadHandle) { - ASSERT(uploadHandle.mappedBuffer != nullptr); - return uploadHandle.startOffset; - } - - void ValidateInvalidUploadHandle(const UploadHandle& uploadHandle) { - ASSERT_EQ(uploadHandle.mappedBuffer, nullptr); - } -} // namespace - -class RingBufferTests : public testing::Test { - protected: - void SetUp() override { - // TODO(bryan.bernhart@intel.com): Create this device through the adapter. - mDevice = std::make_unique(/*adapter*/ nullptr, /*deviceDescriptor*/ nullptr); - } - - null::Device* GetDevice() const { - return mDevice.get(); - } - - std::unique_ptr CreateRingBuffer(size_t size) { - std::unique_ptr ringBuffer = std::make_unique(mDevice.get(), size); - DAWN_UNUSED(ringBuffer->Initialize()); - return ringBuffer; - } - - private: - std::unique_ptr mDevice; -}; - -// Number of basic tests for Ringbuffer -TEST_F(RingBufferTests, BasicTest) { - constexpr size_t sizeInBytes = 64000; - std::unique_ptr buffer = CreateRingBuffer(sizeInBytes); - - // Ensure no requests exist on empty buffer. - EXPECT_TRUE(buffer->Empty()); - - ASSERT_EQ(buffer->GetSize(), sizeInBytes); - - // Ensure failure upon sub-allocating an oversized request. - ValidateInvalidUploadHandle(buffer->SubAllocate(sizeInBytes + 1)); - - // Fill the entire buffer with two requests of equal size. - ValidateValidUploadHandle(buffer->SubAllocate(sizeInBytes / 2)); - ValidateValidUploadHandle(buffer->SubAllocate(sizeInBytes / 2)); - - // Ensure the buffer is full. - ValidateInvalidUploadHandle(buffer->SubAllocate(1)); -} - -// Tests that several ringbuffer allocations do not fail. -TEST_F(RingBufferTests, RingBufferManyAlloc) { - constexpr size_t maxNumOfFrames = 64000; - constexpr size_t frameSizeInBytes = 4; - - std::unique_ptr buffer = CreateRingBuffer(maxNumOfFrames * frameSizeInBytes); - - size_t offset = 0; - for (size_t i = 0; i < maxNumOfFrames; ++i) { - offset = ValidateValidUploadHandle(buffer->SubAllocate(frameSizeInBytes)); - GetDevice()->Tick(); - ASSERT_EQ(offset, i * frameSizeInBytes); - } -} - -// Tests ringbuffer sub-allocations of the same serial are correctly tracked. -TEST_F(RingBufferTests, AllocInSameFrame) { - constexpr size_t maxNumOfFrames = 3; - constexpr size_t frameSizeInBytes = 4; - - std::unique_ptr buffer = CreateRingBuffer(maxNumOfFrames * frameSizeInBytes); - - // F1 - // [xxxx|--------] - - ValidateValidUploadHandle(buffer->SubAllocate(frameSizeInBytes)); - GetDevice()->Tick(); - - // F1 F2 - // [xxxx|xxxx|----] - - ValidateValidUploadHandle(buffer->SubAllocate(frameSizeInBytes)); - - // F1 F2 - // [xxxx|xxxxxxxx] - - size_t offset = ValidateValidUploadHandle(buffer->SubAllocate(frameSizeInBytes)); - - ASSERT_EQ(offset, 8u); - ASSERT_EQ(buffer->GetUsedSize(), frameSizeInBytes * 3); - - buffer->Tick(1); - - // Used size does not change as previous sub-allocations were not tracked. - ASSERT_EQ(buffer->GetUsedSize(), frameSizeInBytes * 3); - - buffer->Tick(2); - - ASSERT_EQ(buffer->GetUsedSize(), 0u); - EXPECT_TRUE(buffer->Empty()); -} - -// Tests ringbuffer sub-allocation at various offsets. -TEST_F(RingBufferTests, RingBufferSubAlloc) { - constexpr size_t maxNumOfFrames = 10; - constexpr size_t frameSizeInBytes = 4; - - std::unique_ptr buffer = CreateRingBuffer(maxNumOfFrames * frameSizeInBytes); - - // Sub-alloc the first eight frames. - for (size_t i = 0; i < 8; ++i) { - ValidateValidUploadHandle(buffer->SubAllocate(frameSizeInBytes)); - buffer->Track(); - GetDevice()->Tick(); - } - - // Each frame corrresponds to the serial number (for simplicity). - // - // F1 F2 F3 F4 F5 F6 F7 F8 - // [xxxx|xxxx|xxxx|xxxx|xxxx|xxxx|xxxx|xxxx|--------] - // - - // Ensure an oversized allocation fails (only 8 bytes left) - ValidateInvalidUploadHandle(buffer->SubAllocate(frameSizeInBytes * 3)); - ASSERT_EQ(buffer->GetUsedSize(), frameSizeInBytes * 8); - - // Reclaim the first 3 frames. - buffer->Tick(3); - - // F4 F5 F6 F7 F8 - // [------------|xxxx|xxxx|xxxx|xxxx|xxxx|--------] - // - ASSERT_EQ(buffer->GetUsedSize(), frameSizeInBytes * 5); - - // Re-try the over-sized allocation. - size_t offset = ValidateValidUploadHandle(buffer->SubAllocate(frameSizeInBytes * 3)); - - // F9 F4 F5 F6 F7 F8 - // [xxxxxxxxxxxx|xxxx|xxxx|xxxx|xxxx|xxxx|xxxxxxxx] - // ^^^^^^^^ wasted - - // In this example, Tick(8) could not reclaim the wasted bytes. The wasted bytes - // were add to F9's sub-allocation. - // TODO(bryan.bernhart@intel.com): Decide if Tick(8) should free these wasted bytes. - - ASSERT_EQ(offset, 0u); - ASSERT_EQ(buffer->GetUsedSize(), frameSizeInBytes * maxNumOfFrames); - - // Ensure we are full. - ValidateInvalidUploadHandle(buffer->SubAllocate(frameSizeInBytes)); - - // Reclaim the next two frames. - buffer->Tick(5); - - // F9 F4 F5 F6 F7 F8 - // [xxxxxxxxxxxx|----|----|xxxx|xxxx|xxxx|xxxxxxxx] - // - ASSERT_EQ(buffer->GetUsedSize(), frameSizeInBytes * 8); - - // Sub-alloc the chunk in the middle. - offset = ValidateValidUploadHandle(buffer->SubAllocate(frameSizeInBytes * 2)); - - ASSERT_EQ(offset, frameSizeInBytes * 3); - ASSERT_EQ(buffer->GetUsedSize(), frameSizeInBytes * maxNumOfFrames); - - // F9 F6 F7 F8 - // [xxxxxxxxxxxx|xxxx|xxxx|xxxx|xxxx|xxxx|xxxxxxxx] - // ^^^^^^^^^ untracked - - // Ensure we are full. - ValidateInvalidUploadHandle(buffer->SubAllocate(frameSizeInBytes)); - - // Reclaim all. - buffer->Tick(maxNumOfFrames); - - EXPECT_TRUE(buffer->Empty()); -}