encounter
/
dawn-cmake
mirror of https://github.com/encounter/dawn-cmake.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Vulkan: Reuse descriptor pools 

This changes puts descriptor pools in the BindGroupLayout so that they
can be reused between descriptors with the same layout.

This makes the DrawCallPerf.Run/Vulkan_NoReuseBindGroups benchmark go
from 1400ns to 800ns per run.

BUG=dawn::256

Change-Id: Ia9baf7f998d9ff4d552e255c80069b67c6a9ac40
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/12920
Commit-Queue: Corentin Wallez <cwallez@chromium.org>
Reviewed-by: Austin Eng <enga@chromium.org>
This commit is contained in:
Corentin Wallez 2019-11-05 15:44:05 +00:00 committed by Commit Bot service account
parent dbf9f7c39c
commit 0558f99119
11 changed files with 255 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
BUILD.gn
View File

@ -426,6 +426,8 @@ source_set("libdawn_native_sources") {
"src/dawn_native/vulkan/CommandRecordingContext.h",
"src/dawn_native/vulkan/ComputePipelineVk.cpp",
"src/dawn_native/vulkan/ComputePipelineVk.h",
"src/dawn_native/vulkan/DescriptorSetService.cpp",
"src/dawn_native/vulkan/DescriptorSetService.h",
"src/dawn_native/vulkan/DeviceVk.cpp",
"src/dawn_native/vulkan/DeviceVk.h",
"src/dawn_native/vulkan/ExternalHandle.h",

2
src/dawn_native/BindGroup.cpp
View File

@ -221,7 +221,7 @@ namespace dawn_native {
return new BindGroupBase(device, ObjectBase::kError);
}
const BindGroupLayoutBase* BindGroupBase::GetLayout() const {
BindGroupLayoutBase* BindGroupBase::GetLayout() {
ASSERT(!IsError());
return mLayout.Get();
}

2
src/dawn_native/BindGroup.h
View File

@ -44,7 +44,7 @@ namespace dawn_native {
static BindGroupBase* MakeError(DeviceBase* device);
const BindGroupLayoutBase* GetLayout() const;
BindGroupLayoutBase* GetLayout();
BufferBinding GetBindingAsBufferBinding(size_t binding);
SamplerBase* GetBindingAsSampler(size_t binding);
TextureViewBase* GetBindingAsTextureView(size_t binding);

158
src/dawn_native/vulkan/BindGroupLayoutVk.cpp
View File

@ -15,9 +15,13 @@
#include "dawn_native/vulkan/BindGroupLayoutVk.h"
#include "common/BitSetIterator.h"
#include "dawn_native/vulkan/DescriptorSetService.h"
#include "dawn_native/vulkan/DeviceVk.h"
#include "dawn_native/vulkan/FencedDeleter.h"
#include "dawn_native/vulkan/VulkanError.h"
#include <map>
namespace dawn_native { namespace vulkan {
namespace {
@ -72,7 +76,7 @@ namespace dawn_native { namespace vulkan {
}
MaybeError BindGroupLayout::Initialize() {
const auto& info = GetBindingInfo();
const LayoutBindingInfo& info = GetBindingInfo();
// Compute the bindings that will be chained in the DescriptorSetLayout create info. We add
// one entry per binding set. This might be optimized by computing continuous ranges of
@ -80,13 +84,13 @@ namespace dawn_native { namespace vulkan {
uint32_t numBindings = 0;
std::array<VkDescriptorSetLayoutBinding, kMaxBindingsPerGroup> bindings;
for (uint32_t bindingIndex : IterateBitSet(info.mask)) {
auto& binding = bindings[numBindings];
binding.binding = bindingIndex;
binding.descriptorType =
VkDescriptorSetLayoutBinding* binding = &bindings[numBindings];
binding->binding = bindingIndex;
binding->descriptorType =
VulkanDescriptorType(info.types[bindingIndex], info.hasDynamicOffset[bindingIndex]);
binding.descriptorCount = 1;
binding.stageFlags = VulkanShaderStageFlags(info.visibilities[bindingIndex]);
binding.pImmutableSamplers = nullptr;
binding->descriptorCount = 1;
binding->stageFlags = VulkanShaderStageFlags(info.visibilities[bindingIndex]);
binding->pImmutableSamplers = nullptr;
numBindings++;
}
@ -99,73 +103,113 @@ namespace dawn_native { namespace vulkan {
createInfo.pBindings = bindings.data();
Device* device = ToBackend(GetDevice());
return CheckVkSuccess(device->fn.CreateDescriptorSetLayout(device->GetVkDevice(),
&createInfo, nullptr, &mHandle),
"CreateDescriptorSetLayout");
DAWN_TRY(CheckVkSuccess(device->fn.CreateDescriptorSetLayout(
device->GetVkDevice(), &createInfo, nullptr, &mHandle),
"CreateDescriptorSetLayout"));
// Compute the size of descriptor pools used for this layout.
std::map<VkDescriptorType, uint32_t> descriptorCountPerType;
for (uint32_t bindingIndex : IterateBitSet(info.mask)) {
VkDescriptorType vulkanType =
VulkanDescriptorType(info.types[bindingIndex], info.hasDynamicOffset[bindingIndex]);
// map::operator[] will return 0 if the key doesn't exist.
descriptorCountPerType[vulkanType]++;
}
mPoolSizes.reserve(descriptorCountPerType.size());
for (const auto& it : descriptorCountPerType) {
mPoolSizes.push_back(VkDescriptorPoolSize{it.first, it.second});
}
return {};
}
BindGroupLayout::~BindGroupLayout() {
Device* device = ToBackend(GetDevice());
// DescriptorSetLayout aren't used by execution on the GPU and can be deleted at any time,
// so we destroy mHandle immediately instead of using the FencedDeleter
if (mHandle != VK_NULL_HANDLE) {
Device* device = ToBackend(GetDevice());
device->fn.DestroyDescriptorSetLayout(device->GetVkDevice(), mHandle, nullptr);
mHandle = VK_NULL_HANDLE;
}
FencedDeleter* deleter = device->GetFencedDeleter();
for (const SingleDescriptorSetAllocation& allocation : mAllocations) {
deleter->DeleteWhenUnused(allocation.pool);
}
mAllocations.clear();
}
VkDescriptorSetLayout BindGroupLayout::GetHandle() const {
return mHandle;
}
BindGroupLayout::PoolSizeSpec BindGroupLayout::ComputePoolSizes(uint32_t* numPoolSizes) const {
uint32_t numSizes = 0;
PoolSizeSpec result{};
ResultOrError<DescriptorSetAllocation> BindGroupLayout::AllocateOneSet() {
Device* device = ToBackend(GetDevice());
// Defines an array and indices into it that will contain for each sampler type at which
// position it is in the PoolSizeSpec, or -1 if it isn't present yet.
enum DescriptorType {
UNIFORM_BUFFER,
SAMPLER,
SAMPLED_IMAGE,
STORAGE_BUFFER,
MAX_TYPE,
};
static_assert(MAX_TYPE == kMaxPoolSizesNeeded, "");
auto ToDescriptorType = [](wgpu::BindingType type) -> DescriptorType {
switch (type) {
case wgpu::BindingType::UniformBuffer:
return UNIFORM_BUFFER;
case wgpu::BindingType::Sampler:
return SAMPLER;
case wgpu::BindingType::SampledTexture:
return SAMPLED_IMAGE;
case wgpu::BindingType::StorageBuffer:
return STORAGE_BUFFER;
default:
UNREACHABLE();
}
};
std::array<int, MAX_TYPE> descriptorTypeIndex;
descriptorTypeIndex.fill(-1);
const auto& info = GetBindingInfo();
for (uint32_t bindingIndex : IterateBitSet(info.mask)) {
DescriptorType type = ToDescriptorType(info.types[bindingIndex]);
if (descriptorTypeIndex[type] == -1) {
descriptorTypeIndex[type] = numSizes;
result[numSizes].type = VulkanDescriptorType(info.types[bindingIndex],
info.hasDynamicOffset[bindingIndex]);
result[numSizes].descriptorCount = 1;
numSizes++;
} else {
result[descriptorTypeIndex[type]].descriptorCount++;
}
// Reuse a previous allocation if available.
if (!mAvailableAllocations.empty()) {
size_t index = mAvailableAllocations.back();
mAvailableAllocations.pop_back();
return {{index, mAllocations[index].set}};
}
*numPoolSizes = numSizes;
return result;
// Create a pool to hold our descriptor set.
// TODO(cwallez@chromium.org): This horribly inefficient, have more than one descriptor
// set per pool.
VkDescriptorPoolCreateInfo createInfo;
createInfo.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO;
createInfo.pNext = nullptr;
createInfo.flags = 0;
createInfo.maxSets = 1;
createInfo.poolSizeCount = static_cast<uint32_t>(mPoolSizes.size());
createInfo.pPoolSizes = mPoolSizes.data();
VkDescriptorPool descriptorPool;
DAWN_TRY(CheckVkSuccess(device->fn.CreateDescriptorPool(device->GetVkDevice(), &createInfo,
nullptr, &descriptorPool),
"CreateDescriptorPool"));
// Allocate our single set.
VkDescriptorSetAllocateInfo allocateInfo;
allocateInfo.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO;
allocateInfo.pNext = nullptr;
allocateInfo.descriptorPool = descriptorPool;
allocateInfo.descriptorSetCount = 1;
allocateInfo.pSetLayouts = &mHandle;
VkDescriptorSet descriptorSet;
MaybeError result = CheckVkSuccess(
device->fn.AllocateDescriptorSets(device->GetVkDevice(), &allocateInfo, &descriptorSet),
"AllocateDescriptorSets");
if (result.IsError()) {
// On an error we can destroy the pool immediately because no command references it.
device->fn.DestroyDescriptorPool(device->GetVkDevice(), descriptorPool, nullptr);
return result.AcquireError();
}
mAllocations.push_back({descriptorPool, descriptorSet});
return {{mAllocations.size() - 1, descriptorSet}};
}
void BindGroupLayout::Deallocate(DescriptorSetAllocation* allocation) {
// We can't reuse the descriptor set right away because the Vulkan spec says in the
// documentation for vkCmdBindDescriptorSets that the set may be consumed any time between
// host execution of the command and the end of the draw/dispatch.
ToBackend(GetDevice())
->GetDescriptorSetService()
->AddDeferredDeallocation(this, allocation->index);
// Clear the content of allocation so that use after frees are more visible.
*allocation = {};
}
void BindGroupLayout::FinishDeallocation(size_t index) {
mAvailableAllocations.push_back(index);
}
}} // namespace dawn_native::vulkan

38
src/dawn_native/vulkan/BindGroupLayoutVk.h
View File

@ -19,12 +19,32 @@
#include "common/vulkan_platform.h"
#include <vector>
namespace dawn_native { namespace vulkan {
class Device;
VkDescriptorType VulkanDescriptorType(wgpu::BindingType type, bool isDynamic);
// Contains a descriptor set along with data necessary to track its allocation.
struct DescriptorSetAllocation {
size_t index = 0;
VkDescriptorSet set = VK_NULL_HANDLE;
};
// In Vulkan descriptor pools have to be sized to an exact number of descriptors. This means
// it's hard to have something where we can mix different types of descriptor sets because
// we don't know if their vector of number of descriptors will be similar.
//
// That's why that in addition to containing the VkDescriptorSetLayout to create
// VkDescriptorSets for its bindgroups, the layout also acts as an allocator for the descriptor
// sets.
//
// The allocations is done with one pool per descriptor set, which is inefficient, but at least
// the pools are reused when no longer used. Minimizing the number of descriptor pool allocation
// is important because creating them can incur GPU memory allocation which is usually an
// expensive syscall.
class BindGroupLayout : public BindGroupLayoutBase {
public:
static ResultOrError<BindGroupLayout*> Create(Device* device,
@ -33,14 +53,26 @@ namespace dawn_native { namespace vulkan {
VkDescriptorSetLayout GetHandle() const;
static constexpr size_t kMaxPoolSizesNeeded = 4;
using PoolSizeSpec = std::array<VkDescriptorPoolSize, kMaxPoolSizesNeeded>;
PoolSizeSpec ComputePoolSizes(uint32_t* numPoolSizes) const;
ResultOrError<DescriptorSetAllocation> AllocateOneSet();
void Deallocate(DescriptorSetAllocation* allocation);
// Interaction with the DescriptorSetService.
void FinishDeallocation(size_t index);
private:
using BindGroupLayoutBase::BindGroupLayoutBase;
MaybeError Initialize();
std::vector<VkDescriptorPoolSize> mPoolSizes;
struct SingleDescriptorSetAllocation {
VkDescriptorPool pool = VK_NULL_HANDLE;
// Descriptor sets are freed when the pool is destroyed.
VkDescriptorSet set = VK_NULL_HANDLE;
};
std::vector<SingleDescriptorSetAllocation> mAllocations;
std::vector<size_t> mAvailableAllocations;
VkDescriptorSetLayout mHandle = VK_NULL_HANDLE;
};

48
src/dawn_native/vulkan/BindGroupVk.cpp
View File

@ -36,37 +36,7 @@ namespace dawn_native { namespace vulkan {
MaybeError BindGroup::Initialize() {
Device* device = ToBackend(GetDevice());
// Create a pool to hold our descriptor set.
// TODO(cwallez@chromium.org): This horribly inefficient, find a way to be better, for
// example by having one pool per bind group layout instead.
uint32_t numPoolSizes = 0;
auto poolSizes = ToBackend(GetLayout())->ComputePoolSizes(&numPoolSizes);
VkDescriptorPoolCreateInfo createInfo;
createInfo.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO;
createInfo.pNext = nullptr;
createInfo.flags = 0;
createInfo.maxSets = 1;
createInfo.poolSizeCount = numPoolSizes;
createInfo.pPoolSizes = poolSizes.data();
DAWN_TRY(CheckVkSuccess(
device->fn.CreateDescriptorPool(device->GetVkDevice(), &createInfo, nullptr, &mPool),
"CreateDescriptorPool"));
// Now do the allocation of one descriptor set, this is very suboptimal too.
VkDescriptorSetLayout vkLayout = ToBackend(GetLayout())->GetHandle();
VkDescriptorSetAllocateInfo allocateInfo;
allocateInfo.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO;
allocateInfo.pNext = nullptr;
allocateInfo.descriptorPool = mPool;
allocateInfo.descriptorSetCount = 1;
allocateInfo.pSetLayouts = &vkLayout;
DAWN_TRY(CheckVkSuccess(
device->fn.AllocateDescriptorSets(device->GetVkDevice(), &allocateInfo, &mHandle),
"AllocateDescriptorSets"));
DAWN_TRY_ASSIGN(mAllocation, ToBackend(GetLayout())->AllocateOneSet());
// Now do a write of a single descriptor set with all possible chained data allocated on the
// stack.
@ -80,7 +50,7 @@ namespace dawn_native { namespace vulkan {
auto& write = writes[numWrites];
write.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET;
write.pNext = nullptr;
write.dstSet = mHandle;
write.dstSet = mAllocation.set;
write.dstBinding = bindingIndex;
write.dstArrayElement = 0;
write.descriptorCount = 1;
@ -108,7 +78,7 @@ namespace dawn_native { namespace vulkan {
TextureView* view = ToBackend(GetBindingAsTextureView(bindingIndex));
writeImageInfo[numWrites].imageView = view->GetHandle();
// TODO(cwallez@chromium.org): This isn't true in general: if the image can has
// TODO(cwallez@chromium.org): This isn't true in general: if the image has
// two read-only usages one of which is Sampled. Works for now though :)
writeImageInfo[numWrites].imageLayout =
VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
@ -123,6 +93,7 @@ namespace dawn_native { namespace vulkan {
numWrites++;
}
// TODO(cwallez@chromium.org): Batch these updates
device->fn.UpdateDescriptorSets(device->GetVkDevice(), numWrites, writes.data(), 0,
nullptr);
@ -130,18 +101,11 @@ namespace dawn_native { namespace vulkan {
}
BindGroup::~BindGroup() {
// The descriptor set doesn't need to be delete because it's done implicitly when the
// descriptor pool is destroyed.
mHandle = VK_NULL_HANDLE;
if (mPool != VK_NULL_HANDLE) {
ToBackend(GetDevice())->GetFencedDeleter()->DeleteWhenUnused(mPool);
mPool = VK_NULL_HANDLE;
}
ToBackend(GetLayout())->Deallocate(&mAllocation);
}
VkDescriptorSet BindGroup::GetHandle() const {
return mHandle;
return mAllocation.set;
}
}} // namespace dawn_native::vulkan

7
src/dawn_native/vulkan/BindGroupVk.h
View File

@ -17,7 +17,7 @@
#include "dawn_native/BindGroup.h"
#include "common/vulkan_platform.h"
#include "dawn_native/vulkan/BindGroupLayoutVk.h"
namespace dawn_native { namespace vulkan {
@ -35,8 +35,9 @@ namespace dawn_native { namespace vulkan {
using BindGroupBase::BindGroupBase;
MaybeError Initialize();
VkDescriptorPool mPool = VK_NULL_HANDLE;
VkDescriptorSet mHandle = VK_NULL_HANDLE;
// The descriptor set in this allocation outlives the BindGroup because it is owned by
// the BindGroupLayout which is referenced by the BindGroup.
DescriptorSetAllocation mAllocation;
};
}} // namespace dawn_native::vulkan

41
src/dawn_native/vulkan/DescriptorSetService.cpp Normal file
View File

@ -0,0 +1,41 @@
// Copyright 2019 The Dawn Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "dawn_native/vulkan/DescriptorSetService.h"
#include "dawn_native/vulkan/BindGroupLayoutVk.h"
#include "dawn_native/vulkan/DeviceVk.h"
namespace dawn_native { namespace vulkan {
DescriptorSetService::DescriptorSetService(Device* device) : mDevice(device) {
}
DescriptorSetService::~DescriptorSetService() {
ASSERT(mDeallocations.Empty());
}
void DescriptorSetService::AddDeferredDeallocation(BindGroupLayout* layout, size_t index) {
mDeallocations.Enqueue({layout, index}, mDevice->GetPendingCommandSerial());
}
void DescriptorSetService::Tick(Serial completedSerial) {
for (Deallocation& dealloc : mDeallocations.IterateUpTo(completedSerial)) {
dealloc.layout->FinishDeallocation(dealloc.index);
}
mDeallocations.ClearUpTo(completedSerial);
}
}} // namespace dawn_native::vulkan

53
src/dawn_native/vulkan/DescriptorSetService.h Normal file
View File

@ -0,0 +1,53 @@
// Copyright 2019 The Dawn Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef DAWNNATIVE_VULKAN_DESCRIPTORSETSERVICE_H_
#define DAWNNATIVE_VULKAN_DESCRIPTORSETSERVICE_H_
#include "common/SerialQueue.h"
#include "dawn_native/vulkan/BindGroupLayoutVk.h"
#include <vector>
namespace dawn_native { namespace vulkan {
class BindGroupLayout;
class Device;
// Handles everything related to descriptor sets that isn't tied to a particular
// BindGroupLayout.
class DescriptorSetService {
public:
DescriptorSetService(Device* device);
~DescriptorSetService();
// Will call layout->FinishDeallocation when the serial is passed.
void AddDeferredDeallocation(BindGroupLayout* layout, size_t index);
void Tick(Serial completedSerial);
private:
Device* mDevice;
struct Deallocation {
Ref<BindGroupLayout> layout;
size_t index;
};
SerialQueue<Deallocation> mDeallocations;
};
}} // namespace dawn_native::vulkan
#endif // DAWNNATIVE_VULKAN_DESCRIPTORSETSERVICE_H_

8
src/dawn_native/vulkan/DeviceVk.cpp
View File

@ -28,6 +28,7 @@
#include "dawn_native/vulkan/BufferVk.h"
#include "dawn_native/vulkan/CommandBufferVk.h"
#include "dawn_native/vulkan/ComputePipelineVk.h"
#include "dawn_native/vulkan/DescriptorSetService.h"
#include "dawn_native/vulkan/FencedDeleter.h"
#include "dawn_native/vulkan/PipelineLayoutVk.h"
#include "dawn_native/vulkan/QueueVk.h"
@ -67,6 +68,7 @@ namespace dawn_native { namespace vulkan {
DAWN_TRY(functions->LoadDeviceProcs(mVkDevice, mDeviceInfo));
GatherQueueFromDevice();
mDescriptorSetService = std::make_unique<DescriptorSetService>(this);
mDeleter = std::make_unique<FencedDeleter>(this);
mMapRequestTracker = std::make_unique<MapRequestTracker>(this);
mRenderPassCache = std::make_unique<RenderPassCache>(this);
@ -131,6 +133,7 @@ namespace dawn_native { namespace vulkan {
// Free services explicitly so that they can free Vulkan objects before vkDestroyDevice
mDynamicUploader = nullptr;
mDescriptorSetService = nullptr;
// Releasing the uploader enqueues buffers to be released.
// Call Tick() again to clear them before releasing the deleter.
@ -216,6 +219,7 @@ namespace dawn_native { namespace vulkan {
CheckPassedFences();
RecycleCompletedCommands();
mDescriptorSetService->Tick(mCompletedSerial);
mMapRequestTracker->Tick(mCompletedSerial);
// Uploader should tick before the resource allocator
@ -261,6 +265,10 @@ namespace dawn_native { namespace vulkan {
return mMapRequestTracker.get();
}
DescriptorSetService* Device::GetDescriptorSetService() const {
return mDescriptorSetService.get();
}
FencedDeleter* Device::GetFencedDeleter() const {
return mDeleter.get();
}

3
src/dawn_native/vulkan/DeviceVk.h
View File

@ -35,6 +35,7 @@ namespace dawn_native { namespace vulkan {
class Adapter;
class BufferUploader;
class DescriptorSetService;
struct ExternalImageDescriptor;
class FencedDeleter;
class MapRequestTracker;
@ -58,6 +59,7 @@ namespace dawn_native { namespace vulkan {
VkQueue GetQueue() const;
BufferUploader* GetBufferUploader() const;
DescriptorSetService* GetDescriptorSetService() const;
FencedDeleter* GetFencedDeleter() const;
MapRequestTracker* GetMapRequestTracker() const;
RenderPassCache* GetRenderPassCache() const;
@ -132,6 +134,7 @@ namespace dawn_native { namespace vulkan {
uint32_t mQueueFamily = 0;
VkQueue mQueue = VK_NULL_HANDLE;
std::unique_ptr<DescriptorSetService> mDescriptorSetService;
std::unique_ptr<FencedDeleter> mDeleter;
std::unique_ptr<MapRequestTracker> mMapRequestTracker;
std::unique_ptr<ResourceMemoryAllocator> mResourceMemoryAllocator;