Residency 4: Add Facilities For Budgeting Device Memory

Use D3D12's QueryDeviceVideoMemoryInfo to get the OS-determined process
budget. Also introduces an export for reserving some amount of process
memory - which keeps Dawn from using the entire process's budget.

Bug: dawn:193
Change-Id: I6c17bd703d7cb24759bcee89c03add46944fec8c
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/16383
Reviewed-by: Austin Eng <enga@chromium.org>
Commit-Queue: Corentin Wallez <cwallez@chromium.org>

BUILD.gn

@@ -325,6 +325,8 @@ source_set("libdawn_native_sources") {
       "src/dawn_native/d3d12/RenderPassBuilderD3D12.h",
       "src/dawn_native/d3d12/RenderPipelineD3D12.cpp",
       "src/dawn_native/d3d12/RenderPipelineD3D12.h",
+      "src/dawn_native/d3d12/ResidencyManagerD3D12.cpp",
+      "src/dawn_native/d3d12/ResidencyManagerD3D12.h",
       "src/dawn_native/d3d12/ResourceAllocatorManagerD3D12.cpp",
       "src/dawn_native/d3d12/ResourceAllocatorManagerD3D12.h",
       "src/dawn_native/d3d12/ResourceHeapAllocationD3D12.cpp",

src/dawn_native/CMakeLists.txt

@@ -198,6 +198,8 @@ if (DAWN_ENABLE_D3D12)
         "d3d12/RenderPassBuilderD3D12.h"
         "d3d12/RenderPipelineD3D12.cpp"
         "d3d12/RenderPipelineD3D12.h"
+        "d3d12/ResidencyManagerD3D12.cpp"
+        "d3d12/ResidencyManagerD3D12.h"
         "d3d12/ResourceAllocatorManagerD3D12.cpp"
         "d3d12/ResourceAllocatorManagerD3D12.h"
         "d3d12/ResourceHeapAllocationD3D12.cpp"

src/dawn_native/Toggles.cpp

@@ -83,6 +83,12 @@ namespace dawn_native {
               "versions of Windows prior to build 1809, or when this toggle is turned off, Dawn "
               "will emulate a render pass.",
               "https://crbug.com/dawn/36"}},
+            {Toggle::UseD3D12ResidencyManagement,
+             {"use_d3d12_residency_management",
+              "Enable residency management. This allows page-in and page-out of resource heaps in "
+              "GPU memory. This component improves overcommitted performance by keeping the most "
+              "recently used resources local to the GPU. Turning this component off can cause "
+              "allocation failures when application memory exceeds physical device memory."}},
             {Toggle::SkipValidation,
              {"skip_validation", "Skip expensive validation of Dawn commands.",
               "https://crbug.com/dawn/271"}},

src/dawn_native/Toggles.h

@@ -32,6 +32,7 @@ namespace dawn_native {
         UseTemporaryBufferInCompressedTextureToTextureCopy,
         UseD3D12ResourceHeapTier2,
         UseD3D12RenderPass,
+        UseD3D12ResidencyManagement,
         SkipValidation,
         UseSpvc,
         UseSpvcParser,

src/dawn_native/d3d12/AdapterD3D12.cpp

@@ -34,7 +34,7 @@ namespace dawn_native { namespace d3d12 {
         }
     };
 
-    Adapter::Adapter(Backend* backend, ComPtr<IDXGIAdapter1> hardwareAdapter)
+    Adapter::Adapter(Backend* backend, ComPtr<IDXGIAdapter3> hardwareAdapter)
         : AdapterBase(backend->GetInstance(), wgpu::BackendType::D3D12),
           mHardwareAdapter(hardwareAdapter),
           mBackend(backend) {
@@ -44,7 +44,7 @@ namespace dawn_native { namespace d3d12 {
         return mDeviceInfo;
     }
 
-    IDXGIAdapter1* Adapter::GetHardwareAdapter() const {
+    IDXGIAdapter3* Adapter::GetHardwareAdapter() const {
         return mHardwareAdapter.Get();
     }
 

src/dawn_native/d3d12/AdapterD3D12.h

@@ -26,11 +26,11 @@ namespace dawn_native { namespace d3d12 {
 
     class Adapter : public AdapterBase {
       public:
-        Adapter(Backend* backend, ComPtr<IDXGIAdapter1> hardwareAdapter);
+        Adapter(Backend* backend, ComPtr<IDXGIAdapter3> hardwareAdapter);
         virtual ~Adapter() = default;
 
         const D3D12DeviceInfo& GetDeviceInfo() const;
-        IDXGIAdapter1* GetHardwareAdapter() const;
+        IDXGIAdapter3* GetHardwareAdapter() const;
         Backend* GetBackend() const;
         ComPtr<ID3D12Device> GetDevice() const;
 
@@ -40,7 +40,7 @@ namespace dawn_native { namespace d3d12 {
         ResultOrError<DeviceBase*> CreateDeviceImpl(const DeviceDescriptor* descriptor) override;
         void InitializeSupportedExtensions();
 
-        ComPtr<IDXGIAdapter1> mHardwareAdapter;
+        ComPtr<IDXGIAdapter3> mHardwareAdapter;
         ComPtr<ID3D12Device> mD3d12Device;
 
         Backend* mBackend;

src/dawn_native/d3d12/BackendD3D12.cpp

@@ -106,7 +106,12 @@ namespace dawn_native { namespace d3d12 {
 
             ASSERT(dxgiAdapter != nullptr);
 
-            std::unique_ptr<Adapter> adapter = std::make_unique<Adapter>(this, dxgiAdapter);
+            ComPtr<IDXGIAdapter3> dxgiAdapter3;
+            HRESULT result = dxgiAdapter.As(&dxgiAdapter3);
+            ASSERT(SUCCEEDED(result));
+
+            std::unique_ptr<Adapter> adapter =
+                std::make_unique<Adapter>(this, std::move(dxgiAdapter3));
             if (GetInstance()->ConsumedError(adapter->Initialize())) {
                 continue;
             }

src/dawn_native/d3d12/D3D12Backend.cpp

@@ -20,6 +20,7 @@
 #include "common/SwapChainUtils.h"
 #include "dawn_native/d3d12/DeviceD3D12.h"
 #include "dawn_native/d3d12/NativeSwapChainImplD3D12.h"
+#include "dawn_native/d3d12/ResidencyManagerD3D12.h"
 #include "dawn_native/d3d12/TextureD3D12.h"
 
 namespace dawn_native { namespace d3d12 {
@@ -50,6 +51,13 @@ namespace dawn_native { namespace d3d12 {
         : ExternalImageDescriptor(ExternalImageDescriptorType::DXGISharedHandle) {
     }
 
+    uint64_t SetExternalMemoryReservation(WGPUDevice device, uint64_t requestedReservationSize) {
+        Device* backendDevice = reinterpret_cast<Device*>(device);
+
+        return backendDevice->GetResidencyManager()->SetExternalMemoryReservation(
+            requestedReservationSize);
+    }
+
     WGPUTexture WrapSharedHandle(WGPUDevice device,
                                  const ExternalImageDescriptorDXGISharedHandle* descriptor) {
         Device* backendDevice = reinterpret_cast<Device*>(device);

src/dawn_native/d3d12/DeviceD3D12.cpp

@@ -32,6 +32,7 @@
 #include "dawn_native/d3d12/PlatformFunctions.h"
 #include "dawn_native/d3d12/QueueD3D12.h"
 #include "dawn_native/d3d12/RenderPipelineD3D12.h"
+#include "dawn_native/d3d12/ResidencyManagerD3D12.h"
 #include "dawn_native/d3d12/ResourceAllocatorManagerD3D12.h"
 #include "dawn_native/d3d12/SamplerD3D12.h"
 #include "dawn_native/d3d12/ShaderModuleD3D12.h"
@@ -79,6 +80,7 @@ namespace dawn_native { namespace d3d12 {
         DAWN_TRY(mShaderVisibleDescriptorAllocator->Initialize());
 
         mMapRequestTracker = std::make_unique<MapRequestTracker>(this);
+        mResidencyManager = std::make_unique<ResidencyManager>(this);
         mResourceAllocatorManager = std::make_unique<ResourceAllocatorManager>(this);
 
         DAWN_TRY(NextSerial());
@@ -154,6 +156,10 @@ namespace dawn_native { namespace d3d12 {
         return mCommandAllocatorManager.get();
     }
 
+    ResidencyManager* Device::GetResidencyManager() const {
+        return mResidencyManager.get();
+    }
+
     ResultOrError<CommandRecordingContext*> Device::GetPendingCommandContext() {
         // Callers of GetPendingCommandList do so to record commands. Only reserve a command
         // allocator when it is needed so we don't submit empty command lists
@@ -403,6 +409,7 @@ namespace dawn_native { namespace d3d12 {
         const bool useResourceHeapTier2 = (GetDeviceInfo().resourceHeapTier >= 2);
         SetToggle(Toggle::UseD3D12ResourceHeapTier2, useResourceHeapTier2);
         SetToggle(Toggle::UseD3D12RenderPass, GetDeviceInfo().supportsRenderPass);
+        SetToggle(Toggle::UseD3D12ResidencyManagement, false);
     }
 
     MaybeError Device::WaitForIdleForDestruction() {

src/dawn_native/d3d12/DeviceD3D12.h

@@ -34,6 +34,7 @@ namespace dawn_native { namespace d3d12 {
     class MapRequestTracker;
     class PlatformFunctions;
     class ResourceAllocatorManager;
+    class ResidencyManager;
 
 #define ASSERT_SUCCESS(hr)            \
     {                                 \
@@ -66,6 +67,7 @@ namespace dawn_native { namespace d3d12 {
         DescriptorHeapAllocator* GetDescriptorHeapAllocator() const;
         MapRequestTracker* GetMapRequestTracker() const;
         CommandAllocatorManager* GetCommandAllocatorManager() const;
+        ResidencyManager* GetResidencyManager() const;
 
         const PlatformFunctions* GetFunctions() const;
         ComPtr<IDXGIFactory4> GetFactory() const;
@@ -161,6 +163,7 @@ namespace dawn_native { namespace d3d12 {
         std::unique_ptr<DescriptorHeapAllocator> mDescriptorHeapAllocator;
         std::unique_ptr<MapRequestTracker> mMapRequestTracker;
         std::unique_ptr<ResourceAllocatorManager> mResourceAllocatorManager;
+        std::unique_ptr<ResidencyManager> mResidencyManager;
         std::unique_ptr<ShaderVisibleDescriptorAllocator> mShaderVisibleDescriptorAllocator;
     };
 

src/dawn_native/d3d12/ResidencyManagerD3D12.cpp

@@ -0,0 +1,69 @@
+// Copyright 2020 The Dawn Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "dawn_native/d3d12/ResidencyManagerD3D12.h"
+
+#include "dawn_native/d3d12/AdapterD3D12.h"
+#include "dawn_native/d3d12/DeviceD3D12.h"
+#include "dawn_native/d3d12/Forward.h"
+#include "dawn_native/d3d12/d3d12_platform.h"
+
+namespace dawn_native { namespace d3d12 {
+
+    ResidencyManager::ResidencyManager(Device* device)
+        : mDevice(device),
+          mResidencyManagementEnabled(
+              device->IsToggleEnabled(Toggle::UseD3D12ResidencyManagement)) {
+        UpdateVideoMemoryInfo();
+    }
+
+    // Allows an application component external to Dawn to cap Dawn's residency budget to prevent
+    // competition for device local memory. Returns the amount of memory reserved, which may be less
+    // that the requested reservation when under pressure.
+    uint64_t ResidencyManager::SetExternalMemoryReservation(uint64_t requestedReservationSize) {
+        mVideoMemoryInfo.externalRequest = requestedReservationSize;
+        UpdateVideoMemoryInfo();
+        return mVideoMemoryInfo.externalReservation;
+    }
+
+    void ResidencyManager::UpdateVideoMemoryInfo() {
+        if (!mResidencyManagementEnabled) {
+            return;
+        }
+
+        DXGI_QUERY_VIDEO_MEMORY_INFO queryVideoMemoryInfo;
+        ToBackend(mDevice->GetAdapter())
+            ->GetHardwareAdapter()
+            ->QueryVideoMemoryInfo(0, DXGI_MEMORY_SEGMENT_GROUP_LOCAL, &queryVideoMemoryInfo);
+
+        // The video memory budget provided by QueryVideoMemoryInfo is defined by the operating
+        // system, and may be lower than expected in certain scenarios. Under memory pressure, we
+        // cap the external reservation to half the available budget, which prevents the external
+        // component from consuming a disproportionate share of memory and ensures that Dawn can
+        // continue to make forward progress. Note the choice to halve memory is arbitrarily chosen
+        // and subject to future experimentation.
+        mVideoMemoryInfo.externalReservation =
+            std::min(queryVideoMemoryInfo.Budget / 2, mVideoMemoryInfo.externalReservation);
+
+        // We cap Dawn's budget to 95% of the provided budget. Leaving some budget unused
+        // decreases fluctuations in the operating-system-defined budget, which improves stability
+        // for both Dawn and other applications on the system. Note the value of 95% is arbitrarily
+        // chosen and subject to future experimentation.
+        static constexpr float kBudgetCap = 0.95;
+        mVideoMemoryInfo.dawnBudget =
+            (queryVideoMemoryInfo.Budget - mVideoMemoryInfo.externalReservation) * kBudgetCap;
+        mVideoMemoryInfo.dawnUsage =
+            queryVideoMemoryInfo.CurrentUsage - mVideoMemoryInfo.externalReservation;
+    }
+}}  // namespace dawn_native::d3d12

src/dawn_native/d3d12/ResidencyManagerD3D12.h

@@ -0,0 +1,45 @@
+// Copyright 2020 The Dawn Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef DAWNNATIVE_D3D12_RESIDENCYMANAGERD3D12_H_
+#define DAWNNATIVE_D3D12_RESIDENCYMANAGERD3D12_H_
+
+#include "dawn_native/dawn_platform.h"
+
+namespace dawn_native { namespace d3d12 {
+
+    class Device;
+
+    class ResidencyManager {
+      public:
+        ResidencyManager(Device* device);
+        uint64_t SetExternalMemoryReservation(uint64_t requestedReservationSize);
+
+      private:
+        struct VideoMemoryInfo {
+            uint64_t dawnBudget;
+            uint64_t dawnUsage;
+            uint64_t externalReservation;
+            uint64_t externalRequest;
+        };
+        void UpdateVideoMemoryInfo();
+
+        Device* mDevice = nullptr;
+        bool mResidencyManagementEnabled = false;
+        VideoMemoryInfo mVideoMemoryInfo = {};
+    };
+
+}}  // namespace dawn_native::d3d12
+
+#endif  // DAWNNATIVE_D3D12_RESIDENCYMANAGERD3D12_H_

src/include/dawn_native/D3D12Backend.h

@@ -38,6 +38,9 @@ namespace dawn_native { namespace d3d12 {
         uint64_t acquireMutexKey;
     };
 
+    DAWN_NATIVE_EXPORT uint64_t SetExternalMemoryReservation(WGPUDevice device,
+                                                             uint64_t requestedReservationSize);
+
     // Note: SharedHandle must be a handle to a texture object.
     DAWN_NATIVE_EXPORT WGPUTexture
     WrapSharedHandle(WGPUDevice device, const ExternalImageDescriptorDXGISharedHandle* descriptor);