D3D12: Cache DXGI keyed mutexes on the external image.

Maintains a 11on12 resource cache on the external image
to prevent re-creating the wrapped resource then
flushing it per produced texture. To prevent
unbounded growth, a basic cap is used which gets
cleared.

This change fixes signficant CPU time spent in the WebGPU command
decoder for video import workloads and excessive memory overhead
from swap chain buffers.

Fixed: dawn:625

Change-Id: I72c07b02f6ab6877a9f21758650962c895933bf9
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/51421
Commit-Queue: Bryan Bernhart <bryan.bernhart@intel.com>
Reviewed-by: Corentin Wallez <cwallez@chromium.org>

src/dawn_native/BUILD.gn

@@ -358,6 +358,8 @@ source_set("dawn_native_sources") {
       "d3d12/CommandRecordingContext.h",
       "d3d12/ComputePipelineD3D12.cpp",
       "d3d12/ComputePipelineD3D12.h",
+      "d3d12/D3D11on12Util.cpp",
+      "d3d12/D3D11on12Util.h",
       "d3d12/D3D12Error.cpp",
       "d3d12/D3D12Error.h",
       "d3d12/D3D12Info.cpp",

src/dawn_native/CMakeLists.txt

@@ -232,6 +232,8 @@ if (DAWN_ENABLE_D3D12)
         "d3d12/CommandRecordingContext.h"
         "d3d12/ComputePipelineD3D12.cpp"
         "d3d12/ComputePipelineD3D12.h"
+        "d3d12/D3D11on12Util.cpp"
+        "d3d12/D3D11on12Util.h"
         "d3d12/D3D12Error.cpp"
         "d3d12/D3D12Error.h"
         "d3d12/D3D12Info.cpp"

src/dawn_native/d3d12/D3D11on12Util.cpp

@@ -0,0 +1,164 @@
+// Copyright 2021 The Dawn Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// D3D12Backend.cpp: contains the definition of symbols exported by D3D12Backend.h so that they
+// can be compiled twice: once export (shared library), once not exported (static library)
+
+#include "dawn_native/d3d12/D3D11on12Util.h"
+
+#include "common/HashUtils.h"
+#include "common/Log.h"
+#include "dawn_native/d3d12/DeviceD3D12.h"
+
+namespace dawn_native { namespace d3d12 {
+
+    void Flush11On12DeviceToAvoidLeaks(ComPtr<ID3D11On12Device> d3d11on12Device) {
+        if (d3d11on12Device == nullptr) {
+            return;
+        }
+
+        ComPtr<ID3D11Device> d3d11Device;
+        if (FAILED(d3d11on12Device.As(&d3d11Device))) {
+            return;
+        }
+
+        ComPtr<ID3D11DeviceContext> d3d11DeviceContext;
+        d3d11Device->GetImmediateContext(&d3d11DeviceContext);
+
+        ASSERT(d3d11DeviceContext != nullptr);
+
+        // 11on12 has a bug where D3D12 resources used only for keyed shared mutexes
+        // are not released until work is submitted to the device context and flushed.
+        // The most minimal work we can get away with is issuing a TiledResourceBarrier.
+
+        // ID3D11DeviceContext2 is available in Win8.1 and above. This suffices for a
+        // D3D12 backend since both D3D12 and 11on12 first appeared in Windows 10.
+        ComPtr<ID3D11DeviceContext2> d3d11DeviceContext2;
+        if (FAILED(d3d11DeviceContext.As(&d3d11DeviceContext2))) {
+            return;
+        }
+
+        d3d11DeviceContext2->TiledResourceBarrier(nullptr, nullptr);
+        d3d11DeviceContext2->Flush();
+    }
+
+    D3D11on12ResourceCacheEntry::D3D11on12ResourceCacheEntry(
+        ComPtr<IDXGIKeyedMutex> dxgiKeyedMutex,
+        ComPtr<ID3D11On12Device> d3d11On12Device)
+        : mDXGIKeyedMutex(std::move(dxgiKeyedMutex)), mD3D11on12Device(std::move(d3d11On12Device)) {
+    }
+
+    D3D11on12ResourceCacheEntry::D3D11on12ResourceCacheEntry(
+        ComPtr<ID3D11On12Device> d3d11On12Device)
+        : mD3D11on12Device(std::move(d3d11On12Device)) {
+    }
+
+    D3D11on12ResourceCacheEntry::~D3D11on12ResourceCacheEntry() {
+        if (mDXGIKeyedMutex == nullptr) {
+            return;
+        }
+
+        ComPtr<ID3D11Resource> d3d11Resource;
+        if (FAILED(mDXGIKeyedMutex.As(&d3d11Resource))) {
+            return;
+        }
+
+        ASSERT(mD3D11on12Device != nullptr);
+
+        ID3D11Resource* d3d11ResourceRaw = d3d11Resource.Get();
+        mD3D11on12Device->ReleaseWrappedResources(&d3d11ResourceRaw, 1);
+
+        d3d11Resource.Reset();
+        mDXGIKeyedMutex.Reset();
+
+        Flush11On12DeviceToAvoidLeaks(std::move(mD3D11on12Device));
+    }
+
+    ComPtr<IDXGIKeyedMutex> D3D11on12ResourceCacheEntry::GetDXGIKeyedMutex() const {
+        ASSERT(mDXGIKeyedMutex != nullptr);
+        return mDXGIKeyedMutex;
+    }
+
+    size_t D3D11on12ResourceCacheEntry::HashFunc::operator()(
+        const Ref<D3D11on12ResourceCacheEntry> a) const {
+        size_t hash = 0;
+        HashCombine(&hash, a->mD3D11on12Device.Get());
+        return hash;
+    }
+
+    bool D3D11on12ResourceCacheEntry::EqualityFunc::operator()(
+        const Ref<D3D11on12ResourceCacheEntry> a,
+        const Ref<D3D11on12ResourceCacheEntry> b) const {
+        return a->mD3D11on12Device == b->mD3D11on12Device;
+    }
+
+    D3D11on12ResourceCache::D3D11on12ResourceCache() = default;
+
+    D3D11on12ResourceCache::~D3D11on12ResourceCache() = default;
+
+    Ref<D3D11on12ResourceCacheEntry> D3D11on12ResourceCache::GetOrCreateD3D11on12Resource(
+        WGPUDevice device,
+        ID3D12Resource* d3d12Resource) {
+        Device* backendDevice = reinterpret_cast<Device*>(device);
+        // The Dawn and 11on12 device share the same D3D12 command queue whereas this external image
+        // could be accessed/produced with multiple Dawn devices. To avoid cross-queue sharing
+        // restrictions, the 11 wrapped resource is forbidden to be shared between Dawn devices by
+        // using the 11on12 device as the cache key.
+        ComPtr<ID3D11On12Device> d3d11on12Device = backendDevice->GetOrCreateD3D11on12Device();
+        if (d3d11on12Device == nullptr) {
+            dawn::ErrorLog() << "Unable to create 11on12 device for external image";
+            return nullptr;
+        }
+
+        D3D11on12ResourceCacheEntry blueprint(d3d11on12Device);
+        auto iter = mCache.find(&blueprint);
+        if (iter != mCache.end()) {
+            return *iter;
+        }
+
+        // We use IDXGIKeyedMutexes to synchronize access between D3D11 and D3D12. D3D11/12 fences
+        // are a viable alternative but are, unfortunately, not available on all versions of Windows
+        // 10. Since D3D12 does not directly support keyed mutexes, we need to wrap the D3D12
+        // resource using 11on12 and QueryInterface the D3D11 representation for the keyed mutex.
+        ComPtr<ID3D11Texture2D> d3d11Texture;
+        D3D11_RESOURCE_FLAGS resourceFlags;
+        resourceFlags.BindFlags = 0;
+        resourceFlags.MiscFlags = D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX;
+        resourceFlags.CPUAccessFlags = 0;
+        resourceFlags.StructureByteStride = 0;
+        if (FAILED(d3d11on12Device->CreateWrappedResource(
+                d3d12Resource, &resourceFlags, D3D12_RESOURCE_STATE_COMMON,
+                D3D12_RESOURCE_STATE_COMMON, IID_PPV_ARGS(&d3d11Texture)))) {
+            return nullptr;
+        }
+
+        ComPtr<IDXGIKeyedMutex> dxgiKeyedMutex;
+        if (FAILED(d3d11Texture.As(&dxgiKeyedMutex))) {
+            return nullptr;
+        }
+
+        // Keep this cache from growing unbounded.
+        // TODO(dawn:625): Consider using a replacement policy based cache.
+        if (mCache.size() > kMaxD3D11on12ResourceCacheSize) {
+            mCache.clear();
+        }
+
+        Ref<D3D11on12ResourceCacheEntry> entry =
+            AcquireRef(new D3D11on12ResourceCacheEntry(dxgiKeyedMutex, std::move(d3d11on12Device)));
+        mCache.insert(entry);
+
+        return entry;
+    }
+
+}}  // namespace dawn_native::d3d12

src/dawn_native/d3d12/D3D11on12Util.h

@@ -0,0 +1,89 @@
+// Copyright 2021 The Dawn Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef DAWNNATIVE_D3D11ON12UTIL_H_
+#define DAWNNATIVE_D3D11ON12UTIL_H_
+
+#include "common/RefCounted.h"
+#include "dawn_native/d3d12/d3d12_platform.h"
+
+#include <dawn_native/DawnNative.h>
+#include <memory>
+#include <unordered_set>
+
+struct ID3D11On12Device;
+struct IDXGIKeyedMutex;
+
+namespace dawn_native { namespace d3d12 {
+
+    // Wraps 11 wrapped resources in a cache.
+    class D3D11on12ResourceCacheEntry : public RefCounted {
+      public:
+        D3D11on12ResourceCacheEntry(ComPtr<ID3D11On12Device> d3d11on12Device);
+        D3D11on12ResourceCacheEntry(ComPtr<IDXGIKeyedMutex> d3d11on12Resource,
+                                    ComPtr<ID3D11On12Device> d3d11on12Device);
+        ~D3D11on12ResourceCacheEntry();
+
+        ComPtr<IDXGIKeyedMutex> GetDXGIKeyedMutex() const;
+
+        // Functors necessary for the
+        // unordered_set<D3D11on12ResourceCacheEntry&>-based cache.
+        struct HashFunc {
+            size_t operator()(const Ref<D3D11on12ResourceCacheEntry> a) const;
+        };
+
+        struct EqualityFunc {
+            bool operator()(const Ref<D3D11on12ResourceCacheEntry> a,
+                            const Ref<D3D11on12ResourceCacheEntry> b) const;
+        };
+
+      private:
+        ComPtr<IDXGIKeyedMutex> mDXGIKeyedMutex;
+        ComPtr<ID3D11On12Device> mD3D11on12Device;
+    };
+
+    // |D3D11on12ResourceCache| maintains a cache of 11 wrapped resources.
+    // Each entry represents a 11 resource that is exclusively accessed by Dawn device.
+    // Since each Dawn device creates and stores a 11on12 device, the 11on12 device
+    // is used as the key for the cache entry which ensures only the same 11 wrapped
+    // resource is re-used and also fully released.
+    //
+    // The cache is primarily needed to avoid repeatedly calling CreateWrappedResource
+    // and special release code per ProduceTexture(device).
+    class D3D11on12ResourceCache {
+      public:
+        D3D11on12ResourceCache();
+        ~D3D11on12ResourceCache();
+
+        Ref<D3D11on12ResourceCacheEntry> GetOrCreateD3D11on12Resource(
+            WGPUDevice device,
+            ID3D12Resource* d3d12Resource);
+
+      private:
+        // TODO(dawn:625): Figure out a large enough cache size.
+        static constexpr uint64_t kMaxD3D11on12ResourceCacheSize = 5;
+
+        // 11on12 resource cache entries are refcounted to ensure if the ExternalImage outlives the
+        // Dawn texture (or vice-versa), we always fully release the 11 wrapped resource without
+        // waiting until Dawn device to shutdown.
+        using Cache = std::unordered_set<Ref<D3D11on12ResourceCacheEntry>,
+                                         D3D11on12ResourceCacheEntry::HashFunc,
+                                         D3D11on12ResourceCacheEntry::EqualityFunc>;
+
+        Cache mCache;
+    };
+
+}}  // namespace dawn_native::d3d12
+
+#endif  // DAWNNATIVE_D3D11ON12UTIL_H_

src/dawn_native/d3d12/D3D12Backend.cpp

@@ -20,6 +20,7 @@
 #include "common/Log.h"
 #include "common/Math.h"
 #include "common/SwapChainUtils.h"
+#include "dawn_native/d3d12/D3D11on12Util.h"
 #include "dawn_native/d3d12/DeviceD3D12.h"
 #include "dawn_native/d3d12/NativeSwapChainImplD3D12.h"
 #include "dawn_native/d3d12/ResidencyManagerD3D12.h"
@@ -69,8 +70,11 @@ namespace dawn_native { namespace d3d12 {
                                  descriptor->nextInChain)
                                  ->internalUsage;
         }
+        mD3D11on12ResourceCache = std::make_unique<D3D11on12ResourceCache>();
     }
 
+    ExternalImageDXGI::~ExternalImageDXGI() = default;
+
     WGPUTexture ExternalImageDXGI::ProduceTexture(
         WGPUDevice device,
         const ExternalImageAccessDescriptorDXGIKeyedMutex* descriptor) {
@@ -97,10 +101,19 @@ namespace dawn_native { namespace d3d12 {
             internalDesc.sType = wgpu::SType::DawnTextureInternalUsageDescriptor;
         }
 
+        Ref<D3D11on12ResourceCacheEntry> d3d11on12Resource =
+            mD3D11on12ResourceCache->GetOrCreateD3D11on12Resource(device, mD3D12Resource.Get());
+        if (d3d11on12Resource == nullptr) {
+            dawn::ErrorLog() << "Unable to create 11on12 resource for external image";
+            return nullptr;
+        }
+
         Ref<TextureBase> texture = backendDevice->CreateExternalTexture(
-            &textureDescriptor, mD3D12Resource, ExternalMutexSerial(descriptor->acquireMutexKey),
+            &textureDescriptor, mD3D12Resource, std::move(d3d11on12Resource),
+            ExternalMutexSerial(descriptor->acquireMutexKey),
             ExternalMutexSerial(descriptor->releaseMutexKey), descriptor->isSwapChainTexture,
             descriptor->isInitialized);
+
         return reinterpret_cast<WGPUTexture>(texture.Detach());
     }
 

src/dawn_native/d3d12/DeviceD3D12.cpp

@@ -23,6 +23,7 @@
 #include "dawn_native/d3d12/CommandAllocatorManager.h"
 #include "dawn_native/d3d12/CommandBufferD3D12.h"
 #include "dawn_native/d3d12/ComputePipelineD3D12.h"
+#include "dawn_native/d3d12/D3D11on12Util.h"
 #include "dawn_native/d3d12/D3D12Error.h"
 #include "dawn_native/d3d12/PipelineLayoutD3D12.h"
 #include "dawn_native/d3d12/PlatformFunctions.h"
@@ -458,90 +459,43 @@ namespace dawn_native { namespace d3d12 {
                                                          initialUsage);
     }
 
-    Ref<TextureBase> Device::CreateExternalTexture(const TextureDescriptor* descriptor,
-                                                   ComPtr<ID3D12Resource> d3d12Texture,
-                                                   ExternalMutexSerial acquireMutexKey,
-                                                   ExternalMutexSerial releaseMutexKey,
-                                                   bool isSwapChainTexture,
-                                                   bool isInitialized) {
+    Ref<TextureBase> Device::CreateExternalTexture(
+        const TextureDescriptor* descriptor,
+        ComPtr<ID3D12Resource> d3d12Texture,
+        Ref<D3D11on12ResourceCacheEntry> d3d11on12Resource,
+        ExternalMutexSerial acquireMutexKey,
+        ExternalMutexSerial releaseMutexKey,
+        bool isSwapChainTexture,
+        bool isInitialized) {
         Ref<Texture> dawnTexture;
-        if (ConsumedError(Texture::CreateExternalImage(this, descriptor, std::move(d3d12Texture),
-                                                       acquireMutexKey, releaseMutexKey,
-                                                       isSwapChainTexture, isInitialized),
-                          &dawnTexture)) {
+        if (ConsumedError(
+                Texture::CreateExternalImage(this, descriptor, std::move(d3d12Texture),
+                                             std::move(d3d11on12Resource), acquireMutexKey,
+                                             releaseMutexKey, isSwapChainTexture, isInitialized),
+                &dawnTexture)) {
             return nullptr;
         }
         return {dawnTexture};
     }
 
-    // We use IDXGIKeyedMutexes to synchronize access between D3D11 and D3D12. D3D11/12 fences
-    // are a viable alternative but are, unfortunately, not available on all versions of Windows
-    // 10. Since D3D12 does not directly support keyed mutexes, we need to wrap the D3D12
-    // resource using 11on12 and QueryInterface the D3D11 representation for the keyed mutex.
-    ResultOrError<ComPtr<IDXGIKeyedMutex>> Device::CreateKeyedMutexForTexture(
-        ID3D12Resource* d3d12Resource) {
+    ComPtr<ID3D11On12Device> Device::GetOrCreateD3D11on12Device() {
         if (mD3d11On12Device == nullptr) {
             ComPtr<ID3D11Device> d3d11Device;
-            ComPtr<ID3D11DeviceContext> d3d11DeviceContext;
             D3D_FEATURE_LEVEL d3dFeatureLevel;
             IUnknown* const iUnknownQueue = mCommandQueue.Get();
-            DAWN_TRY(CheckHRESULT(GetFunctions()->d3d11on12CreateDevice(
-                                      mD3d12Device.Get(), 0, nullptr, 0, &iUnknownQueue, 1, 1,
-                                      &d3d11Device, &d3d11DeviceContext, &d3dFeatureLevel),
-                                  "D3D12 11on12 device create"));
+            if (FAILED(GetFunctions()->d3d11on12CreateDevice(mD3d12Device.Get(), 0, nullptr, 0,
+                                                             &iUnknownQueue, 1, 1, &d3d11Device,
+                                                             nullptr, &d3dFeatureLevel))) {
+                return nullptr;
+            }
 
             ComPtr<ID3D11On12Device> d3d11on12Device;
-            DAWN_TRY(CheckHRESULT(d3d11Device.As(&d3d11on12Device),
-                                  "D3D12 QueryInterface ID3D11Device to ID3D11On12Device"));
+            HRESULT hr = d3d11Device.As(&d3d11on12Device);
+            ASSERT(SUCCEEDED(hr));
 
-            ComPtr<ID3D11DeviceContext2> d3d11DeviceContext2;
-            DAWN_TRY(
-                CheckHRESULT(d3d11DeviceContext.As(&d3d11DeviceContext2),
-                             "D3D12 QueryInterface ID3D11DeviceContext to ID3D11DeviceContext2"));
-
-            mD3d11On12DeviceContext = std::move(d3d11DeviceContext2);
             mD3d11On12Device = std::move(d3d11on12Device);
         }
-
-        ComPtr<ID3D11Texture2D> d3d11Texture;
-        D3D11_RESOURCE_FLAGS resourceFlags;
-        resourceFlags.BindFlags = 0;
-        resourceFlags.MiscFlags = D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX;
-        resourceFlags.CPUAccessFlags = 0;
-        resourceFlags.StructureByteStride = 0;
-        DAWN_TRY(CheckHRESULT(mD3d11On12Device->CreateWrappedResource(
-                                  d3d12Resource, &resourceFlags, D3D12_RESOURCE_STATE_COMMON,
-                                  D3D12_RESOURCE_STATE_COMMON, IID_PPV_ARGS(&d3d11Texture)),
-                              "D3D12 creating a wrapped resource"));
-
-        ComPtr<IDXGIKeyedMutex> dxgiKeyedMutex;
-        DAWN_TRY(CheckHRESULT(d3d11Texture.As(&dxgiKeyedMutex),
-                              "D3D12 QueryInterface ID3D11Texture2D to IDXGIKeyedMutex"));
-
-        return std::move(dxgiKeyedMutex);
-    }
-
-    void Device::ReleaseKeyedMutexForTexture(ComPtr<IDXGIKeyedMutex> dxgiKeyedMutex) {
-        ComPtr<ID3D11Resource> d3d11Resource;
-        HRESULT hr = dxgiKeyedMutex.As(&d3d11Resource);
-        if (FAILED(hr)) {
-            return;
-        }
-
-        ID3D11Resource* d3d11ResourceRaw = d3d11Resource.Get();
-        mD3d11On12Device->ReleaseWrappedResources(&d3d11ResourceRaw, 1);
-
-        d3d11Resource.Reset();
-        dxgiKeyedMutex.Reset();
-
-        // 11on12 has a bug where D3D12 resources used only for keyed shared mutexes
-        // are not released until work is submitted to the device context and flushed.
-        // The most minimal work we can get away with is issuing a TiledResourceBarrier.
-
-        // ID3D11DeviceContext2 is available in Win8.1 and above. This suffices for a
-        // D3D12 backend since both D3D12 and 11on12 first appeared in Windows 10.
-        mD3d11On12DeviceContext->TiledResourceBarrier(nullptr, nullptr);
-        mD3d11On12DeviceContext->Flush();
+        return mD3d11On12Device;
     }
 
     const D3D12DeviceInfo& Device::GetDeviceInfo() const {

src/dawn_native/d3d12/DeviceD3D12.h

@@ -124,13 +124,13 @@ namespace dawn_native { namespace d3d12 {
 
         Ref<TextureBase> CreateExternalTexture(const TextureDescriptor* descriptor,
                                                ComPtr<ID3D12Resource> d3d12Texture,
+                                               Ref<D3D11on12ResourceCacheEntry> d3d11on12Resource,
                                                ExternalMutexSerial acquireMutexKey,
                                                ExternalMutexSerial releaseMutexKey,
                                                bool isSwapChainTexture,
                                                bool isInitialized);
-        ResultOrError<ComPtr<IDXGIKeyedMutex>> CreateKeyedMutexForTexture(
-            ID3D12Resource* d3d12Resource);
-        void ReleaseKeyedMutexForTexture(ComPtr<IDXGIKeyedMutex> dxgiKeyedMutex);
+
+        ComPtr<ID3D11On12Device> GetOrCreateD3D11on12Device();
 
         void InitTogglesFromDriver();
 
@@ -192,9 +192,8 @@ namespace dawn_native { namespace d3d12 {
         ComPtr<ID3D12CommandQueue> mCommandQueue;
         ComPtr<ID3D12SharingContract> mD3d12SharingContract;
 
-        // 11on12 device and device context corresponding to mCommandQueue
+        // 11on12 device corresponding to mCommandQueue
         ComPtr<ID3D11On12Device> mD3d11On12Device;
-        ComPtr<ID3D11DeviceContext2> mD3d11On12DeviceContext;
 
         ComPtr<ID3D12CommandSignature> mDispatchIndirectSignature;
         ComPtr<ID3D12CommandSignature> mDrawIndirectSignature;

src/dawn_native/d3d12/TextureD3D12.cpp

@@ -21,6 +21,7 @@
 #include "dawn_native/Error.h"
 #include "dawn_native/d3d12/BufferD3D12.h"
 #include "dawn_native/d3d12/CommandRecordingContext.h"
+#include "dawn_native/d3d12/D3D11on12Util.h"
 #include "dawn_native/d3d12/D3D12Error.h"
 #include "dawn_native/d3d12/DeviceD3D12.h"
 #include "dawn_native/d3d12/HeapD3D12.h"
@@ -418,18 +419,20 @@ namespace dawn_native { namespace d3d12 {
     }
 
     // static
-    ResultOrError<Ref<Texture>> Texture::CreateExternalImage(Device* device,
-                                                             const TextureDescriptor* descriptor,
-                                                             ComPtr<ID3D12Resource> d3d12Texture,
-                                                             ExternalMutexSerial acquireMutexKey,
-                                                             ExternalMutexSerial releaseMutexKey,
-                                                             bool isSwapChainTexture,
-                                                             bool isInitialized) {
+    ResultOrError<Ref<Texture>> Texture::CreateExternalImage(
+        Device* device,
+        const TextureDescriptor* descriptor,
+        ComPtr<ID3D12Resource> d3d12Texture,
+        Ref<D3D11on12ResourceCacheEntry> d3d11on12Resource,
+        ExternalMutexSerial acquireMutexKey,
+        ExternalMutexSerial releaseMutexKey,
+        bool isSwapChainTexture,
+        bool isInitialized) {
         Ref<Texture> dawnTexture =
             AcquireRef(new Texture(device, descriptor, TextureState::OwnedExternal));
-        DAWN_TRY(dawnTexture->InitializeAsExternalTexture(descriptor, std::move(d3d12Texture),
-                                                          acquireMutexKey, releaseMutexKey,
-                                                          isSwapChainTexture));
+        DAWN_TRY(dawnTexture->InitializeAsExternalTexture(
+            descriptor, std::move(d3d12Texture), std::move(d3d11on12Resource), acquireMutexKey,
+            releaseMutexKey, isSwapChainTexture));
 
         // Importing a multi-planar format must be initialized. This is required because
         // a shared multi-planar format cannot be initialized by Dawn.
@@ -453,22 +456,20 @@ namespace dawn_native { namespace d3d12 {
         return std::move(dawnTexture);
     }
 
-    MaybeError Texture::InitializeAsExternalTexture(const TextureDescriptor* descriptor,
-                                                    ComPtr<ID3D12Resource> d3d12Texture,
-                                                    ExternalMutexSerial acquireMutexKey,
-                                                    ExternalMutexSerial releaseMutexKey,
-                                                    bool isSwapChainTexture) {
-        Device* dawnDevice = ToBackend(GetDevice());
-
-        ComPtr<IDXGIKeyedMutex> dxgiKeyedMutex;
-        DAWN_TRY_ASSIGN(dxgiKeyedMutex, dawnDevice->CreateKeyedMutexForTexture(d3d12Texture.Get()));
-
-        DAWN_TRY(CheckHRESULT(dxgiKeyedMutex->AcquireSync(uint64_t(acquireMutexKey), INFINITE),
+    MaybeError Texture::InitializeAsExternalTexture(
+        const TextureDescriptor* descriptor,
+        ComPtr<ID3D12Resource> d3d12Texture,
+        Ref<D3D11on12ResourceCacheEntry> d3d11on12Resource,
+        ExternalMutexSerial acquireMutexKey,
+        ExternalMutexSerial releaseMutexKey,
+        bool isSwapChainTexture) {
+        DAWN_TRY(CheckHRESULT(d3d11on12Resource->GetDXGIKeyedMutex()->AcquireSync(
+                                  uint64_t(acquireMutexKey), INFINITE),
                               "D3D12 acquiring shared mutex"));
 
         mAcquireMutexKey = acquireMutexKey;
         mReleaseMutexKey = releaseMutexKey;
-        mDxgiKeyedMutex = std::move(dxgiKeyedMutex);
+        mD3D11on12Resource = std::move(d3d11on12Resource);
         mSwapChainTexture = isSwapChainTexture;
 
         D3D12_RESOURCE_DESC desc = d3d12Texture->GetDesc();
@@ -583,9 +584,8 @@ namespace dawn_native { namespace d3d12 {
         // ID3D12SharingContract::Present.
         mSwapChainTexture = false;
 
-        if (mDxgiKeyedMutex != nullptr) {
-            mDxgiKeyedMutex->ReleaseSync(uint64_t(mReleaseMutexKey));
-            device->ReleaseKeyedMutexForTexture(std::move(mDxgiKeyedMutex));
+        if (mD3D11on12Resource != nullptr) {
+            mD3D11on12Resource->GetDXGIKeyedMutex()->ReleaseSync(uint64_t(mReleaseMutexKey));
         }
     }
 
@@ -754,7 +754,7 @@ namespace dawn_native { namespace d3d12 {
         // Textures with keyed mutexes can be written from other graphics queues. Hence, they
         // must be acquired before command list submission to ensure work from the other queues
         // has finished. See Device::ExecuteCommandContext.
-        if (mDxgiKeyedMutex != nullptr) {
+        if (mD3D11on12Resource != nullptr) {
             commandContext->AddToSharedTextureList(this);
         }
     }

src/dawn_native/d3d12/TextureD3D12.h

@@ -28,6 +28,7 @@ namespace dawn_native { namespace d3d12 {
 
     class CommandRecordingContext;
     class Device;
+    class D3D11on12ResourceCacheEntry;
 
     DXGI_FORMAT D3D12TextureFormat(wgpu::TextureFormat format);
     MaybeError ValidateD3D12TextureCanBeWrapped(ID3D12Resource* d3d12Resource,
@@ -39,13 +40,15 @@ namespace dawn_native { namespace d3d12 {
       public:
         static ResultOrError<Ref<Texture>> Create(Device* device,
                                                   const TextureDescriptor* descriptor);
-        static ResultOrError<Ref<Texture>> CreateExternalImage(Device* device,
-                                                               const TextureDescriptor* descriptor,
-                                                               ComPtr<ID3D12Resource> d3d12Texture,
-                                                               ExternalMutexSerial acquireMutexKey,
-                                                               ExternalMutexSerial releaseMutexKey,
-                                                               bool isSwapChainTexture,
-                                                               bool isInitialized);
+        static ResultOrError<Ref<Texture>> CreateExternalImage(
+            Device* device,
+            const TextureDescriptor* descriptor,
+            ComPtr<ID3D12Resource> d3d12Texture,
+            Ref<D3D11on12ResourceCacheEntry> d3d11on12Resource,
+            ExternalMutexSerial acquireMutexKey,
+            ExternalMutexSerial releaseMutexKey,
+            bool isSwapChainTexture,
+            bool isInitialized);
         static ResultOrError<Ref<Texture>> Create(Device* device,
                                                   const TextureDescriptor* descriptor,
                                                   ComPtr<ID3D12Resource> d3d12Texture);
@@ -89,6 +92,7 @@ namespace dawn_native { namespace d3d12 {
         MaybeError InitializeAsInternalTexture();
         MaybeError InitializeAsExternalTexture(const TextureDescriptor* descriptor,
                                                ComPtr<ID3D12Resource> d3d12Texture,
+                                               Ref<D3D11on12ResourceCacheEntry> d3d11on12Resource,
                                                ExternalMutexSerial acquireMutexKey,
                                                ExternalMutexSerial releaseMutexKey,
                                                bool isSwapChainTexture);
@@ -127,7 +131,7 @@ namespace dawn_native { namespace d3d12 {
 
         ExternalMutexSerial mAcquireMutexKey = ExternalMutexSerial(0);
         ExternalMutexSerial mReleaseMutexKey = ExternalMutexSerial(0);
-        ComPtr<IDXGIKeyedMutex> mDxgiKeyedMutex;
+        Ref<D3D11on12ResourceCacheEntry> mD3D11on12Resource;
     };
 
     class TextureView final : public TextureViewBase {

src/include/dawn_native/D3D12Backend.h

@@ -29,6 +29,9 @@ struct ID3D12Device;
 struct ID3D12Resource;
 
 namespace dawn_native { namespace d3d12 {
+
+    class D3D11on12ResourceCache;
+
     DAWN_NATIVE_EXPORT Microsoft::WRL::ComPtr<ID3D12Device> GetD3D12Device(WGPUDevice device);
     DAWN_NATIVE_EXPORT DawnSwapChainImplementation CreateNativeSwapChainImpl(WGPUDevice device,
                                                                              HWND window);
@@ -62,6 +65,8 @@ namespace dawn_native { namespace d3d12 {
 
     class DAWN_NATIVE_EXPORT ExternalImageDXGI {
       public:
+        ~ExternalImageDXGI();
+
         // Note: SharedHandle must be a handle to a texture object.
         static std::unique_ptr<ExternalImageDXGI> Create(
             WGPUDevice device,
@@ -85,6 +90,8 @@ namespace dawn_native { namespace d3d12 {
         WGPUTextureFormat mFormat;
         uint32_t mMipLevelCount;
         uint32_t mSampleCount;
+
+        std::unique_ptr<D3D11on12ResourceCache> mD3D11on12ResourceCache;
     };
 
     struct DAWN_NATIVE_EXPORT AdapterDiscoveryOptions : public AdapterDiscoveryOptionsBase {

src/tests/end2end/D3D12ResourceWrappingTests.cpp

@@ -319,18 +319,21 @@ class D3D12SharedHandleUsageTests : public D3D12ResourceTestBase {
     }
 
     // Clear a texture on a given device
-    void ClearImage(wgpu::Texture wrappedTexture, const wgpu::Color& clearColor) {
+    void ClearImage(wgpu::Texture wrappedTexture,
+                    const wgpu::Color& clearColor,
+                    wgpu::Device wgpuDevice) {
         wgpu::TextureView wrappedView = wrappedTexture.CreateView();
 
         // Submit a clear operation
         utils::ComboRenderPassDescriptor renderPassDescriptor({wrappedView}, {});
         renderPassDescriptor.cColorAttachments[0].clearColor = clearColor;
 
-        wgpu::CommandEncoder encoder = device.CreateCommandEncoder();
+        wgpu::CommandEncoder encoder = wgpuDevice.CreateCommandEncoder();
         wgpu::RenderPassEncoder pass = encoder.BeginRenderPass(&renderPassDescriptor);
         pass.EndPass();
 
         wgpu::CommandBuffer commands = encoder.Finish();
+        wgpu::Queue queue = wgpuDevice.GetQueue();
         queue.Submit(1, &commands);
     }
 
@@ -517,7 +520,7 @@ TEST_P(D3D12SharedHandleUsageTests, ClearInD3D12ReadbackInD3D11) {
     ASSERT_NE(dawnTexture.Get(), nullptr);
 
     const wgpu::Color d3d12ClearColor{0.0f, 0.0f, 1.0f, 1.0f};
-    ClearImage(dawnTexture, d3d12ClearColor);
+    ClearImage(dawnTexture, d3d12ClearColor, device);
 
     dawnTexture.Destroy();
 
@@ -547,10 +550,10 @@ TEST_P(D3D12SharedHandleUsageTests, ClearTwiceInD3D12ReadbackInD3D11) {
     ASSERT_NE(dawnTexture.Get(), nullptr);
 
     const wgpu::Color d3d12ClearColor1{0.0f, 0.0f, 1.0f, 1.0f};
-    ClearImage(dawnTexture, d3d12ClearColor1);
+    ClearImage(dawnTexture, d3d12ClearColor1, device);
 
     const wgpu::Color d3d12ClearColor2{0.0f, 1.0f, 1.0f, 1.0f};
-    ClearImage(dawnTexture, d3d12ClearColor2);
+    ClearImage(dawnTexture, d3d12ClearColor2, device);
 
     dawnTexture.Destroy();
 
@@ -594,7 +597,7 @@ TEST_P(D3D12SharedHandleUsageTests, ReuseExternalImage) {
     {
         const wgpu::Color solidRed{1.0f, 0.0f, 0.0f, 1.0f};
         ASSERT_NE(texture.Get(), nullptr);
-        ClearImage(texture.Get(), solidRed);
+        ClearImage(texture.Get(), solidRed, device);
 
         EXPECT_PIXEL_RGBA8_EQ(RGBA8(0xFF, 0, 0, 0xFF), texture.Get(), 0, 0);
     }
@@ -620,7 +623,7 @@ TEST_P(D3D12SharedHandleUsageTests, ReuseExternalImage) {
     {
         const wgpu::Color solidBlue{0.0f, 0.0f, 1.0f, 1.0f};
         ASSERT_NE(texture.Get(), nullptr);
-        ClearImage(texture.Get(), solidBlue);
+        ClearImage(texture.Get(), solidBlue, device);
 
         EXPECT_PIXEL_RGBA8_EQ(RGBA8(0, 0, 0xFF, 0xFF), texture.Get(), 0, 0);
     }
@@ -653,5 +656,54 @@ TEST_P(D3D12SharedHandleUsageTests, ExternalImageUsage) {
     ASSERT_NE(texture.Get(), nullptr);
 }
 
+// Verify two Dawn devices can reuse the same external image.
+TEST_P(D3D12SharedHandleUsageTests, ReuseExternalImageWithMultipleDevices) {
+    DAWN_TEST_UNSUPPORTED_IF(UsesWire());
+
+    wgpu::Texture texture;
+    ComPtr<ID3D11Texture2D> d3d11Texture;
+    std::unique_ptr<dawn_native::d3d12::ExternalImageDXGI> externalImage;
+
+    // Create the Dawn texture then clear it to red using the first (default) device.
+    WrapSharedHandle(&baseDawnDescriptor, &baseD3dDescriptor, &texture, &d3d11Texture,
+                     &externalImage);
+    const wgpu::Color solidRed{1.0f, 0.0f, 0.0f, 1.0f};
+    ASSERT_NE(texture.Get(), nullptr);
+    ClearImage(texture.Get(), solidRed, device);
+
+    EXPECT_PIXEL_RGBA8_EQ(RGBA8(0xFF, 0, 0, 0xFF), texture.Get(), 0, 0);
+
+    // Release the texture so we can re-acquire another one from the same external image.
+    texture.Destroy();
+
+    // Create the Dawn texture then clear it to blue using the second device.
+    dawn_native::d3d12::ExternalImageAccessDescriptorDXGIKeyedMutex externalAccessDesc;
+    externalAccessDesc.acquireMutexKey = 1;
+    externalAccessDesc.releaseMutexKey = 2;
+    externalAccessDesc.usage = static_cast<WGPUTextureUsageFlags>(baseDawnDescriptor.usage);
+
+    wgpu::Device otherDevice = wgpu::Device::Acquire(GetAdapter().CreateDevice());
+
+    wgpu::Texture otherTexture = wgpu::Texture::Acquire(
+        externalImage->ProduceTexture(otherDevice.Get(), &externalAccessDesc));
+
+    ASSERT_NE(otherTexture.Get(), nullptr);
+    const wgpu::Color solidBlue{0.0f, 0.0f, 1.0f, 1.0f};
+    ClearImage(otherTexture.Get(), solidBlue, otherDevice);
+
+    otherTexture.Destroy();
+
+    // Re-create the Dawn texture using the first (default) device.
+    externalAccessDesc.acquireMutexKey = 2;
+    externalAccessDesc.isInitialized = true;
+    texture =
+        wgpu::Texture::Acquire(externalImage->ProduceTexture(device.Get(), &externalAccessDesc));
+    ASSERT_NE(texture.Get(), nullptr);
+
+    // Ensure the texture is still blue.
+
+    EXPECT_PIXEL_RGBA8_EQ(RGBA8(0, 0, 0xFF, 0xFF), texture.Get(), 0, 0);
+}
+
 DAWN_INSTANTIATE_TEST(D3D12SharedHandleValidation, D3D12Backend());
 DAWN_INSTANTIATE_TEST(D3D12SharedHandleUsageTests, D3D12Backend());