Add workaround to blit depth to depth

Fixes T2T depth copies on Mac Intel where the destination
subresource is a non-zero mip/layer.

Fixed: dawn:1083
Change-Id: If344b46c3fd436d857906850fc0ac5ccb4b93e1d
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/117592
Commit-Queue: Austin Eng <enga@chromium.org>
Reviewed-by: Corentin Wallez <cwallez@chromium.org>
Kokoro: Kokoro <noreply+kokoro@google.com>

src/dawn/native/BUILD.gn

@@ -201,6 +201,8 @@ source_set("sources") {
     "BindingInfo.h",
     "BlitBufferToDepthStencil.cpp",
     "BlitBufferToDepthStencil.h",
+    "BlitDepthToDepth.cpp",
+    "BlitDepthToDepth.h",
     "Blob.cpp",
     "Blob.h",
     "BlobCache.cpp",

src/dawn/native/BlitDepthToDepth.cpp

@@ -0,0 +1,227 @@
+// Copyright 2023 The Dawn Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "dawn/native/BlitDepthToDepth.h"
+
+#include <utility>
+#include <vector>
+
+#include "dawn/common/Assert.h"
+#include "dawn/native/BindGroup.h"
+#include "dawn/native/CommandEncoder.h"
+#include "dawn/native/Device.h"
+#include "dawn/native/InternalPipelineStore.h"
+#include "dawn/native/RenderPassEncoder.h"
+#include "dawn/native/RenderPipeline.h"
+
+namespace dawn::native {
+
+namespace {
+
+constexpr char kBlitToDepthShaders[] = R"(
+
+@vertex fn vert_fullscreen_quad(
+  @builtin(vertex_index) vertex_index : u32,
+) -> @builtin(position) vec4<f32> {
+  const pos = array<vec2<f32>, 3>(
+      vec2<f32>(-1.0, -1.0),
+      vec2<f32>( 3.0, -1.0),
+      vec2<f32>(-1.0,  3.0));
+  return vec4<f32>(pos[vertex_index], 0.0, 1.0);
+}
+
+@group(0) @binding(0) var src_tex : texture_depth_2d;
+
+// Load the depth value and return it as the frag_depth.
+@fragment fn blit_to_depth(@builtin(position) position : vec4<f32>) -> @builtin(frag_depth) f32 {
+  return textureLoad(src_tex, vec2<u32>(position.xy), 0);
+}
+
+)";
+
+ResultOrError<Ref<RenderPipelineBase>> GetOrCreateDepthBlitPipeline(DeviceBase* device,
+                                                                    wgpu::TextureFormat format) {
+    InternalPipelineStore* store = device->GetInternalPipelineStore();
+    {
+        auto it = store->depthBlitPipelines.find(format);
+        if (it != store->depthBlitPipelines.end()) {
+            return it->second;
+        }
+    }
+
+    ShaderModuleWGSLDescriptor wgslDesc = {};
+    ShaderModuleDescriptor shaderModuleDesc = {};
+    shaderModuleDesc.nextInChain = &wgslDesc;
+    wgslDesc.source = kBlitToDepthShaders;
+
+    Ref<ShaderModuleBase> shaderModule;
+    DAWN_TRY_ASSIGN(shaderModule, device->CreateShaderModule(&shaderModuleDesc));
+
+    FragmentState fragmentState = {};
+    fragmentState.module = shaderModule.Get();
+    fragmentState.entryPoint = "blit_to_depth";
+
+    DepthStencilState dsState = {};
+    dsState.format = format;
+    dsState.depthWriteEnabled = true;
+
+    RenderPipelineDescriptor renderPipelineDesc = {};
+    renderPipelineDesc.vertex.module = shaderModule.Get();
+    renderPipelineDesc.vertex.entryPoint = "vert_fullscreen_quad";
+    renderPipelineDesc.depthStencil = &dsState;
+    renderPipelineDesc.fragment = &fragmentState;
+
+    Ref<RenderPipelineBase> pipeline;
+    DAWN_TRY_ASSIGN(pipeline, device->CreateRenderPipeline(&renderPipelineDesc));
+
+    store->depthBlitPipelines[format] = pipeline;
+    return pipeline;
+}
+
+}  // namespace
+
+MaybeError BlitDepthToDepth(DeviceBase* device,
+                            CommandEncoder* commandEncoder,
+                            const TextureCopy& src,
+                            const TextureCopy& dst,
+                            const Extent3D& copyExtent) {
+    // ASSERT that the texture have depth and are not multisampled.
+    ASSERT(src.texture->GetFormat().HasDepth());
+    ASSERT(dst.texture->GetFormat().HasDepth());
+    ASSERT(src.texture->GetSampleCount() == 1u);
+    ASSERT(dst.texture->GetSampleCount() == 1u);
+
+    // Note: because depth texture subresources must be copied in full, this blit
+    // does not need to handle copy subrects.
+
+    // Allow internal usages since we need to use the source as a texture binding, and
+    // the destination as a render attachment.
+    auto scope = commandEncoder->MakeInternalUsageScope();
+
+    Ref<RenderPipelineBase> pipeline;
+    DAWN_TRY_ASSIGN(pipeline,
+                    GetOrCreateDepthBlitPipeline(device, dst.texture->GetFormat().format));
+
+    Ref<BindGroupLayoutBase> bgl;
+    DAWN_TRY_ASSIGN(bgl, pipeline->GetBindGroupLayout(0));
+
+    // TODO(crbug.com/dawn/838)
+    // Metal Intel drivers incorrectly create texture views starting at a  nonzero array layer. They
+    // also don't textureLoad in the shader at a non-zero array index correctly. Workaround this
+    // issue by copying the non-zero array slices to a single-layer texture. That texture will be be
+    // sampled as the source instead.
+    std::vector<Ref<TextureViewBase>> srcViews;
+    srcViews.reserve(copyExtent.depthOrArrayLayers);
+    for (uint32_t z = 0; z < copyExtent.depthOrArrayLayers; ++z) {
+        uint32_t layer = src.origin.z + z;
+        Ref<TextureViewBase> srcView;
+        if (layer == 0u) {
+            // The zero'th slice. We can use the original texture.
+            TextureViewDescriptor viewDesc = {};
+            viewDesc.aspect = wgpu::TextureAspect::DepthOnly;
+            viewDesc.dimension = wgpu::TextureViewDimension::e2D;
+            viewDesc.baseMipLevel = src.mipLevel;
+            viewDesc.mipLevelCount = 1u;
+            DAWN_TRY_ASSIGN(srcView, src.texture->CreateView(&viewDesc));
+        } else {
+            // Create a single-layer intermediate texture to use as the texture source.
+            TextureDescriptor intermediateTexDesc = {};
+            intermediateTexDesc.format = src.texture->GetFormat().format;
+            intermediateTexDesc.usage =
+                wgpu::TextureUsage::TextureBinding | wgpu::TextureUsage::CopyDst;
+            intermediateTexDesc.size = {copyExtent.width, copyExtent.height};
+
+            Ref<TextureBase> intermediateTexture;
+            DAWN_TRY_ASSIGN(intermediateTexture, device->CreateTexture(&intermediateTexDesc));
+
+            // Copy from the original texture source into the intermediate.
+            {
+                ImageCopyTexture intermediateSrc;
+                intermediateSrc.texture = src.texture.Get();
+                intermediateSrc.mipLevel = src.mipLevel;
+                intermediateSrc.origin = {0, 0, layer};
+                intermediateSrc.aspect = wgpu::TextureAspect::All;
+
+                ImageCopyTexture intermediateDst;
+                intermediateDst.texture = intermediateTexture.Get();
+                intermediateDst.mipLevel = 0u;
+                intermediateDst.origin = {0, 0, 0};
+                intermediateDst.aspect = wgpu::TextureAspect::All;
+
+                // Note: This does not recurse infinitely because the workaround to
+                // blit depth is not needed if the destination level and layer is 0,
+                // and the copy depth is 1.
+                commandEncoder->APICopyTextureToTexture(&intermediateSrc, &intermediateDst,
+                                                        &intermediateTexDesc.size);
+            }
+
+            // Create a texture view pointing to the intermediate texture.
+            TextureViewDescriptor viewDesc = {};
+            viewDesc.aspect = wgpu::TextureAspect::DepthOnly;
+            DAWN_TRY_ASSIGN(srcView, intermediateTexture->CreateView(&viewDesc));
+        }
+        srcViews.push_back(std::move(srcView));
+    }
+
+    // For each copied layer, blit from the source into the destination.
+    for (uint32_t z = 0; z < copyExtent.depthOrArrayLayers; ++z) {
+        Ref<BindGroupBase> bindGroup;
+        {
+            BindGroupEntry bgEntry = {};
+            bgEntry.binding = 0;
+            bgEntry.textureView = srcViews[z].Get();
+
+            BindGroupDescriptor bgDesc = {};
+            bgDesc.layout = bgl.Get();
+            bgDesc.entryCount = 1;
+            bgDesc.entries = &bgEntry;
+            DAWN_TRY_ASSIGN(bindGroup,
+                            device->CreateBindGroup(&bgDesc, UsageValidationMode::Internal));
+        }
+
+        Ref<TextureViewBase> dstView;
+        {
+            TextureViewDescriptor viewDesc = {};
+            viewDesc.dimension = wgpu::TextureViewDimension::e2D;
+            viewDesc.baseArrayLayer = dst.origin.z + z;
+            viewDesc.arrayLayerCount = 1;
+            viewDesc.baseMipLevel = dst.mipLevel;
+            viewDesc.mipLevelCount = 1;
+            DAWN_TRY_ASSIGN(dstView, dst.texture->CreateView(&viewDesc));
+        }
+
+        RenderPassDepthStencilAttachment dsAttachment = {};
+        dsAttachment.view = dstView.Get();
+        dsAttachment.depthLoadOp = wgpu::LoadOp::Load;
+        dsAttachment.depthStoreOp = wgpu::StoreOp::Store;
+        if (dst.texture->GetFormat().HasStencil()) {
+            dsAttachment.stencilLoadOp = wgpu::LoadOp::Load;
+            dsAttachment.stencilStoreOp = wgpu::StoreOp::Store;
+        }
+
+        RenderPassDescriptor rpDesc = {};
+        rpDesc.depthStencilAttachment = &dsAttachment;
+
+        // Draw to perform the blit.
+        Ref<RenderPassEncoder> pass = AcquireRef(commandEncoder->APIBeginRenderPass(&rpDesc));
+        pass->APISetBindGroup(0, bindGroup.Get());
+        pass->APISetPipeline(pipeline.Get());
+        pass->APIDraw(3, 1, 0, 0);
+        pass->APIEnd();
+    }
+
+    return {};
+}
+
+}  // namespace dawn::native

src/dawn/native/BlitDepthToDepth.h

@@ -0,0 +1,35 @@
+// Copyright 2023 The Dawn Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef SRC_DAWN_NATIVE_BLITDEPTHTODEPTH_H_
+#define SRC_DAWN_NATIVE_BLITDEPTHTODEPTH_H_
+
+#include "dawn/native/Error.h"
+
+namespace dawn::native {
+
+class DeviceBase;
+class CommandEncoder;
+struct TextureCopy;
+struct Extent3D;
+
+MaybeError BlitDepthToDepth(DeviceBase* device,
+                            CommandEncoder* commandEncoder,
+                            const TextureCopy& src,
+                            const TextureCopy& dst,
+                            const Extent3D& copyExtent);
+
+}  // namespace dawn::native
+
+#endif  // SRC_DAWN_NATIVE_BLITDEPTHTODEPTH_H_

src/dawn/native/CMakeLists.txt

@@ -49,6 +49,8 @@ target_sources(dawn_native PRIVATE
     "BindingInfo.h"
     "BlitBufferToDepthStencil.cpp"
     "BlitBufferToDepthStencil.h"
+    "BlitDepthToDepth.cpp"
+    "BlitDepthToDepth.h"
     "Blob.cpp"
     "Blob.h"
     "BlobCache.cpp"

src/dawn/native/CommandEncoder.cpp

@@ -23,6 +23,7 @@
 #include "dawn/native/ApplyClearColorValueWithDrawHelper.h"
 #include "dawn/native/BindGroup.h"
 #include "dawn/native/BlitBufferToDepthStencil.h"
+#include "dawn/native/BlitDepthToDepth.h"
 #include "dawn/native/Buffer.h"
 #include "dawn/native/ChainUtils_autogen.h"
 #include "dawn/native/CommandBuffer.h"
@@ -1368,18 +1369,44 @@ void CommandEncoder::APICopyTextureToTextureHelper(const ImageCopyTexture* sourc
             mTopLevelTextures.insert(source->texture);
             mTopLevelTextures.insert(destination->texture);
 
-            CopyTextureToTextureCmd* copy =
-                allocator->Allocate<CopyTextureToTextureCmd>(Command::CopyTextureToTexture);
-            copy->source.texture = source->texture;
-            copy->source.origin = source->origin;
-            copy->source.mipLevel = source->mipLevel;
-            copy->source.aspect = ConvertAspect(source->texture->GetFormat(), source->aspect);
-            copy->destination.texture = destination->texture;
-            copy->destination.origin = destination->origin;
-            copy->destination.mipLevel = destination->mipLevel;
-            copy->destination.aspect =
-                ConvertAspect(destination->texture->GetFormat(), destination->aspect);
-            copy->copySize = *copySize;
+            Aspect aspect = ConvertAspect(source->texture->GetFormat(), source->aspect);
+            ASSERT(aspect == ConvertAspect(destination->texture->GetFormat(), destination->aspect));
+
+            TextureCopy src;
+            src.texture = source->texture;
+            src.origin = source->origin;
+            src.mipLevel = source->mipLevel;
+            src.aspect = aspect;
+
+            TextureCopy dst;
+            dst.texture = destination->texture;
+            dst.origin = destination->origin;
+            dst.mipLevel = destination->mipLevel;
+            dst.aspect = aspect;
+
+            const bool blitDepth =
+                (aspect & Aspect::Depth) &&
+                GetDevice()->IsToggleEnabled(
+                    Toggle::UseBlitForDepthTextureToTextureCopyToNonzeroSubresource) &&
+                (dst.mipLevel > 0 || dst.origin.z > 0 || copySize->depthOrArrayLayers > 1);
+
+            // If we're not using a blit, or there are aspects other than depth,
+            // issue the copy. This is because if there's also stencil, we still need the copy
+            // command to copy the stencil portion.
+            if (!blitDepth || aspect != Aspect::Depth) {
+                CopyTextureToTextureCmd* copy =
+                    allocator->Allocate<CopyTextureToTextureCmd>(Command::CopyTextureToTexture);
+                copy->source = src;
+                copy->destination = dst;
+                copy->copySize = *copySize;
+            }
+
+            // Use a blit to copy the depth aspect.
+            if (blitDepth) {
+                DAWN_TRY_CONTEXT(BlitDepthToDepth(GetDevice(), this, src, dst, *copySize),
+                                 "copying depth aspect from %s to %s using blit workaround.",
+                                 source->texture, destination->texture);
+            }
 
             return {};
         },

src/dawn/native/InternalPipelineStore.h

@@ -65,6 +65,8 @@ struct InternalPipelineStore {
         std::array<Ref<RenderPipelineBase>, 8> setStencilPipelines;
     };
     std::unordered_map<wgpu::TextureFormat, BlitR8ToStencilPipelines> blitR8ToStencilPipelines;
+
+    std::unordered_map<wgpu::TextureFormat, Ref<RenderPipelineBase>> depthBlitPipelines;
 };
 
 }  // namespace dawn::native

src/dawn/native/Texture.cpp

@@ -575,6 +575,15 @@ TextureBase::TextureBase(DeviceBase* device,
         // in a render pass.
         AddInternalUsage(wgpu::TextureUsage::RenderAttachment);
     }
+    if (mFormat.HasDepth() &&
+        device->IsToggleEnabled(Toggle::UseBlitForDepthTextureToTextureCopyToNonzeroSubresource)) {
+        if (mInternalUsage & wgpu::TextureUsage::CopySrc) {
+            AddInternalUsage(wgpu::TextureUsage::TextureBinding);
+        }
+        if (mInternalUsage & wgpu::TextureUsage::CopyDst) {
+            AddInternalUsage(wgpu::TextureUsage::RenderAttachment);
+        }
+    }
 }
 
 TextureBase::~TextureBase() = default;

src/dawn/native/Toggles.cpp

@@ -354,6 +354,11 @@ static constexpr ToggleEnumAndInfoList kToggleNameAndInfoList = {{
       "texture. Works around an issue where stencil writes by copy commands are not visible "
       "to a render or compute pass.",
       "https://crbug.com/dawn/1389"}},
+    {Toggle::UseBlitForDepthTextureToTextureCopyToNonzeroSubresource,
+     {"use_blit_for_depth_texture_to_texture_copy_to_nonzero_subresource",
+      "Use a blit to copy from a depth texture to the nonzero subresource of a depth texture. "
+      "Works around an issue where nonzero layers are not written.",
+      "https://crbug.com/dawn/1083"}},
     {Toggle::DisallowDeprecatedAPIs,
      {"disallow_deprecated_apis",
       "Disallow all deprecated paths by changing the deprecation warnings to validation error for "

src/dawn/native/Toggles.h

@@ -86,6 +86,7 @@ enum class Toggle {
     MetalUseBothDepthAndStencilAttachmentsForCombinedDepthStencilFormats,
     UseBlitForBufferToDepthTextureCopy,
     UseBlitForBufferToStencilTextureCopy,
+    UseBlitForDepthTextureToTextureCopyToNonzeroSubresource,
     DisallowDeprecatedAPIs,
 
     // Unresolved issues.

src/dawn/native/metal/DeviceMTL.mm

@@ -263,6 +263,7 @@ void Device::InitTogglesFromDriver() {
                   true);
         SetToggle(Toggle::UseBlitForBufferToStencilTextureCopy, true);
         SetToggle(Toggle::UseBlitForBufferToDepthTextureCopy, true);
+        SetToggle(Toggle::UseBlitForDepthTextureToTextureCopyToNonzeroSubresource, true);
 
         if ([NSProcessInfo.processInfo
                 isOperatingSystemAtLeastVersion:NSOperatingSystemVersion{12, 0, 0}]) {