Refactor code to have rudimentary support for using SPVC

This is the first step for having a fully operational SPVC usage
path. This version of SPVC integration uses SPVC for setting up the
options to the compiler, but a lot of the actual interaction with
spirv-cross is done in Dawn, just via SPVC's compiler object.

Future CLs will migrate more of the spirv-cross interaction to using
the SPVC API, eventually removing the need for Dawn to know about
spirv-cross.

BUG=dawn:288

Change-Id: I68e0773f910d7fe967235b6987b3debe1d13883f
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/14143
Commit-Queue: Ryan Harrison <rharrison@chromium.org>
Reviewed-by: Austin Eng <enga@chromium.org>

BUILD.gn

@@ -134,7 +134,6 @@ source_set("libdawn_native_sources") {
     ":libdawn_native_headers",
     ":libdawn_native_utils_gen",
     "${dawn_root}/src/common",
-    "${dawn_shaderc_dir}:libshaderc_spvc",
     "${dawn_shaderc_dir}:spirv_cross",
     "${dawn_spirv_tools_dir}:spvtools_val",
   ]
@@ -149,6 +148,7 @@ source_set("libdawn_native_sources") {
   # libdawn_native target
   public_deps = [
     ":dawn_platform",
+    "${dawn_shaderc_dir}:libshaderc_spvc",
   ]
 
   sources = get_target_outputs(":libdawn_native_utils_gen")

DEPS

@@ -69,7 +69,7 @@ deps = {
     'condition': 'dawn_standalone',
   },
   'third_party/shaderc': {
-    'url': '{chromium_git}/external/github.com/google/shaderc@30f0559dd2782b3c65ad3840da2bf8e81711e9b7',
+    'url': '{chromium_git}/external/github.com/google/shaderc@efedd6739684bb2d4183b45af111b4942b465e5b',
     'condition': 'dawn_standalone',
   },
 

src/dawn_native/ShaderModule.h

@@ -24,6 +24,8 @@
 
 #include "dawn_native/dawn_platform.h"
 
+#include "spvc/spvc.hpp"
+
 #include <array>
 #include <bitset>
 #include <vector>
@@ -79,6 +81,9 @@ namespace dawn_native {
             bool operator()(const ShaderModuleBase* a, const ShaderModuleBase* b) const;
         };
 
+      protected:
+        shaderc_spvc::Context mSpvcContext;
+
       private:
         ShaderModuleBase(DeviceBase* device, ObjectBase::ErrorTag tag);
 

src/dawn_native/d3d12/ComputePipelineD3D12.cpp

@@ -32,8 +32,8 @@ namespace dawn_native { namespace d3d12 {
         // SPRIV-cross does matrix multiplication expecting row major matrices
         compileFlags |= D3DCOMPILE_PACK_MATRIX_ROW_MAJOR;
 
-        const ShaderModule* module = ToBackend(descriptor->computeStage.module);
+        ShaderModule* module = ToBackend(descriptor->computeStage.module);
-        const std::string& hlslSource = module->GetHLSLSource(ToBackend(GetLayout()));
+        const std::string hlslSource = module->GetHLSLSource(ToBackend(GetLayout()));
 
         ComPtr<ID3DBlob> compiledShader;
         ComPtr<ID3DBlob> errors;

src/dawn_native/d3d12/RenderPipelineD3D12.cpp

@@ -315,7 +315,7 @@ namespace dawn_native { namespace d3d12 {
 
         wgpu::ShaderStage renderStages = wgpu::ShaderStage::Vertex | wgpu::ShaderStage::Fragment;
         for (auto stage : IterateStages(renderStages)) {
-            const ShaderModule* module = nullptr;
+            ShaderModule* module = nullptr;
             const char* entryPoint = nullptr;
             const char* compileTarget = nullptr;
             D3D12_SHADER_BYTECODE* shader = nullptr;

src/dawn_native/d3d12/ShaderModuleD3D12.cpp

@@ -40,18 +40,46 @@ namespace dawn_native { namespace d3d12 {
 
     MaybeError ShaderModule::Initialize(const ShaderModuleDescriptor* descriptor) {
         mSpirv.assign(descriptor->code, descriptor->code + descriptor->codeSize);
-        spirv_cross::CompilerHLSL compiler(mSpirv);
+        if (GetDevice()->IsToggleEnabled(Toggle::UseSpvc)) {
+            shaderc_spvc::CompileOptions options;
+            shaderc_spvc_status status =
+                mSpvcContext.InitializeForHlsl(descriptor->code, descriptor->codeSize, options);
+            if (status != shaderc_spvc_status_success) {
+                return DAWN_VALIDATION_ERROR("Unable to initialize instance of spvc");
+            }
+
+            spirv_cross::Compiler* compiler =
+                reinterpret_cast<spirv_cross::Compiler*>(mSpvcContext.GetCompiler());
+            ExtractSpirvInfo(*compiler);
+        } else {
+            spirv_cross::CompilerHLSL compiler(descriptor->code, descriptor->codeSize);
             ExtractSpirvInfo(compiler);
+        }
         return {};
     }
 
-    const std::string ShaderModule::GetHLSLSource(PipelineLayout* layout) const {
+    const std::string ShaderModule::GetHLSLSource(PipelineLayout* layout) {
-        spirv_cross::CompilerHLSL compiler(mSpirv);
+        std::unique_ptr<spirv_cross::CompilerHLSL> compiler_impl;
+        spirv_cross::CompilerHLSL* compiler;
+        if (GetDevice()->IsToggleEnabled(Toggle::UseSpvc)) {
+            shaderc_spvc::CompileOptions options;
 
-        // If these options are changed, the values in DawnSPIRVCrossHLSLFastFuzzer.cpp need to be
+            options.SetHLSLShaderModel(51);
-        // updated.
+            // PointCoord and PointSize are not supported in HLSL
+            // TODO (hao.x.li@intel.com): The point_coord_compat and point_size_compat are
+            // required temporarily for https://bugs.chromium.org/p/dawn/issues/detail?id=146,
+            // but should be removed once WebGPU requires there is no gl_PointSize builtin.
+            // See https://github.com/gpuweb/gpuweb/issues/332
+            options.SetHLSLPointCoordCompat(true);
+            options.SetHLSLPointSizeCompat(true);
+
+            mSpvcContext.InitializeForHlsl(mSpirv.data(), mSpirv.size(), options);
+            compiler = reinterpret_cast<spirv_cross::CompilerHLSL*>(mSpvcContext.GetCompiler());
+            // TODO(rharrison): Check status & have some sort of meaningful error path
+        } else {
+            // If these options are changed, the values in DawnSPIRVCrossHLSLFastFuzzer.cpp need to
+            // be updated.
             spirv_cross::CompilerGLSL::Options options_glsl;
-        compiler.set_common_options(options_glsl);
 
             spirv_cross::CompilerHLSL::Options options_hlsl;
             options_hlsl.shader_model = 51;
@@ -62,7 +90,12 @@ namespace dawn_native { namespace d3d12 {
             // See https://github.com/gpuweb/gpuweb/issues/332
             options_hlsl.point_coord_compat = true;
             options_hlsl.point_size_compat = true;
-        compiler.set_hlsl_options(options_hlsl);
+
+            compiler_impl = std::make_unique<spirv_cross::CompilerHLSL>(mSpirv);
+            compiler = compiler_impl.get();
+            compiler->set_common_options(options_glsl);
+            compiler->set_hlsl_options(options_hlsl);
+        }
 
         const ModuleBindingInfo& moduleBindingInfo = GetBindingInfo();
         for (uint32_t group : IterateBitSet(layout->GetBindGroupLayoutsMask())) {
@@ -73,11 +106,18 @@ namespace dawn_native { namespace d3d12 {
                 const BindingInfo& bindingInfo = groupBindingInfo[binding];
                 if (bindingInfo.used) {
                     uint32_t bindingOffset = bindingOffsets[binding];
-                    compiler.set_decoration(bindingInfo.id, spv::DecorationBinding, bindingOffset);
+                    compiler->set_decoration(bindingInfo.id, spv::DecorationBinding, bindingOffset);
                 }
             }
         }
-        return compiler.compile();
+        if (GetDevice()->IsToggleEnabled(Toggle::UseSpvc)) {
+            shaderc_spvc::CompilationResult result;
+            mSpvcContext.CompileShader(&result);
+            // TODO(rharrison): Check status & have some sort of meaningful error path
+            return result.GetStringOutput();
+        } else {
+            return compiler->compile();
+        }
     }
 
 }}  // namespace dawn_native::d3d12

src/dawn_native/d3d12/ShaderModuleD3D12.h

@@ -27,7 +27,7 @@ namespace dawn_native { namespace d3d12 {
         static ResultOrError<ShaderModule*> Create(Device* device,
                                                    const ShaderModuleDescriptor* descriptor);
 
-        const std::string GetHLSLSource(PipelineLayout* layout) const;
+        const std::string GetHLSLSource(PipelineLayout* layout);
 
       private:
         ShaderModule(Device* device, const ShaderModuleDescriptor* descriptor);

src/dawn_native/metal/ShaderModuleMTL.mm

@@ -56,18 +56,51 @@ namespace dawn_native { namespace metal {
 
     MaybeError ShaderModule::Initialize(const ShaderModuleDescriptor* descriptor) {
         mSpirv.assign(descriptor->code, descriptor->code + descriptor->codeSize);
+        if (GetDevice()->IsToggleEnabled(Toggle::UseSpvc)) {
+            shaderc_spvc::CompileOptions options;
+            shaderc_spvc_status status =
+                mSpvcContext.InitializeForGlsl(descriptor->code, descriptor->codeSize, options);
+            if (status != shaderc_spvc_status_success) {
+                return DAWN_VALIDATION_ERROR("Unable to initialize instance of spvc");
+            }
+
+            spirv_cross::CompilerMSL* compiler =
+                reinterpret_cast<spirv_cross::CompilerMSL*>(mSpvcContext.GetCompiler());
+            ExtractSpirvInfo(*compiler);
+        } else {
             spirv_cross::CompilerMSL compiler(mSpirv);
             ExtractSpirvInfo(compiler);
+        }
         return {};
     }
 
     ShaderModule::MetalFunctionData ShaderModule::GetFunction(const char* functionName,
                                                               SingleShaderStage functionStage,
                                                               const PipelineLayout* layout) const {
-        spirv_cross::CompilerMSL compiler(mSpirv);
+        std::unique_ptr<spirv_cross::CompilerMSL> compiler_impl;
+        spirv_cross::CompilerMSL* compiler;
+        if (GetDevice()->IsToggleEnabled(Toggle::UseSpvc)) {
+            // If these options are changed, the values in DawnSPIRVCrossGLSLFastFuzzer.cpp need to
+            // be updated.
+            shaderc_spvc::CompileOptions options;
 
-        // If these options are changed, the values in DawnSPIRVCrossMSLFastFuzzer.cpp need to be
+            // Disable PointSize builtin for https://bugs.chromium.org/p/dawn/issues/detail?id=146
-        // updated.
+            // Because Metal will reject PointSize builtin if the shader is compiled into a render
+            // pipeline that uses a non-point topology.
+            // TODO (hao.x.li@intel.com): Remove this once WebGPU requires there is no
+            // gl_PointSize builtin (https://github.com/gpuweb/gpuweb/issues/332).
+            options.SetMSLEnablePointSizeBuiltIn(false);
+
+            // Always use vertex buffer 30 (the last one in the vertex buffer table) to contain
+            // the shader storage buffer lengths.
+            options.SetMSLBufferSizeBufferIndex(kBufferLengthBufferSlot);
+            mSpvcContext.InitializeForMsl(mSpirv.data(), mSpirv.size(), options);
+            // TODO(rharrison): Handle initialize failing
+
+            compiler = reinterpret_cast<spirv_cross::CompilerMSL*>(mSpvcContext.GetCompiler());
+        } else {
+            // If these options are changed, the values in DawnSPIRVCrossMSLFastFuzzer.cpp need to
+            // be updated.
             spirv_cross::CompilerMSL::Options options_msl;
 
             // Disable PointSize builtin for https://bugs.chromium.org/p/dawn/issues/detail?id=146
@@ -81,7 +114,10 @@ namespace dawn_native { namespace metal {
             // the shader storage buffer lengths.
             options_msl.buffer_size_buffer_index = kBufferLengthBufferSlot;
 
-        compiler.set_msl_options(options_msl);
+            compiler_impl = std::make_unique<spirv_cross::CompilerMSL>(mSpirv);
+            compiler = compiler_impl.get();
+            compiler->set_msl_options(options_msl);
+        }
 
         // By default SPIRV-Cross will give MSL resources indices in increasing order.
         // To make the MSL indices match the indices chosen in the PipelineLayout, we build
@@ -100,7 +136,7 @@ namespace dawn_native { namespace metal {
                     mslBinding.binding = binding;
                     mslBinding.msl_buffer = mslBinding.msl_texture = mslBinding.msl_sampler = index;
 
-                    compiler.add_msl_resource_binding(mslBinding);
+                    compiler->add_msl_resource_binding(mslBinding);
                 }
             }
         }
@@ -109,14 +145,14 @@ namespace dawn_native { namespace metal {
 
         {
             spv::ExecutionModel executionModel = SpirvExecutionModelForStage(functionStage);
-            auto size = compiler.get_entry_point(functionName, executionModel).workgroup_size;
+            auto size = compiler->get_entry_point(functionName, executionModel).workgroup_size;
             result.localWorkgroupSize = MTLSizeMake(size.x, size.y, size.z);
         }
 
         {
             // SPIRV-Cross also supports re-ordering attributes but it seems to do the correct thing
             // by default.
-            std::string msl = compiler.compile();
+            std::string msl = compiler->compile();
             NSString* mslSource = [NSString stringWithFormat:@"%s", msl.c_str()];
 
             auto mtlDevice = ToBackend(GetDevice())->GetMTLDevice();
@@ -140,7 +176,7 @@ namespace dawn_native { namespace metal {
             [library release];
         }
 
-        result.needsStorageBufferLength = compiler.needs_buffer_size_buffer();
+        result.needsStorageBufferLength = compiler->needs_buffer_size_buffer();
 
         return result;
     }

src/dawn_native/null/DeviceNull.cpp

@@ -129,9 +129,22 @@ namespace dawn_native { namespace null {
         const ShaderModuleDescriptor* descriptor) {
         auto module = new ShaderModule(this, descriptor);
 
+        if (IsToggleEnabled(Toggle::UseSpvc)) {
+            shaderc_spvc::CompileOptions options;
+            shaderc_spvc::Context context;
+            shaderc_spvc_status status =
+                context.InitializeForGlsl(descriptor->code, descriptor->codeSize, options);
+            if (status != shaderc_spvc_status_success) {
+                return DAWN_VALIDATION_ERROR("Unable to initialize instance of spvc");
+            }
+
+            spirv_cross::Compiler* compiler =
+                reinterpret_cast<spirv_cross::Compiler*>(context.GetCompiler());
+            module->ExtractSpirvInfo(*compiler);
+        } else {
             spirv_cross::Compiler compiler(descriptor->code, descriptor->codeSize);
             module->ExtractSpirvInfo(compiler);
-
+        }
         return module;
     }
     ResultOrError<SwapChainBase*> Device::CreateSwapChainImpl(

src/dawn_native/opengl/ShaderModuleGL.cpp

@@ -70,15 +70,42 @@ namespace dawn_native { namespace opengl {
     }
 
     MaybeError ShaderModule::Initialize(const ShaderModuleDescriptor* descriptor) {
-        spirv_cross::CompilerGLSL compiler(descriptor->code, descriptor->codeSize);
+        std::unique_ptr<spirv_cross::CompilerGLSL> compiler_impl;
-        // If these options are changed, the values in DawnSPIRVCrossGLSLFastFuzzer.cpp need to be
+        spirv_cross::CompilerGLSL* compiler;
-        // updated.
+
+        if (GetDevice()->IsToggleEnabled(Toggle::UseSpvc)) {
+            // If these options are changed, the values in DawnSPIRVCrossGLSLFastFuzzer.cpp need to
+            // be updated.
+            shaderc_spvc::CompileOptions options;
+
+            // The range of Z-coordinate in the clipping volume of OpenGL is [-w, w], while it is
+            // [0, w] in D3D12, Metal and Vulkan, so we should normalize it in shaders in all
+            // backends. See the documentation of
+            // spirv_cross::CompilerGLSL::Options::vertex::fixup_clipspace for more details.
+            options.SetFlipVertY(true);
+            options.SetFixupClipspace(true);
+
+            // TODO(cwallez@chromium.org): discover the backing context version and use that.
+#if defined(DAWN_PLATFORM_APPLE)
+            options.SetGLSLLanguageVersion(410);
+#else
+            options.SetGLSLLanguageVersion(440);
+#endif
+            shaderc_spvc_status status =
+                mSpvcContext.InitializeForGlsl(descriptor->code, descriptor->codeSize, options);
+            if (status != shaderc_spvc_status_success)
+                return DAWN_VALIDATION_ERROR("Unable to initialize instance of spvc");
+
+            compiler = reinterpret_cast<spirv_cross::CompilerGLSL*>(mSpvcContext.GetCompiler());
+        } else {
+            // If these options are changed, the values in DawnSPIRVCrossGLSLFastFuzzer.cpp need to
+            // be updated.
             spirv_cross::CompilerGLSL::Options options;
 
-        // The range of Z-coordinate in the clipping volume of OpenGL is [-w, w], while it is [0, w]
+            // The range of Z-coordinate in the clipping volume of OpenGL is [-w, w], while it is
-        // in D3D12, Metal and Vulkan, so we should normalize it in shaders in all backends.
+            // [0, w] in D3D12, Metal and Vulkan, so we should normalize it in shaders in all
-        // See the documentation of spirv_cross::CompilerGLSL::Options::vertex::fixup_clipspace for
+            // backends. See the documentation of
-        // more details.
+            // spirv_cross::CompilerGLSL::Options::vertex::fixup_clipspace for more details.
             options.vertex.flip_vert_y = true;
             options.vertex.fixup_clipspace = true;
 
@@ -88,30 +115,35 @@ namespace dawn_native { namespace opengl {
 #else
         options.version = 440;
 #endif
-        compiler.set_common_options(options);
 
-        ExtractSpirvInfo(compiler);
+        compiler_impl =
+            std::make_unique<spirv_cross::CompilerGLSL>(descriptor->code, descriptor->codeSize);
+        compiler = compiler_impl.get();
+        compiler->set_common_options(options);
+        }
+
+        ExtractSpirvInfo(*compiler);
 
         const auto& bindingInfo = GetBindingInfo();
 
         // Extract bindings names so that it can be used to get its location in program.
         // Now translate the separate sampler / textures into combined ones and store their info.
         // We need to do this before removing the set and binding decorations.
-        compiler.build_combined_image_samplers();
+        compiler->build_combined_image_samplers();
 
-        for (const auto& combined : compiler.get_combined_image_samplers()) {
+        for (const auto& combined : compiler->get_combined_image_samplers()) {
             mCombinedInfo.emplace_back();
 
             auto& info = mCombinedInfo.back();
             info.samplerLocation.group =
-                compiler.get_decoration(combined.sampler_id, spv::DecorationDescriptorSet);
+                compiler->get_decoration(combined.sampler_id, spv::DecorationDescriptorSet);
             info.samplerLocation.binding =
-                compiler.get_decoration(combined.sampler_id, spv::DecorationBinding);
+                compiler->get_decoration(combined.sampler_id, spv::DecorationBinding);
             info.textureLocation.group =
-                compiler.get_decoration(combined.image_id, spv::DecorationDescriptorSet);
+                compiler->get_decoration(combined.image_id, spv::DecorationDescriptorSet);
             info.textureLocation.binding =
-                compiler.get_decoration(combined.image_id, spv::DecorationBinding);
+                compiler->get_decoration(combined.image_id, spv::DecorationBinding);
-            compiler.set_name(combined.combined_id, info.GetName());
+            compiler->set_name(combined.combined_id, info.GetName());
         }
 
         // Change binding names to be "dawn_binding_<group>_<binding>".
@@ -121,14 +153,22 @@ namespace dawn_native { namespace opengl {
             for (uint32_t binding = 0; binding < kMaxBindingsPerGroup; ++binding) {
                 const auto& info = bindingInfo[group][binding];
                 if (info.used) {
-                    compiler.set_name(info.base_type_id, GetBindingName(group, binding));
+                    compiler->set_name(info.base_type_id, GetBindingName(group, binding));
-                    compiler.unset_decoration(info.id, spv::DecorationBinding);
+                    compiler->unset_decoration(info.id, spv::DecorationBinding);
-                    compiler.unset_decoration(info.id, spv::DecorationDescriptorSet);
+                    compiler->unset_decoration(info.id, spv::DecorationDescriptorSet);
                 }
             }
         }
 
-        mGlslSource = compiler.compile();
+        if (GetDevice()->IsToggleEnabled(Toggle::UseSpvc)) {
+            shaderc_spvc::CompilationResult result;
+            shaderc_spvc_status status = mSpvcContext.CompileShader(&result);
+            if (status != shaderc_spvc_status_success)
+                return DAWN_VALIDATION_ERROR("Unable to compile shader using spvc");
+            mGlslSource = result.GetStringOutput();
+        } else {
+            mGlslSource = compiler->compile();
+        }
         return {};
     }
 

src/dawn_native/vulkan/ShaderModuleVk.cpp

@@ -39,8 +39,21 @@ namespace dawn_native { namespace vulkan {
     MaybeError ShaderModule::Initialize(const ShaderModuleDescriptor* descriptor) {
         // Use SPIRV-Cross to extract info from the SPIRV even if Vulkan consumes SPIRV. We want to
         // have a translation step eventually anyway.
+        if (GetDevice()->IsToggleEnabled(Toggle::UseSpvc)) {
+            shaderc_spvc::CompileOptions options;
+            shaderc_spvc_status status =
+                mSpvcContext.InitializeForGlsl(descriptor->code, descriptor->codeSize, options);
+            if (status != shaderc_spvc_status_success) {
+                return DAWN_VALIDATION_ERROR("Unable to initialize instance of spvc");
+            }
+
+            spirv_cross::Compiler* compiler =
+                reinterpret_cast<spirv_cross::Compiler*>(mSpvcContext.GetCompiler());
+            ExtractSpirvInfo(*compiler);
+        } else {
             spirv_cross::Compiler compiler(descriptor->code, descriptor->codeSize);
             ExtractSpirvInfo(compiler);
+        }
 
         VkShaderModuleCreateInfo createInfo;
         createInfo.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO;

src/fuzzers/DawnSPIRVCrossGLSLFastFuzzer.cpp

@@ -35,7 +35,7 @@ namespace {
             options.SetGLSLLanguageVersion(440);
             options.SetFixupClipspace(true);
             if (context.InitializeForGlsl(input.data(), input.size(), options) ==
-                shaderc_compilation_status_success) {
+                shaderc_spvc_status_success) {
                 context.CompileShader(&result);
             }
         });

src/fuzzers/DawnSPIRVCrossHLSLFastFuzzer.cpp

@@ -41,7 +41,7 @@ namespace {
             options.SetHLSLPointCoordCompat(true);
             options.SetHLSLPointSizeCompat(true);
             if (context.InitializeForHlsl(input.data(), input.size(), options) ==
-                shaderc_compilation_status_success) {
+                shaderc_spvc_status_success) {
                 context.CompileShader(&result);
             }
         });

src/fuzzers/DawnSPIRVCrossMSLFastFuzzer.cpp

@@ -34,7 +34,7 @@ namespace {
 
             // Using the options that are used by Dawn, they appear in ShaderModuleMTL.mm
             if (context.InitializeForMsl(input.data(), input.size(), options) ==
-                shaderc_compilation_status_success) {
+                shaderc_spvc_status_success) {
                 context.CompileShader(&result);
             }
         });