main: Replace --dawn-validation with --validate

Performs output validation with spirv-val for SPIR-V (as before), HLSL
validation with DXC, and MSL validation with the Metal Shader Compiler.

Disable HLSL tests that fail to validate

Bug: tint:812
Change-Id: If78c351b4e23c7fb50d333eacf9ee7cc81d18564
Reviewed-on: https://dawn-review.googlesource.com/c/tint/+/51280
Reviewed-by: Antonio Maiorano <amaiorano@google.com>
Reviewed-by: Ben Clayton <bclayton@google.com>
Kokoro: Kokoro <noreply+kokoro@google.com>
Commit-Queue: Antonio Maiorano <amaiorano@google.com>

src/BUILD.gn

@@ -208,6 +208,43 @@ tint_language_header("cldebuginfo100") {
   grammar_file = "${tint_spirv_headers_dir}/include/spirv/unified1/extinst.opencl.debuginfo.100.grammar.json"
 }
 
+###############################################################################
+# Helper library for IO operations
+# Only to be used by tests and sample executable
+###############################################################################
+source_set("tint_utils_io") {
+  sources = [
+    "utils/io/command.h",
+    "utils/io/tmpfile.h",
+  ]
+
+  if (is_linux || is_mac) {
+    sources += [ "utils/io/command_posix.cc" ]
+    sources += [ "utils/io/tmpfile_posix.cc" ]
+  } else if (is_win) {
+    sources += [ "utils/io/command_windows.cc" ]
+    sources += [ "utils/io/tmpfile_windows.cc" ]
+  } else {
+    sources += [ "utils/io/command_other.cc" ]
+    sources += [ "utils/io/tmpfile_other.cc" ]
+  }
+
+  public_deps = [ ":libtint_core_all_src" ]
+}
+
+###############################################################################
+# Helper library for validating generated shaders
+# As this depends on tint_utils_io, this is only to be used by tests and sample
+# executable
+###############################################################################
+source_set("tint_val") {
+  sources = [
+    "val/val.cc",
+    "val/val.h",
+  ]
+  public_deps = [ ":tint_utils_io" ]
+}
+
 ###############################################################################
 # Library - Tint core and optional modules of libtint
 ###############################################################################
@@ -464,9 +501,9 @@ libtint_source_set("libtint_core_all_src") {
     "sem/i32_type.cc",
     "sem/i32_type.h",
     "sem/info.h",
+    "sem/intrinsic.h",
     "sem/intrinsic_type.cc",
     "sem/intrinsic_type.h",
-    "sem/intrinsic.h",
     "sem/matrix_type.cc",
     "sem/matrix_type.h",
     "sem/multisampled_texture_type.cc",

src/CMakeLists.txt

@@ -449,6 +449,23 @@ if(${TINT_BUILD_HLSL_WRITER})
   )
 endif()
 
+## Tint IO utilities. Used by tint_val.
+add_library(tint_utils_io
+  utils/io/command_${TINT_OS_CC_SUFFIX}.cc
+  utils/io/command.h
+  utils/io/tmpfile_${TINT_OS_CC_SUFFIX}.cc
+  utils/io/tmpfile.h
+)
+tint_default_compile_options(tint_utils_io)
+
+## Tint validation utilities. Used by tests and the tint executable.
+add_library(tint_val
+  val/val.cc
+  val/val.h
+)
+tint_default_compile_options(tint_val)
+target_link_libraries(tint_val tint_utils_io)
+
 ## Tint library
 add_library(libtint ${TINT_LIB_SRCS})
 tint_default_compile_options(libtint)
@@ -603,16 +620,12 @@ if(${TINT_BUILD_TESTS})
     sem/type_manager_test.cc
     sem/u32_type_test.cc
     sem/vector_type_test.cc
-    utils/command_${TINT_OS_CC_SUFFIX}.cc
-    utils/command_test.cc
-    utils/command.h
     utils/get_or_create_test.cc
     utils/hash_test.cc
+    utils/io/command_test.cc
+    utils/io/tmpfile_test.cc
     utils/math_test.cc
     utils/scoped_assignment_test.cc
-    utils/tmpfile_${TINT_OS_CC_SUFFIX}.cc
-    utils/tmpfile_test.cc
-    utils/tmpfile.h
     utils/unique_vector_test.cc
     writer/append_vector_test.cc
     writer/float_to_string_test.cc
@@ -913,7 +926,7 @@ if(${TINT_BUILD_TESTS})
   ## Test executable
   target_include_directories(
       tint_unittests PRIVATE ${gmock_SOURCE_DIR}/include)
-  target_link_libraries(tint_unittests libtint gmock)
+  target_link_libraries(tint_unittests libtint gmock tint_val)
   tint_default_compile_options(tint_unittests)
 
   if(${TINT_BUILD_SPV_READER} OR ${TINT_BUILD_SPV_WRITER})

src/test_main.cc

@@ -14,7 +14,7 @@
 
 #include "gmock/gmock.h"
 #include "src/reader/spirv/parser_impl_test_helper.h"
-#include "src/utils/command.h"
+#include "src/utils/io/command.h"
 #include "src/writer/hlsl/test_helper.h"
 #include "src/writer/msl/test_helper.h"
 

src/utils/io/command.h

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef SRC_UTILS_COMMAND_H_
-#define SRC_UTILS_COMMAND_H_
+#ifndef SRC_UTILS_IO_COMMAND_H_
+#define SRC_UTILS_IO_COMMAND_H_
 
 #include <string>
 #include <utility>
@@ -81,4 +81,4 @@ class Command {
 }  // namespace utils
 }  // namespace tint
 
-#endif  //  SRC_UTILS_COMMAND_H_
+#endif  //  SRC_UTILS_IO_COMMAND_H_

src/utils/io/command_other.cc

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "src/utils/command.h"
+#include "src/utils/io/command.h"
 
 namespace tint {
 namespace utils {

src/utils/io/command_posix.cc

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "src/utils/command.h"
+#include "src/utils/io/command.h"
 
 #include <sys/poll.h>
 #include <sys/stat.h>

src/utils/io/command_test.cc

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "src/utils/command.h"
+#include "src/utils/io/command.h"
 
 #include "gtest/gtest.h"
 

src/utils/io/command_windows.cc

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "src/utils/command.h"
+#include "src/utils/io/command.h"
 
 #define WIN32_LEAN_AND_MEAN 1
 #include <Windows.h>

src/utils/io/tmpfile.h

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef SRC_UTILS_TMPFILE_H_
-#define SRC_UTILS_TMPFILE_H_
+#ifndef SRC_UTILS_IO_TMPFILE_H_
+#define SRC_UTILS_IO_TMPFILE_H_
 
 #include <sstream>
 #include <string>
@@ -73,4 +73,4 @@ class TmpFile {
 }  // namespace utils
 }  // namespace tint
 
-#endif  //  SRC_UTILS_TMPFILE_H_
+#endif  //  SRC_UTILS_IO_TMPFILE_H_

src/utils/io/tmpfile_other.cc

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "src/utils/tmpfile.h"
+#include "src/utils/io/tmpfile.h"
 
 namespace tint {
 namespace utils {

src/utils/io/tmpfile_posix.cc

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "src/utils/tmpfile.h"
+#include "src/utils/io/tmpfile.h"
 
 #include <unistd.h>
 #include <limits>

src/utils/io/tmpfile_test.cc

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "src/utils/tmpfile.h"
+#include "src/utils/io/tmpfile.h"
 
 #include <fstream>
 

src/utils/io/tmpfile_windows.cc

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "src/utils/tmpfile.h"
+#include "src/utils/io/tmpfile.h"
 
 #include <stdio.h>
 #include <cstdio>

src/val/val.cc

@@ -0,0 +1,133 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "src/val/val.h"
+
+#include "src/ast/module.h"
+#include "src/program.h"
+#include "src/utils/io/command.h"
+#include "src/utils/io/tmpfile.h"
+
+namespace tint {
+namespace val {
+
+Result Hlsl(const std::string& dxc_path,
+            const std::string& source,
+            Program* program) {
+  Result result;
+
+  auto dxc = utils::Command(dxc_path);
+  if (!dxc.Found()) {
+    result.output = "DXC not found at '" + std::string(dxc_path) + "'";
+    result.failed = true;
+    return result;
+  }
+
+  result.source = source;
+
+  utils::TmpFile file;
+  file << source;
+
+  bool found_an_entrypoint = false;
+  for (auto* func : program->AST().Functions()) {
+    if (func->IsEntryPoint()) {
+      found_an_entrypoint = true;
+
+      const char* profile = "";
+
+      switch (func->pipeline_stage()) {
+        case ast::PipelineStage::kNone:
+          result.output = "Invalid PipelineStage";
+          result.failed = true;
+          return result;
+        case ast::PipelineStage::kVertex:
+          profile = "-T vs_6_0";
+          break;
+        case ast::PipelineStage::kFragment:
+          profile = "-T ps_6_0";
+          break;
+        case ast::PipelineStage::kCompute:
+          profile = "-T cs_6_0";
+          break;
+      }
+
+      auto name = program->Symbols().NameFor(func->symbol());
+      auto res = dxc(profile, "-E " + name, file.Path());
+      if (!res.out.empty()) {
+        if (!result.output.empty()) {
+          result.output += "\n";
+        }
+        result.output += res.out;
+      }
+      if (!res.err.empty()) {
+        if (!result.output.empty()) {
+          result.output += "\n";
+        }
+        result.output += res.err;
+      }
+      result.failed = (res.error_code != 0);
+    }
+  }
+
+  if (!found_an_entrypoint) {
+    result.output = "No entrypoint found";
+    result.failed = true;
+    return result;
+  }
+
+  return result;
+}
+
+Result Msl(const std::string& xcrun_path, const std::string& source) {
+  Result result;
+
+  auto xcrun = utils::Command(xcrun_path);
+  if (!xcrun.Found()) {
+    result.output = "xcrun not found at '" + std::string(xcrun_path) + "'";
+    result.failed = true;
+    return result;
+  }
+
+  result.source = source;
+
+  utils::TmpFile file(".metal");
+  file << result.source;
+
+#ifdef _WIN32
+  // On Windows, we should actually be running metal.exe from the Metal
+  // Developer Tools for Windows
+  auto res = xcrun("-x", "metal", "-c", "-o", "NUL", file.Path());
+#else
+  auto res =
+      xcrun("-sdk", "macosx", "metal", "-o", "/dev/null", "-c", file.Path());
+#endif
+  if (!res.out.empty()) {
+    if (!result.output.empty()) {
+      result.output += "\n";
+    }
+    result.output += res.out;
+  }
+  if (!res.err.empty()) {
+    if (!result.output.empty()) {
+      result.output += "\n";
+    }
+    result.output += res.err;
+  }
+  result.failed = (res.error_code != 0);
+
+  return result;
+}
+
+}  // namespace val
+}  // namespace tint

src/val/val.h

@@ -0,0 +1,58 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef SRC_VAL_VAL_H_
+#define SRC_VAL_VAL_H_
+
+#include <string>
+
+// Forward declarations
+namespace tint {
+class Program;
+}  // namespace tint
+
+namespace tint {
+namespace val {
+
+/// The return structure of Validate()
+struct Result {
+  /// True if validation passed
+  bool failed = false;
+  /// Output of DXC.
+  std::string output;
+  /// The generated source that was compiled
+  std::string source;
+};
+
+/// Hlsl attempts to compile the shader with DXC, verifying that the shader
+/// compiles successfully.
+/// @param dxc_path path to DXC
+/// @param source the generated HLSL source
+/// @param program the HLSL program
+/// @return the result of the compile
+Result Hlsl(const std::string& dxc_path,
+            const std::string& source,
+            Program* program);
+
+/// Msl attempts to compile the shader with the Metal Shader Compiler,
+/// verifying that the shader compiles successfully.
+/// @param xcrun_path path to xcrun
+/// @param source the generated MSL source
+/// @return the result of the compile
+Result Msl(const std::string& xcrun_path, const std::string& source);
+
+}  // namespace val
+}  // namespace tint
+
+#endif  // SRC_VAL_VAL_H_

src/writer/hlsl/test_helper.cc

@@ -14,9 +14,6 @@
 
 #include "src/writer/hlsl/test_helper.h"
 
-#include "src/utils/command.h"
-#include "src/utils/tmpfile.h"
-
 namespace tint {
 namespace writer {
 namespace hlsl {
@@ -31,81 +28,16 @@ void EnableHLSLValidation(const char* dxc) {
   dxc_path = dxc;
 }
 
-CompileResult Compile(Program* program, GeneratorImpl* generator) {
-  CompileResult result;
-
+val::Result Validate(Program* program, GeneratorImpl* generator) {
   if (!dxc_path) {
-    result.status = CompileResult::Status::kVerificationNotEnabled;
-    return result;
-  }
-
-  auto dxc = utils::Command(dxc_path);
-  if (!dxc.Found()) {
-    result.output = "DXC not found at '" + std::string(dxc_path) + "'";
-    result.status = CompileResult::Status::kFailed;
-    return result;
+    return val::Result{};
   }
 
   std::ostringstream hlsl;
   if (!generator->Generate(hlsl)) {
-    result.output = generator->error();
-    result.status = CompileResult::Status::kFailed;
-    return result;
+    return {true, generator->error(), ""};
   }
-  result.hlsl = hlsl.str();
-
-  utils::TmpFile file;
-  file << result.hlsl;
-
-  bool found_an_entrypoint = false;
-  for (auto* func : program->AST().Functions()) {
-    if (func->IsEntryPoint()) {
-      found_an_entrypoint = true;
-
-      const char* profile = "";
-
-      switch (func->pipeline_stage()) {
-        case ast::PipelineStage::kNone:
-          result.output = "Invalid PipelineStage";
-          result.status = CompileResult::Status::kFailed;
-          return result;
-        case ast::PipelineStage::kVertex:
-          profile = "-T vs_6_0";
-          break;
-        case ast::PipelineStage::kFragment:
-          profile = "-T ps_6_0";
-          break;
-        case ast::PipelineStage::kCompute:
-          profile = "-T cs_6_0";
-          break;
-      }
-
-      auto name = program->Symbols().NameFor(func->symbol());
-      auto res = dxc(profile, "-E " + name, file.Path());
-      if (!res.out.empty()) {
-        if (!result.output.empty()) {
-          result.output += "\n";
-        }
-        result.output += res.out;
-      }
-      if (!res.err.empty()) {
-        if (!result.output.empty()) {
-          result.output += "\n";
-        }
-        result.output += res.err;
-      }
-      result.status = (res.error_code == 0) ? CompileResult::Status::kSuccess
-                                            : CompileResult::Status::kFailed;
-    }
-  }
-
-  if (!found_an_entrypoint) {
-    result.output = "No entrypoint found";
-    result.status = CompileResult::Status::kFailed;
-    return result;
-  }
-
-  return result;
+  return val::Hlsl(dxc_path, hlsl.str(), program);
 }
 
 }  // namespace hlsl

src/writer/hlsl/test_helper.h

@@ -23,6 +23,7 @@
 #include "src/transform/hlsl.h"
 #include "src/transform/manager.h"
 #include "src/transform/renamer.h"
+#include "src/val/val.h"
 #include "src/writer/hlsl/generator_impl.h"
 
 namespace tint {
@@ -34,23 +35,11 @@ namespace hlsl {
 /// @param dxc_path the path to the DXC executable
 void EnableHLSLValidation(const char* dxc_path);
 
-/// The return structure of Compile()
-struct CompileResult {
-  /// Status is an enumerator of status codes from Compile()
-  enum class Status { kSuccess, kFailed, kVerificationNotEnabled };
-  /// The resulting status of the compile
-  Status status;
-  /// Output of DXC.
-  std::string output;
-  /// The HLSL source that was compiled
-  std::string hlsl;
-};
-
-/// Compile attempts to compile the shader with DXC if found on PATH.
+/// Validate attempts to compile the shader with DXC if found on PATH.
 /// @param program the HLSL program
 /// @param generator the HLSL generator
 /// @return the result of the compile
-CompileResult Compile(Program* program, GeneratorImpl* generator);
+val::Result Validate(Program* program, GeneratorImpl* generator);
 
 /// Helper class for testing
 template <typename BODY>
@@ -122,10 +111,10 @@ class TestHelperBase : public BODY, public ProgramBuilder {
   /// If DXC finds problems the test will fail.
   /// If DXC is not on `PATH` then Validate() does nothing.
   void Validate() const {
-    auto res = Compile(program.get(), gen_.get());
-    if (res.status == CompileResult::Status::kFailed) {
+    auto res = hlsl::Validate(program.get(), gen_.get());
+    if (res.failed) {
       FAIL() << "HLSL Validation failed.\n\n"
-             << res.hlsl << "\n\n"
+             << res.source << "\n\n"
              << res.output;
     }
   }

src/writer/msl/test_helper.cc

@@ -14,8 +14,8 @@
 
 #include "src/writer/msl/test_helper.h"
 
-#include "src/utils/command.h"
-#include "src/utils/tmpfile.h"
+#include "src/utils/io/command.h"
+#include "src/utils/io/tmpfile.h"
 
 namespace tint {
 namespace writer {
@@ -31,50 +31,16 @@ void EnableMSLValidation(const char* xcrun) {
   xcrun_path = xcrun;
 }
 
-CompileResult Compile(Program* program) {
-  CompileResult result;
-
+val::Result Validate(Program* program) {
   if (!xcrun_path) {
-    result.status = CompileResult::Status::kVerificationNotEnabled;
-    return result;
-  }
-
-  auto xcrun = utils::Command(xcrun_path);
-  if (!xcrun.Found()) {
-    result.output = "xcrun not found at '" + std::string(xcrun_path) + "'";
-    result.status = CompileResult::Status::kFailed;
-    return result;
+    return val::Result{};
   }
 
   auto gen = std::make_unique<GeneratorImpl>(program);
   if (!gen->Generate()) {
-    result.output = gen->error();
-    result.status = CompileResult::Status::kFailed;
-    return result;
+    return {true, gen->error(), ""};
   }
-  result.msl = gen->result();
-
-  utils::TmpFile file(".metal");
-  file << result.msl;
-
-  auto xcrun_res =
-      xcrun("-sdk", "macosx", "metal", "-o", "/dev/null", "-c", file.Path());
-  if (!xcrun_res.out.empty()) {
-    if (!result.output.empty()) {
-      result.output += "\n";
-    }
-    result.output += xcrun_res.out;
-  }
-  if (!xcrun_res.err.empty()) {
-    if (!result.output.empty()) {
-      result.output += "\n";
-    }
-    result.output += xcrun_res.err;
-  }
-  result.status = (xcrun_res.error_code == 0) ? CompileResult::Status::kSuccess
-                                              : CompileResult::Status::kFailed;
-
-  return result;
+  return val::Msl(xcrun_path, gen->result());
 }
 
 }  // namespace msl

src/writer/msl/test_helper.h

@@ -22,6 +22,7 @@
 #include "gtest/gtest.h"
 #include "src/program_builder.h"
 #include "src/transform/msl.h"
+#include "src/val/val.h"
 #include "src/writer/msl/generator_impl.h"
 
 namespace tint {
@@ -33,22 +34,10 @@ namespace msl {
 /// @param xcrun_path the path to the `xcrun` executable
 void EnableMSLValidation(const char* xcrun_path);
 
-/// The return structure of Compile()
-struct CompileResult {
-  /// Status is an enumerator of status codes from Compile()
-  enum class Status { kSuccess, kFailed, kVerificationNotEnabled };
-  /// The resulting status of the compilation
-  Status status;
-  /// Output of the Metal compiler
-  std::string output;
-  /// The MSL source that was compiled
-  std::string msl;
-};
-
-/// Compile attempts to compile the shader with xcrun if found on PATH.
+/// Validate attempts to compile the shader with DXC if found on PATH.
 /// @param program the MSL program
 /// @return the result of the compile
-CompileResult Compile(Program* program);
+val::Result Validate(Program* program);
 
 /// Helper class for testing
 template <typename BASE>
@@ -115,9 +104,11 @@ class TestHelperBase : public BASE, public ProgramBuilder {
   /// This function does nothing, if the Metal compiler path has not been
   /// configured by calling `EnableMSLValidation()`.
   void Validate() {
-    auto res = Compile(program.get());
-    if (res.status == CompileResult::Status::kFailed) {
-      FAIL() << "MSL Validation failed.\n\n" << res.msl << "\n\n" << res.output;
+    auto res = msl::Validate(program.get());
+    if (res.failed) {
+      FAIL() << "MSL Validation failed.\n\n"
+             << res.source << "\n\n"
+             << res.output;
     }
   }