Add toggle to control if validity is expected in fuzzer

This toggle controls if the fuzzer will throw a fatal error in the
case that the shader becomes invalid.

Currently the fuzzers do no guarantee that the options that are
provided are correct/valid, so there are many uninteresting cases that
become invalid due to the limited nature of the fuzzers, not due to
bugs in the code. The default off state of this toggle will suppress
this noise.

Once https://bugs.chromium.org/p/tint/issues/detail?id=1356 is
implemented this toggle can be default on.

BUG=tint:1357,chromium:1294533

Change-Id: I7170e5a30691105c97e20d8337aadf81ac2bc3bc
Reviewed-on: https://dawn-review.googlesource.com/c/tint/+/79840
Reviewed-by: Ben Clayton <bclayton@google.com>
Auto-Submit: Ryan Harrison <rharrison@chromium.org>
Kokoro: Kokoro <noreply+kokoro@google.com>
Commit-Queue: Ryan Harrison <rharrison@chromium.org>

fuzzers/cli.cc

@@ -39,22 +39,25 @@ run it with -help=1 to check out libfuzzer parameters.
   -tint_help
                        Show this message. Note that there is also a -help=1
                        parameter that will display libfuzzer's help message.
+
+  -tint_enforce_validity=
+                       If `true`, the fuzzer will enforce that Tint does not
+                       generate invalid shaders. Currently `false` by default
+                       since options provided by the fuzzer are not guaranteed
+                       to be correct.
+                       See https://bugs.chromium.org/p/tint/issues/detail?id=1356
 )";
 
-bool HasPrefix(const char* str, const char* prefix) {
+[[noreturn]] void InvalidParam(const std::string& param) {
-  return strncmp(str, prefix, strlen(prefix)) == 0;
-}
-
-[[noreturn]] void InvalidParam(const char* param) {
   std::cout << "Invalid value for " << param << std::endl;
   std::cout << kHelpMessage << std::endl;
   exit(1);
 }
 
-bool ParseBool(const char* value, bool* out) {
+bool ParseBool(const std::string& value, bool* out) {
-  if (!strcmp(value, "true")) {
+  if (value.compare("true") == 0) {
     *out = true;
-  } else if (!strcmp(value, "false")) {
+  } else if (value.compare("false") == 0) {
     *out = false;
   } else {
     return false;
@@ -69,16 +72,22 @@ CliParams ParseCliParams(int* argc, char** argv) {
   auto help = false;
 
   for (int i = *argc - 1; i > 0; --i) {
-    auto param = argv[i];
+    std::string param(argv[i]);
     auto recognized_parameter = true;
 
-    if (HasPrefix(param, "-tint_dump_input=")) {
+    if (std::string::npos != param.find("-tint_dump_input=")) {
-      if (!ParseBool(param + sizeof("-tint_dump_input=") - 1,
+      if (!ParseBool(param.substr(std::string("-tint_dump_input=").length()),
                      &cli_params.dump_input)) {
         InvalidParam(param);
       }
-    } else if (!strcmp(param, "-tint_help")) {
+    } else if (std::string::npos != param.find("-tint_help")) {
       help = true;
+    } else if (std::string::npos != param.find("-tint_enforce_validity=")) {
+      if (!ParseBool(
+              param.substr(std::string("-tint_enforce_validity=").length()),
+              &cli_params.enforce_validity)) {
+        InvalidParam(param);
+      }
     } else {
       recognized_parameter = false;
     }

fuzzers/cli.h

@@ -24,6 +24,7 @@ namespace fuzzers {
 /// help message
 struct CliParams {
   bool dump_input = false;
+  bool enforce_validity = false;
 };
 
 /// @brief Parses CLI parameters.

fuzzers/tint_all_transforms_fuzzer.cc

@@ -28,6 +28,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
     fuzzers::CommonFuzzer fuzzer(InputFormat::kWGSL, OutputFormat::kSpv);
     fuzzer.SetTransformManager(tb.manager(), tb.data_map());
     fuzzer.SetDumpInput(GetCliParams().dump_input);
+    fuzzer.SetEnforceValidity(GetCliParams().enforce_validity);
 
     fuzzer.Run(data, size);
   }
@@ -40,6 +41,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
     fuzzers::CommonFuzzer fuzzer(InputFormat::kWGSL, OutputFormat::kHLSL);
     fuzzer.SetTransformManager(tb.manager(), tb.data_map());
     fuzzer.SetDumpInput(GetCliParams().dump_input);
+    fuzzer.SetEnforceValidity(GetCliParams().enforce_validity);
 
     fuzzer.Run(data, size);
   }
@@ -53,6 +55,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
     fuzzers::CommonFuzzer fuzzer(InputFormat::kWGSL, OutputFormat::kMSL);
     fuzzer.SetTransformManager(tb.manager(), tb.data_map());
     fuzzer.SetDumpInput(GetCliParams().dump_input);
+    fuzzer.SetEnforceValidity(GetCliParams().enforce_validity);
 
     fuzzer.Run(data, size);
   }
@@ -65,6 +68,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
     fuzzers::CommonFuzzer fuzzer(InputFormat::kWGSL, OutputFormat::kSpv);
     fuzzer.SetTransformManager(tb.manager(), tb.data_map());
     fuzzer.SetDumpInput(GetCliParams().dump_input);
+    fuzzer.SetEnforceValidity(GetCliParams().enforce_validity);
 
     fuzzer.Run(data, size);
   }

fuzzers/tint_binding_remapper_fuzzer.cc

@@ -26,6 +26,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
   fuzzers::CommonFuzzer fuzzer(InputFormat::kWGSL, OutputFormat::kWGSL);
   fuzzer.SetTransformManager(tb.manager(), tb.data_map());
   fuzzer.SetDumpInput(GetCliParams().dump_input);
+  fuzzer.SetEnforceValidity(GetCliParams().enforce_validity);
 
   return fuzzer.Run(data, size);
 }

fuzzers/tint_common_fuzzer.cc

@@ -57,15 +57,27 @@ namespace {
   FATAL_ERROR(diagnostics, "");
 }
 
-// Wrapping this in a macro so it can be a one-liner in the code, but not
+// Wrapping in a macro, so it can be a one-liner in the code, but not
-// introducing another level in the stack trace. This will help with de-duping
+// introduce another level in the stack trace. This will help with de-duping
 // ClusterFuzz issues.
-#define CHECK_INSPECTOR(inspector)                           \
+#define CHECK_INSPECTOR(program, inspector)                    \
   do {                                                         \
-    if (inspector.has_error()) {                             \
+    if ((inspector).has_error()) {                             \
-      FATAL_ERROR(program->Diagnostics(),                    \
+      if (!enforce_validity) {                                 \
-                  "Inspector failed: " + inspector.error()); \
+        return;                                                \
       }                                                        \
+      FATAL_ERROR((program)->Diagnostics(),                    \
+                  "Inspector failed: " + (inspector).error()); \
+    }                                                          \
+  } while (false)
+
+// Wrapping in a macro to make code more readable and help with issue de-duping.
+#define VALIDITY_ERROR(diags, msg_string) \
+  do {                                    \
+    if (!enforce_validity) {              \
+      return 0;                           \
+    }                                     \
+    FATAL_ERROR(diags, msg_string);       \
   } while (false)
 
 bool SPIRVToolsValidationCheck(const tint::Program& program,
@@ -192,6 +204,7 @@ int CommonFuzzer::Run(const uint8_t* data, size_t size) {
 #endif  // TINT_BUILD_SPV_READER
 
   RunInspector(&program);
+  diagnostics_ = program.Diagnostics();
 
   if (transform_manager_) {
     auto out = transform_manager_->Run(&program, *transform_inputs_);
@@ -201,9 +214,9 @@ int CommonFuzzer::Run(const uint8_t* data, size_t size) {
       for (const auto& diag : out.program.Diagnostics()) {
         if (diag.severity > diag::Severity::Error ||
             diag.system != diag::System::Transform) {
-          FATAL_ERROR(out.program.Diagnostics(),
+          VALIDITY_ERROR(program.Diagnostics(),
-                      "Fuzzing detected valid input program being transformed "
+                         "Fuzzing detected valid input program being "
-                      "into an invalid output program");
+                         "transformed into an invalid output program");
         }
       }
     }
@@ -218,7 +231,8 @@ int CommonFuzzer::Run(const uint8_t* data, size_t size) {
       auto result = writer::wgsl::Generate(&program, options_wgsl_);
       generated_wgsl_ = std::move(result.wgsl);
       if (!result.success) {
-        FATAL_ERROR(program.Diagnostics(),
+        VALIDITY_ERROR(
+            program.Diagnostics(),
             "WGSL writer errored on validated input:\n" + result.error);
       }
 #endif  // TINT_BUILD_WGSL_WRITER
@@ -229,12 +243,13 @@ int CommonFuzzer::Run(const uint8_t* data, size_t size) {
       auto result = writer::spirv::Generate(&program, options_spirv_);
       generated_spirv_ = std::move(result.spirv);
       if (!result.success) {
-        FATAL_ERROR(
+        VALIDITY_ERROR(
             program.Diagnostics(),
             "SPIR-V writer errored on validated input:\n" + result.error);
       }
+
       if (!SPIRVToolsValidationCheck(program, generated_spirv_)) {
-        FATAL_ERROR(program.Diagnostics(),
+        VALIDITY_ERROR(program.Diagnostics(),
                        "Fuzzing detected invalid spirv being emitted by Tint");
       }
 
@@ -246,7 +261,8 @@ int CommonFuzzer::Run(const uint8_t* data, size_t size) {
       auto result = writer::hlsl::Generate(&program, options_hlsl_);
       generated_hlsl_ = std::move(result.hlsl);
       if (!result.success) {
-        FATAL_ERROR(program.Diagnostics(),
+        VALIDITY_ERROR(
+            program.Diagnostics(),
             "HLSL writer errored on validated input:\n" + result.error);
       }
 #endif  // TINT_BUILD_HLSL_WRITER
@@ -257,7 +273,8 @@ int CommonFuzzer::Run(const uint8_t* data, size_t size) {
       auto result = writer::msl::Generate(&program, options_msl_);
       generated_msl_ = std::move(result.msl);
       if (!result.success) {
-        FATAL_ERROR(program.Diagnostics(),
+        VALIDITY_ERROR(
+            program.Diagnostics(),
             "MSL writer errored on validated input:\n" + result.error);
       }
 #endif  // TINT_BUILD_MSL_WRITER
@@ -270,64 +287,65 @@ int CommonFuzzer::Run(const uint8_t* data, size_t size) {
 
 void CommonFuzzer::RunInspector(Program* program) {
   inspector::Inspector inspector(program);
+  diagnostics_ = program->Diagnostics();
 
   auto entry_points = inspector.GetEntryPoints();
-  CHECK_INSPECTOR(inspector);
+  CHECK_INSPECTOR(program, inspector);
 
   auto constant_ids = inspector.GetConstantIDs();
-  CHECK_INSPECTOR(inspector);
+  CHECK_INSPECTOR(program, inspector);
 
   auto constant_name_to_id = inspector.GetConstantNameToIdMap();
-  CHECK_INSPECTOR(inspector);
+  CHECK_INSPECTOR(program, inspector);
 
   for (auto& ep : entry_points) {
     inspector.GetRemappedNameForEntryPoint(ep.name);
-    CHECK_INSPECTOR(inspector);
+    CHECK_INSPECTOR(program, inspector);
 
     inspector.GetStorageSize(ep.name);
-    CHECK_INSPECTOR(inspector);
+    CHECK_INSPECTOR(program, inspector);
 
     inspector.GetResourceBindings(ep.name);
-    CHECK_INSPECTOR(inspector);
+    CHECK_INSPECTOR(program, inspector);
 
     inspector.GetUniformBufferResourceBindings(ep.name);
-    CHECK_INSPECTOR(inspector);
+    CHECK_INSPECTOR(program, inspector);
 
     inspector.GetStorageBufferResourceBindings(ep.name);
-    CHECK_INSPECTOR(inspector);
+    CHECK_INSPECTOR(program, inspector);
 
     inspector.GetReadOnlyStorageBufferResourceBindings(ep.name);
-    CHECK_INSPECTOR(inspector);
+    CHECK_INSPECTOR(program, inspector);
 
     inspector.GetSamplerResourceBindings(ep.name);
-    CHECK_INSPECTOR(inspector);
+    CHECK_INSPECTOR(program, inspector);
 
     inspector.GetComparisonSamplerResourceBindings(ep.name);
-    CHECK_INSPECTOR(inspector);
+    CHECK_INSPECTOR(program, inspector);
 
     inspector.GetSampledTextureResourceBindings(ep.name);
-    CHECK_INSPECTOR(inspector);
+    CHECK_INSPECTOR(program, inspector);
 
     inspector.GetMultisampledTextureResourceBindings(ep.name);
-    CHECK_INSPECTOR(inspector);
+    CHECK_INSPECTOR(program, inspector);
 
     inspector.GetWriteOnlyStorageTextureResourceBindings(ep.name);
-    CHECK_INSPECTOR(inspector);
+    CHECK_INSPECTOR(program, inspector);
 
     inspector.GetDepthTextureResourceBindings(ep.name);
-    CHECK_INSPECTOR(inspector);
+    CHECK_INSPECTOR(program, inspector);
 
     inspector.GetDepthMultisampledTextureResourceBindings(ep.name);
-    CHECK_INSPECTOR(inspector);
+    CHECK_INSPECTOR(program, inspector);
 
     inspector.GetExternalTextureResourceBindings(ep.name);
-    CHECK_INSPECTOR(inspector);
+    CHECK_INSPECTOR(program, inspector);
 
     inspector.GetSamplerTextureUses(ep.name);
-    CHECK_INSPECTOR(inspector);
+    CHECK_INSPECTOR(program, inspector);
 
     inspector.GetWorkgroupStorageSize(ep.name);
-    CHECK_INSPECTOR(inspector);
+    CHECK_INSPECTOR(program, inspector);
   }
 }
 

fuzzers/tint_common_fuzzer.h

@@ -51,6 +51,8 @@ class CommonFuzzer {
 
   void SetDumpInput(bool enabled) { dump_input_ = enabled; }
 
+  void SetEnforceValidity(bool enabled) { enforce_validity = enabled; }
+
   int Run(const uint8_t* data, size_t size);
 
   const tint::diag::List& Diagnostics() const { return diagnostics_; }
@@ -90,6 +92,7 @@ class CommonFuzzer {
   transform::DataMap* transform_inputs_ = nullptr;
   bool dump_input_ = false;
   tint::diag::List diagnostics_;
+  bool enforce_validity = false;
 
   std::vector<uint32_t> generated_spirv_;
   std::string generated_wgsl_;

fuzzers/tint_first_index_offset_fuzzer.cc

@@ -26,6 +26,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
   fuzzers::CommonFuzzer fuzzer(InputFormat::kWGSL, OutputFormat::kWGSL);
   fuzzer.SetTransformManager(tb.manager(), tb.data_map());
   fuzzer.SetDumpInput(GetCliParams().dump_input);
+  fuzzer.SetEnforceValidity(GetCliParams().enforce_validity);
 
   return fuzzer.Run(data, size);
 }

fuzzers/tint_renamer_fuzzer.cc

@@ -26,6 +26,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
   fuzzers::CommonFuzzer fuzzer(InputFormat::kWGSL, OutputFormat::kWGSL);
   fuzzer.SetTransformManager(tb.manager(), tb.data_map());
   fuzzer.SetDumpInput(GetCliParams().dump_input);
+  fuzzer.SetEnforceValidity(GetCliParams().enforce_validity);
 
   return fuzzer.Run(data, size);
 }

fuzzers/tint_robustness_fuzzer.cc

@@ -26,6 +26,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
   tint::fuzzers::CommonFuzzer fuzzer(InputFormat::kWGSL, OutputFormat::kWGSL);
   fuzzer.SetTransformManager(tb.manager(), tb.data_map());
   fuzzer.SetDumpInput(GetCliParams().dump_input);
+  fuzzer.SetEnforceValidity(GetCliParams().enforce_validity);
 
   return fuzzer.Run(data, size);
 }

fuzzers/tint_single_entry_point_fuzzer.cc

@@ -26,6 +26,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
   fuzzers::CommonFuzzer fuzzer(InputFormat::kWGSL, OutputFormat::kWGSL);
   fuzzer.SetTransformManager(tb.manager(), tb.data_map());
   fuzzer.SetDumpInput(GetCliParams().dump_input);
+  fuzzer.SetEnforceValidity(GetCliParams().enforce_validity);
 
   return fuzzer.Run(data, size);
 }

fuzzers/tint_spv_reader_hlsl_writer_fuzzer.cc

@@ -24,6 +24,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
   tint::fuzzers::ReaderWriterFuzzer fuzzer(InputFormat::kSpv,
                                            OutputFormat::kHLSL);
   fuzzer.SetDumpInput(GetCliParams().dump_input);
+  fuzzer.SetEnforceValidity(GetCliParams().enforce_validity);
+
   return fuzzer.Run(data, size);
 }
 

fuzzers/tint_spv_reader_msl_writer_fuzzer.cc

@@ -28,6 +28,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
                                            OutputFormat::kMSL);
   fuzzer.SetOptionsMsl(options);
   fuzzer.SetDumpInput(GetCliParams().dump_input);
+  fuzzer.SetEnforceValidity(GetCliParams().enforce_validity);
+
   return fuzzer.Run(data, size);
 }
 

fuzzers/tint_spv_reader_spv_writer_fuzzer.cc

@@ -28,6 +28,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
                                            OutputFormat::kSpv);
   fuzzer.SetOptionsSpirv(options);
   fuzzer.SetDumpInput(GetCliParams().dump_input);
+  fuzzer.SetEnforceValidity(GetCliParams().enforce_validity);
+
   return fuzzer.Run(data, size);
 }
 

fuzzers/tint_spv_reader_wgsl_writer_fuzzer.cc

@@ -24,6 +24,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
   tint::fuzzers::ReaderWriterFuzzer fuzzer(InputFormat::kSpv,
                                            OutputFormat::kWGSL);
   fuzzer.SetDumpInput(GetCliParams().dump_input);
+  fuzzer.SetEnforceValidity(GetCliParams().enforce_validity);
+
   return fuzzer.Run(data, size);
 }
 

fuzzers/tint_vertex_pulling_fuzzer.cc

@@ -26,6 +26,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
   tint::fuzzers::CommonFuzzer fuzzer(InputFormat::kWGSL, OutputFormat::kWGSL);
   fuzzer.SetTransformManager(tb.manager(), tb.data_map());
   fuzzer.SetDumpInput(GetCliParams().dump_input);
+  fuzzer.SetEnforceValidity(GetCliParams().enforce_validity);
 
   return fuzzer.Run(data, size);
 }

fuzzers/tint_wgsl_reader_hlsl_writer_fuzzer.cc

@@ -24,6 +24,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
   tint::fuzzers::ReaderWriterFuzzer fuzzer(InputFormat::kWGSL,
                                            OutputFormat::kHLSL);
   fuzzer.SetDumpInput(GetCliParams().dump_input);
+  fuzzer.SetEnforceValidity(GetCliParams().enforce_validity);
+
   return fuzzer.Run(data, size);
 }
 

fuzzers/tint_wgsl_reader_msl_writer_fuzzer.cc

@@ -28,6 +28,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
                                            OutputFormat::kMSL);
   fuzzer.SetOptionsMsl(options);
   fuzzer.SetDumpInput(GetCliParams().dump_input);
+  fuzzer.SetEnforceValidity(GetCliParams().enforce_validity);
+
   return fuzzer.Run(data, size);
 }
 

fuzzers/tint_wgsl_reader_spv_writer_fuzzer.cc

@@ -28,6 +28,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
                                            OutputFormat::kSpv);
   fuzzer.SetOptionsSpirv(options);
   fuzzer.SetDumpInput(GetCliParams().dump_input);
+  fuzzer.SetEnforceValidity(GetCliParams().enforce_validity);
+
   return fuzzer.Run(data, size);
 }
 

fuzzers/tint_wgsl_reader_wgsl_writer_fuzzer.cc

@@ -24,6 +24,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
   tint::fuzzers::ReaderWriterFuzzer fuzzer(InputFormat::kWGSL,
                                            OutputFormat::kWGSL);
   fuzzer.SetDumpInput(GetCliParams().dump_input);
+  fuzzer.SetEnforceValidity(GetCliParams().enforce_validity);
+
   return fuzzer.Run(data, size);
 }