diff --git a/BUILD.gn b/BUILD.gn index de8f25844e..0329c0bda0 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -941,8 +941,9 @@ source_set("dawn_white_box_tests_sources") { deps = [ ":dawn_utils", - ":libdawn_native", ":libdawn_native_sources", + ":libdawn_native_static", # Static because the tests both link against and have libdawn_native + # sources. MSVC errors when both importing and exporting symbols. ":libdawn_wire", "${dawn_root}/src/common", "${dawn_root}/src/dawn:dawncpp", diff --git a/docs/fuzzing.md b/docs/fuzzing.md index bd7cc61ae0..8c7b7baffa 100644 --- a/docs/fuzzing.md +++ b/docs/fuzzing.md @@ -4,17 +4,23 @@ The `dawn_wire_server_and_frontend_fuzzer` sets up Dawn using the Null backend, and passes inputs to the wire server. This fuzzes the `dawn_wire` deserialization, as well as Dawn's frontend validation. +## `dawn_wire_server_and_vulkan_backend_fuzzer` + +The `dawn_wire_server_and_vulkan_backend_fuzzer` is like `dawn_wire_server_and_frontend_fuzzer` but it runs using a Vulkan CPU backend such as Swiftshader. This fuzzer supports error injection by using the first bytes of the fuzzing input as a Vulkan call index for which to mock a failure. + ## Updating the Seed Corpus Using a seed corpus significantly improves the efficiency of fuzzing. Dawn's fuzzers use interesting testcases discovered in previous fuzzing runs to seed future runs. Fuzzing can be further improved by using Dawn tests as a example of API usage which allows the fuzzer to quickly discover and use new API entrypoints and usage patterns. -The script [update_fuzzer_seed_corpus.sh](../scripts/update_fuzzer_seed_corpus.sh) can be used to capture a trace while running Dawn tests, and upload it to the existing fuzzzer seed corpus. +The script [update_fuzzer_seed_corpus.sh](../scripts/update_fuzzer_seed_corpus.sh) can be used to capture a trace while running Dawn tests, and upload it to the existing fuzzer seed corpus. It does the following steps: +1. Builds the provided test and fuzzer targets. +2. Runs the provided test target with `--use-wire --wire-trace-dir=tmp_dir1 [additional_test_args]` to dump traces of the tests. +3. Generates one variant of each trace for every possible error index, by running the fuzzer target with `--injected-error-testcase-dir=tmp_dir2 ...`. +4. Minimizes all testcases by running the fuzzer target with `-merge=1 tmp_dir3 tmp_dir1 tmp_dir2`. To run the script: -1. Make sure gcloud is installed: https://g3doc.corp.google.com/cloud/sdk/g3doc/index.md?cl=head -2. Login with @google.com credentials: `gcloud auth login` -3. You must be in a Chromium checkout using the GN arg `use_libfuzzer=true` -4. Run `./third_party/dawn/scripts/update_fuzzer_seed_corpus.sh `. +1. You must be in a Chromium checkout using the GN arg `use_libfuzzer=true` +2. Run `./third_party/dawn/scripts/update_fuzzer_seed_corpus.sh [additional_test_args]`. - Example: `./third_party/dawn/scripts/update_fuzzer_seed_corpus.sh out/fuzz dawn_wire_server_and_frontend_fuzzer dawn_end2end_tests` -5. The script will print instructions for testing, and then uploading new inputs. Please, only upload inputs after testing the fuzzer with new inputs, and verifying there is a meaningful change in coverage. + Example: `./third_party/dawn/scripts/update_fuzzer_seed_corpus.sh out/fuzz dawn_wire_server_and_vulkan_backend_fuzzer dawn_end2end_tests --gtest_filter=*Vulkan` +3. The script will print instructions for testing, and then uploading new inputs. Please, only upload inputs after testing the fuzzer with new inputs, and verifying there is a meaningful change in coverage. Uploading requires [gcloud](https://g3doc.corp.google.com/cloud/sdk/g3doc/index.md?cl=head) to be logged in with @google.com credentials: `gcloud auth login`. diff --git a/scripts/update_fuzzer_seed_corpus.sh b/scripts/update_fuzzer_seed_corpus.sh index 8438d6ae9b..966c1d6c89 100755 --- a/scripts/update_fuzzer_seed_corpus.sh +++ b/scripts/update_fuzzer_seed_corpus.sh @@ -20,29 +20,39 @@ # Exit if anything fails set -e -if [ "$#" -ne 3 ]; then +if [ "$#" -lt 3 ]; then cat << EOF Usage: - $0 + $0 [additional_test_args...] Example: - $0 out/fuzz dawn_wire_server_and_frontend_fuzzer dawn_end2end_tests + $0 out/fuzz dawn_wire_server_and_vulkan_backend_fuzzer dawn_end2end_tests --gtest_filter=*Vulkan EOF exit 1 fi +all_args=("$@") out_dir=$1 fuzzer_name=$2 test_name=$3 +additional_test_args=("${all_args[@]:3}") testcase_dir="/tmp/testcases/${fuzzer_name}/" +injected_error_testcase_dir="/tmp/testcases/${fuzzer_name}_injected/" minimized_testcase_dir="/tmp/testcases/${fuzzer_name}_minimized/" +# Print commands so it's clear what is being executed +set -x + # Make a directory for temporarily storing testcases mkdir -p "$testcase_dir" +# Make an empty directory for temporarily storing testcases with injected errors +rm -rf "$injected_error_testcase_dir" +mkdir -p "$injected_error_testcase_dir" + # Make an empty directory for temporarily storing minimized testcases rm -rf "$minimized_testcase_dir" mkdir -p "$minimized_testcase_dir" @@ -54,10 +64,16 @@ fuzzer_binary="${out_dir}/${fuzzer_name}" test_binary="${out_dir}/${test_name}" # Run the test binary -$test_binary --use-wire --wire-trace-dir="$testcase_dir" +$test_binary --use-wire --wire-trace-dir="$testcase_dir" $additional_test_args -# Run the fuzzer to minimize the corpus -$fuzzer_binary -merge=1 "$minimized_testcase_dir" "$testcase_dir" +# Run the fuzzer over the testcases to inject errors +$fuzzer_binary --injected-error-testcase-dir="$injected_error_testcase_dir" -runs=0 "$testcase_dir" + +# Run the fuzzer to minimize the testcases + injected errors +$fuzzer_binary -merge=1 "$minimized_testcase_dir" "$injected_error_testcase_dir" "$testcase_dir" + +# Turn off command printing +set +x if [ -z "$(ls -A $minimized_testcase_dir)" ]; then cat << EOF diff --git a/src/dawn_native/ErrorInjector.cpp b/src/dawn_native/ErrorInjector.cpp index d82102816e..836ef1ecab 100644 --- a/src/dawn_native/ErrorInjector.cpp +++ b/src/dawn_native/ErrorInjector.cpp @@ -15,6 +15,7 @@ #include "dawn_native/ErrorInjector.h" #include "common/Assert.h" +#include "dawn_native/DawnNative.h" namespace dawn_native { diff --git a/src/dawn_native/ErrorInjector.h b/src/dawn_native/ErrorInjector.h index 7fe3581f31..4d7d2b8a2b 100644 --- a/src/dawn_native/ErrorInjector.h +++ b/src/dawn_native/ErrorInjector.h @@ -26,12 +26,7 @@ namespace dawn_native { bool injected; }; - void EnableErrorInjector(); - void DisableErrorInjector(); - void ClearErrorInjector(); - bool ErrorInjectorEnabled(); - uint64_t AcquireErrorInjectorCallCount(); bool ShouldInjectError(); @@ -48,8 +43,6 @@ namespace dawn_native { return MaybeInjectError(errorTypes...); } - void InjectErrorAt(uint64_t index); - } // namespace dawn_native #if defined(DAWN_ENABLE_ERROR_INJECTION) diff --git a/src/fuzzers/DawnSPIRVCrossGLSLFastFuzzer.cpp b/src/fuzzers/DawnSPIRVCrossGLSLFastFuzzer.cpp index b3468eb0cf..7a658e0851 100644 --- a/src/fuzzers/DawnSPIRVCrossGLSLFastFuzzer.cpp +++ b/src/fuzzers/DawnSPIRVCrossGLSLFastFuzzer.cpp @@ -45,6 +45,10 @@ namespace { } // namespace +extern "C" int LLVMFuzzerInitialize(int* argc, char*** argv) { + return 0; +} + extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { return DawnSPIRVCrossFuzzer::Run(data, size, GLSLFastFuzzTask); } diff --git a/src/fuzzers/DawnSPIRVCrossHLSLFastFuzzer.cpp b/src/fuzzers/DawnSPIRVCrossHLSLFastFuzzer.cpp index 0566179f3a..e404be2d7c 100644 --- a/src/fuzzers/DawnSPIRVCrossHLSLFastFuzzer.cpp +++ b/src/fuzzers/DawnSPIRVCrossHLSLFastFuzzer.cpp @@ -51,6 +51,10 @@ namespace { } // namespace +extern "C" int LLVMFuzzerInitialize(int* argc, char*** argv) { + return 0; +} + extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { return DawnSPIRVCrossFuzzer::Run(data, size, FuzzTask); } diff --git a/src/fuzzers/DawnSPIRVCrossMSLFastFuzzer.cpp b/src/fuzzers/DawnSPIRVCrossMSLFastFuzzer.cpp index 5c4e6894fb..e4ff96457d 100644 --- a/src/fuzzers/DawnSPIRVCrossMSLFastFuzzer.cpp +++ b/src/fuzzers/DawnSPIRVCrossMSLFastFuzzer.cpp @@ -44,6 +44,10 @@ namespace { } // namespace +extern "C" int LLVMFuzzerInitialize(int* argc, char*** argv) { + return 0; +} + extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { return DawnSPIRVCrossFuzzer::Run(data, size, FuzzTask); } diff --git a/src/fuzzers/DawnSPVCglslFastFuzzer.cpp b/src/fuzzers/DawnSPVCglslFastFuzzer.cpp index 9a67c3ccd2..6fbaf82ff7 100644 --- a/src/fuzzers/DawnSPVCglslFastFuzzer.cpp +++ b/src/fuzzers/DawnSPVCglslFastFuzzer.cpp @@ -18,6 +18,10 @@ #include "spvc/spvc.hpp" +extern "C" int LLVMFuzzerInitialize(int* argc, char*** argv) { + return 0; +} + extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { shaderc_spvc::Context context; if (!context.IsValid()) { diff --git a/src/fuzzers/DawnSPVChlslFastFuzzer.cpp b/src/fuzzers/DawnSPVChlslFastFuzzer.cpp index f59bd2cfe9..540ac6567a 100644 --- a/src/fuzzers/DawnSPVChlslFastFuzzer.cpp +++ b/src/fuzzers/DawnSPVChlslFastFuzzer.cpp @@ -18,6 +18,10 @@ #include "spvc/spvc.hpp" +extern "C" int LLVMFuzzerInitialize(int* argc, char*** argv) { + return 0; +} + extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { shaderc_spvc::Context context; if (!context.IsValid()) { diff --git a/src/fuzzers/DawnSPVCmslFastFuzzer.cpp b/src/fuzzers/DawnSPVCmslFastFuzzer.cpp index 331d9247d8..70cb21e48f 100644 --- a/src/fuzzers/DawnSPVCmslFastFuzzer.cpp +++ b/src/fuzzers/DawnSPVCmslFastFuzzer.cpp @@ -18,6 +18,10 @@ #include "spvc/spvc.hpp" +extern "C" int LLVMFuzzerInitialize(int* argc, char*** argv) { + return 0; +} + extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { shaderc_spvc::Context context; if (!context.IsValid()) { diff --git a/src/fuzzers/DawnWireServerAndFrontendFuzzer.cpp b/src/fuzzers/DawnWireServerAndFrontendFuzzer.cpp index 7911738314..66de7662e4 100644 --- a/src/fuzzers/DawnWireServerAndFrontendFuzzer.cpp +++ b/src/fuzzers/DawnWireServerAndFrontendFuzzer.cpp @@ -17,21 +17,28 @@ #include "common/Assert.h" #include "dawn_native/DawnNative.h" -extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { - return DawnWireServerFuzzer::Run(data, size, [](dawn_native::Instance* instance) { - instance->DiscoverDefaultAdapters(); - - std::vector adapters = instance->GetAdapters(); - - wgpu::Device nullDevice; - for (dawn_native::Adapter adapter : adapters) { - if (adapter.GetBackendType() == dawn_native::BackendType::Null) { - nullDevice = wgpu::Device::Acquire(adapter.CreateDevice()); - break; - } - } - - ASSERT(nullDevice.Get() != nullptr); - return nullDevice; - }); +extern "C" int LLVMFuzzerInitialize(int* argc, char*** argv) { + return DawnWireServerFuzzer::Initialize(argc, argv); +} + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + return DawnWireServerFuzzer::Run( + data, size, + [](dawn_native::Instance* instance) { + instance->DiscoverDefaultAdapters(); + + std::vector adapters = instance->GetAdapters(); + + wgpu::Device nullDevice; + for (dawn_native::Adapter adapter : adapters) { + if (adapter.GetBackendType() == dawn_native::BackendType::Null) { + nullDevice = wgpu::Device::Acquire(adapter.CreateDevice()); + break; + } + } + + ASSERT(nullDevice.Get() != nullptr); + return nullDevice; + }, + false /* supportsErrorInjection */); } diff --git a/src/fuzzers/DawnWireServerAndVulkanBackendFuzzer.cpp b/src/fuzzers/DawnWireServerAndVulkanBackendFuzzer.cpp index ce39917862..266b75691c 100644 --- a/src/fuzzers/DawnWireServerAndVulkanBackendFuzzer.cpp +++ b/src/fuzzers/DawnWireServerAndVulkanBackendFuzzer.cpp @@ -14,25 +14,29 @@ #include "DawnWireServerFuzzer.h" -#include "common/Assert.h" #include "dawn_native/DawnNative.h" -extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { - return DawnWireServerFuzzer::Run(data, size, [](dawn_native::Instance* instance) { - instance->DiscoverDefaultAdapters(); - - std::vector adapters = instance->GetAdapters(); - - wgpu::Device device; - for (dawn_native::Adapter adapter : adapters) { - if (adapter.GetBackendType() == dawn_native::BackendType::Vulkan && - adapter.GetDeviceType() == dawn_native::DeviceType::CPU) { - device = wgpu::Device::Acquire(adapter.CreateDevice()); - break; - } - } - - ASSERT(device.Get() != nullptr); - return device; - }); +extern "C" int LLVMFuzzerInitialize(int* argc, char*** argv) { + return DawnWireServerFuzzer::Initialize(argc, argv); +} + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + return DawnWireServerFuzzer::Run( + data, size, + [](dawn_native::Instance* instance) { + instance->DiscoverDefaultAdapters(); + + std::vector adapters = instance->GetAdapters(); + + wgpu::Device device; + for (dawn_native::Adapter adapter : adapters) { + if (adapter.GetBackendType() == dawn_native::BackendType::Vulkan && + adapter.GetDeviceType() == dawn_native::DeviceType::CPU) { + device = wgpu::Device::Acquire(adapter.CreateDevice()); + break; + } + } + return device; + }, + true /* supportsErrorInjection */); } diff --git a/src/fuzzers/DawnWireServerFuzzer.cpp b/src/fuzzers/DawnWireServerFuzzer.cpp index f7a19f1ab3..522e281e71 100644 --- a/src/fuzzers/DawnWireServerFuzzer.cpp +++ b/src/fuzzers/DawnWireServerFuzzer.cpp @@ -15,11 +15,14 @@ #include "DawnWireServerFuzzer.h" #include "common/Assert.h" +#include "common/Log.h" +#include "common/SystemUtils.h" #include "dawn/dawn_proc.h" #include "dawn/webgpu_cpp.h" #include "dawn_native/DawnNative.h" #include "dawn_wire/WireServer.h" +#include #include namespace { @@ -42,6 +45,9 @@ namespace { WGPUProcDeviceCreateSwapChain sOriginalDeviceCreateSwapChain = nullptr; + std::string sInjectedErrorTestcaseOutDir; + uint64_t sOutputFileNumber = 0; + WGPUSwapChain ErrorDeviceCreateSwapChain(WGPUDevice device, const WGPUSwapChainDescriptor*) { WGPUSwapChainDescriptor desc; desc.nextInChain = nullptr; @@ -53,7 +59,61 @@ namespace { } // namespace -int DawnWireServerFuzzer::Run(const uint8_t* data, size_t size, MakeDeviceFn MakeDevice) { +int DawnWireServerFuzzer::Initialize(int* argc, char*** argv) { + ASSERT(argc != nullptr && argv != nullptr); + + // The first argument (the fuzzer binary) always stays the same. + int argcOut = 1; + + for (int i = 1; i < *argc; ++i) { + constexpr const char kInjectedErrorTestcaseDirArg[] = "--injected-error-testcase-dir="; + if (strstr((*argv)[i], kInjectedErrorTestcaseDirArg) == (*argv)[i]) { + sInjectedErrorTestcaseOutDir = (*argv)[i] + strlen(kInjectedErrorTestcaseDirArg); + const char* sep = GetPathSeparator(); + if (sInjectedErrorTestcaseOutDir.back() != *sep) { + sInjectedErrorTestcaseOutDir += sep; + } + // Log so that it's clear the fuzzer found the argument. + dawn::InfoLog() << "Generating injected errors, output dir is: \"" + << sInjectedErrorTestcaseOutDir << "\""; + continue; + } + + // Move any unconsumed arguments to the next slot in the output array. + (*argv)[argcOut++] = (*argv)[i]; + } + + // Write the argument count + *argc = argcOut; + + return 0; +} + +int DawnWireServerFuzzer::Run(const uint8_t* data, + size_t size, + MakeDeviceFn MakeDevice, + bool supportsErrorInjection) { + bool didInjectError = false; + + if (supportsErrorInjection) { + dawn_native::EnableErrorInjector(); + + // Clear the error injector since it has the previous run's call counts. + dawn_native::ClearErrorInjector(); + + // If we're outputing testcases with injected errors, we run the fuzzer on the original + // input data, and prepend injected errors to it. In the case, where we're NOT outputing, + // we use the first bytes as the injected error index. + if (sInjectedErrorTestcaseOutDir.empty() && size >= sizeof(uint64_t)) { + // Otherwise, use the first bytes as the injected error index. + dawn_native::InjectErrorAt(*reinterpret_cast(data)); + didInjectError = true; + + data += sizeof(uint64_t); + size -= sizeof(uint64_t); + } + } + DawnProcTable procs = dawn_native::GetProcs(); // Swapchains receive a pointer to an implementation. The fuzzer will pass garbage in so we @@ -67,7 +127,11 @@ int DawnWireServerFuzzer::Run(const uint8_t* data, size_t size, MakeDeviceFn Mak std::unique_ptr instance = std::make_unique(); wgpu::Device device = MakeDevice(instance.get()); - ASSERT(device); + if (!device) { + // We should only ever fail device creation if an error was injected. + ASSERT(didInjectError); + return 0; + } DevNull devNull; dawn_wire::WireServerDescriptor serverDesc = {}; @@ -87,5 +151,27 @@ int DawnWireServerFuzzer::Run(const uint8_t* data, size_t size, MakeDeviceFn Mak device = nullptr; instance = nullptr; + // If we support error injection, and an output directory was provided, output copies of the + // original testcase data, prepended with the injected error index. + if (supportsErrorInjection && !sInjectedErrorTestcaseOutDir.empty()) { + const uint64_t injectedCallCount = dawn_native::AcquireErrorInjectorCallCount(); + + auto WriteTestcase = [&](uint64_t i) { + std::ofstream outFile( + sInjectedErrorTestcaseOutDir + "injected_error_testcase_" + + std::to_string(sOutputFileNumber++), + std::ios_base::out | std::ios_base::binary | std::ios_base::trunc); + outFile.write(reinterpret_cast(&i), sizeof(i)); + outFile.write(reinterpret_cast(data), size); + }; + + for (uint64_t i = 0; i < injectedCallCount; ++i) { + WriteTestcase(i); + } + + // Also add a testcase where the injected error is so large no errors should occur. + WriteTestcase(std::numeric_limits::max()); + } + return 0; } diff --git a/src/fuzzers/DawnWireServerFuzzer.h b/src/fuzzers/DawnWireServerFuzzer.h index b4c011ebea..c2e176765f 100644 --- a/src/fuzzers/DawnWireServerFuzzer.h +++ b/src/fuzzers/DawnWireServerFuzzer.h @@ -27,6 +27,8 @@ namespace DawnWireServerFuzzer { using MakeDeviceFn = std::function; - int Run(const uint8_t* data, size_t size, MakeDeviceFn MakeDevice); + int Initialize(int* argc, char*** argv); + + int Run(const uint8_t* data, size_t size, MakeDeviceFn MakeDevice, bool supportsErrorInjection); } // namespace DawnWireServerFuzzer diff --git a/src/fuzzers/StandaloneFuzzerMain.cpp b/src/fuzzers/StandaloneFuzzerMain.cpp index 9755e1d646..33411991e9 100644 --- a/src/fuzzers/StandaloneFuzzerMain.cpp +++ b/src/fuzzers/StandaloneFuzzerMain.cpp @@ -17,11 +17,17 @@ #include #include +extern "C" int LLVMFuzzerInitialize(int* argc, char*** argv); extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size); int main(int argc, char** argv) { + if (LLVMFuzzerInitialize(&argc, &argv)) { + std::cerr << "Failed to initialize fuzzer target" << std::endl; + return 1; + } + if (argc != 2) { - std::cout << "Usage: FILE" << std::endl; + std::cout << "Usage: [options] FILE" << std::endl; return 1; } diff --git a/src/include/dawn_native/DawnNative.h b/src/include/dawn_native/DawnNative.h index 61fc0c0f96..a7e8d6c2e0 100644 --- a/src/include/dawn_native/DawnNative.h +++ b/src/include/dawn_native/DawnNative.h @@ -161,6 +161,14 @@ namespace dawn_native { // Backdoor to get the order of the ProcMap for testing DAWN_NATIVE_EXPORT std::vector GetProcMapNamesForTesting(); + + // ErrorInjector functions used for testing only. Defined in dawn_native/ErrorInjector.cpp + DAWN_NATIVE_EXPORT void EnableErrorInjector(); + DAWN_NATIVE_EXPORT void DisableErrorInjector(); + DAWN_NATIVE_EXPORT void ClearErrorInjector(); + DAWN_NATIVE_EXPORT uint64_t AcquireErrorInjectorCallCount(); + DAWN_NATIVE_EXPORT void InjectErrorAt(uint64_t index); + } // namespace dawn_native #endif // DAWNNATIVE_DAWNNATIVE_H_