tint: Skip e2e test validation for known-good

Cache the hash of the test output, and only re-validate if the output changes.
This speeds up incremental end-to-end testing.

Change-Id: I44bc1eff62320df1e80524e8481d1c012fb67f42
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/109761
Commit-Queue: Ben Clayton <bclayton@google.com>
Reviewed-by: James Price <jrprice@google.com>
Kokoro: Kokoro <noreply+kokoro@google.com>
Auto-Submit: Ben Clayton <bclayton@google.com>

.gitignore

@@ -39,6 +39,9 @@
 /tools/memory
 /out
 
+# Tint test validation cache file
+/test/tint/validation.cache
+
 # Modified from https://www.gitignore.io/api/vim,macos,linux,emacs,windows,sublimetext,visualstudio,visualstudiocode,intellij
 
 ### Emacs ###

src/tint/cmd/main.cc

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <charconv>
 #include <cstdio>
 #include <fstream>
 #include <iostream>
@@ -62,6 +63,11 @@ namespace {
     exit(1);
 }
 
+/// Prints the given hash value in a format string that the end-to-end test runner can parse.
+void PrintHash(uint32_t hash) {
+    std::cout << "<<HASH: 0x" << std::hex << hash << ">>" << std::endl;
+}
+
 enum class Format {
     kNone = -1,
     kSpirv,
@@ -82,9 +88,12 @@ struct Options {
     bool parse_only = false;
     bool disable_workgroup_init = false;
     bool validate = false;
+    bool print_hash = false;
     bool demangle = false;
     bool dump_inspector_bindings = false;
 
+    std::unordered_set<uint32_t> skip_hash;
+
     Format format = Format::kNone;
 
     bool emit_single_entry_point = false;
@@ -122,8 +131,7 @@ const char kUsage[] = R"(Usage: tint [options] <input-file>
   -o <name>                 -- Output file name.  Use "-" for standard output
   --transform <name list>   -- Runs transforms, name list is comma separated
                                Available transforms:
-${transforms}
+${transforms} --parse-only              -- Stop after parsing the input
-  --parse-only              -- Stop after parsing the input
   --disable-workgroup-init  -- Disable workgroup memory zero initialization.
   --demangle                -- Preserve original source names. Demangle them.
                                Affects AST dumping, and text-based output languages.
@@ -135,6 +143,9 @@ ${transforms}
                                default to binding 0 of the largest used group plus 1,
                                or group 0 if no resource bound.
   --validate                -- Validates the generated shader with all available validators
+  --skip-hash <hash list>   -- Skips validation if the hash of the output is equal to any
+                               of the hash codes in the comma separated list of hashes
+  --print-hash              -- Emit the hash of the output program
   --fxc                     -- Path to FXC dll, used to validate HLSL output.
                                When specified, automatically enables HLSL validation with FXC
   --dxc                     -- Path to DXC executable, used to validate HLSL output.
@@ -431,6 +442,24 @@ bool ParseArgs(const std::vector<std::string>& args, Options* opts) {
             opts->dump_inspector_bindings = true;
         } else if (arg == "--validate") {
             opts->validate = true;
+        } else if (arg == "--skip-hash") {
+            ++i;
+            if (i >= args.size()) {
+                std::cerr << "Missing hash value for " << arg << std::endl;
+                return false;
+            }
+            for (auto hash : split_on_comma(args[i])) {
+                uint32_t value = 0;
+                int base = 10;
+                if (hash.size() > 2 && hash[0] == '0' && (hash[1] == 'x' || hash[1] == 'X')) {
+                    hash = hash.substr(2);
+                    base = 16;
+                }
+                std::from_chars(hash.data(), hash.data() + hash.size(), value, base);
+                opts->skip_hash.emplace(value);
+            }
+        } else if (arg == "--print-hash") {
+            opts->print_hash = true;
         } else if (arg == "--fxc") {
             ++i;
             if (i >= args.size()) {
@@ -682,7 +711,12 @@ bool GenerateSpirv(const tint::Program* program, const Options& options) {
         }
     }
 
-    if (options.validate) {
+    const auto hash = tint::utils::CRC32(result.spirv.data(), result.spirv.size());
+    if (options.print_hash) {
+        PrintHash(hash);
+    }
+
+    if (options.validate && options.skip_hash.count(hash) == 0) {
         // Use Vulkan 1.1, since this is what Tint, internally, uses.
         spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_1);
         tools.SetMessageConsumer(
@@ -722,7 +756,12 @@ bool GenerateWgsl(const tint::Program* program, const Options& options) {
         return false;
     }
 
-    if (options.validate) {
+    const auto hash = tint::utils::CRC32(result.wgsl.data(), result.wgsl.size());
+    if (options.print_hash) {
+        PrintHash(hash);
+    }
+
+    if (options.validate && options.skip_hash.count(hash) == 0) {
         // Attempt to re-parse the output program with Tint's WGSL reader.
         auto source = std::make_unique<tint::Source::File>(options.input_filename, result.wgsl);
         auto reparsed_program = tint::reader::wgsl::Parse(source.get());
@@ -772,7 +811,12 @@ bool GenerateMsl(const tint::Program* program, const Options& options) {
         return false;
     }
 
-    if (options.validate) {
+    const auto hash = tint::utils::CRC32(result.msl.c_str());
+    if (options.print_hash) {
+        PrintHash(hash);
+    }
+
+    if (options.validate && options.skip_hash.count(hash) == 0) {
         tint::val::Result res;
 #ifdef TINT_ENABLE_MSL_VALIDATION_USING_METAL_API
         res = tint::val::MslUsingMetalAPI(result.msl);
@@ -828,11 +872,17 @@ bool GenerateHlsl(const tint::Program* program, const Options& options) {
         return false;
     }
 
+    const auto hash = tint::utils::CRC32(result.hlsl.c_str());
+    if (options.print_hash) {
+        PrintHash(hash);
+    }
+
     // If --fxc or --dxc was passed, then we must explicitly find and validate with that respective
     // compiler.
     const bool must_validate_dxc = !options.dxc_path.empty();
     const bool must_validate_fxc = !options.fxc_path.empty();
-    if (options.validate || must_validate_dxc || must_validate_fxc) {
+    if ((options.validate || must_validate_dxc || must_validate_fxc) &&
+        (options.skip_hash.count(hash) == 0)) {
         tint::val::Result dxc_res;
         bool dxc_found = false;
         if (options.validate || must_validate_dxc) {
@@ -959,7 +1009,12 @@ bool GenerateGlsl(const tint::Program* program, const Options& options) {
             return false;
         }
 
-        if (options.validate) {
+        const auto hash = tint::utils::CRC32(result.glsl.c_str());
+        if (options.print_hash) {
+            PrintHash(hash);
+        }
+
+        if (options.validate && options.skip_hash.count(hash) == 0) {
             for (auto entry_pt : result.entry_points) {
                 EShLanguage lang = pipeline_stage_to_esh_language(entry_pt.second);
                 glslang::TShader shader(lang);

src/tint/utils/crc32.h

@@ -16,56 +16,62 @@
 #define SRC_TINT_UTILS_CRC32_H_
 
 #include <stdint.h>
+#include <cstddef>
 
 namespace tint::utils {
 
-/// @returns the CRC32 of the string `s`.
+constexpr uint32_t kCRC32LUT[] = {
+    0,          0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3,
+    0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91,
+    0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7,
+    0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5,
+    0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b,
+    0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59,
+    0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f,
+    0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d,
+    0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433,
+    0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01,
+    0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457,
+    0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65,
+    0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb,
+    0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9,
+    0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
+    0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad,
+    0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683,
+    0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1,
+    0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7,
+    0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5,
+    0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b,
+    0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79,
+    0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f,
+    0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d,
+    0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713,
+    0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21,
+    0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777,
+    0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45,
+    0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db,
+    0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
+    0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf,
+    0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d};
+
+/// @returns the CRC32 of the string @p s.
 /// @note this function can be used to calculate the CRC32 of a string literal
 /// at compile time.
 /// @see https://en.wikipedia.org/wiki/Cyclic_redundancy_check#CRC-32_algorithm
 constexpr uint32_t CRC32(const char* s) {
-    constexpr uint32_t kLUT[] = {
-        0,          0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535,
-        0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd,
-        0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d,
-        0x6ddde4eb, 0xf4d4b551, 0x83d385c7, 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec,
-        0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4,
-        0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c,
-        0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, 0x26d930ac,
-        0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f,
-        0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 0x2f6f7c87, 0x58684c11, 0xc1611dab,
-        0xb6662d3d, 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f,
-        0x9fbfe4a5, 0xe8b8d433, 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb,
-        0x086d3d2d, 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e,
-        0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea,
-        0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, 0x4db26158, 0x3ab551ce,
-        0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a,
-        0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9,
-        0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409,
-        0xce61e49f, 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81,
-        0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 0xead54739,
-        0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8,
-        0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 0xf00f9344, 0x8708a3d2, 0x1e01f268,
-        0x6906c2fe, 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0,
-        0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8,
-        0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b,
-        0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef,
-        0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703,
-        0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7,
-        0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a,
-        0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae,
-        0x0cb61b38, 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242,
-        0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, 0x88085ae6,
-        0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45,
-        0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d,
-        0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5,
-        0x47b2cf7f, 0x30b5ffe9, 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605,
-        0xcdd70693, 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94,
-        0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d};
-
     uint32_t crc = 0xffffffff;
     for (auto* p = s; *p != '\0'; ++p) {
-        crc = (crc >> 8) ^ kLUT[static_cast<uint8_t>(crc) ^ static_cast<uint8_t>(*p)];
+        crc = (crc >> 8) ^ kCRC32LUT[static_cast<uint8_t>(crc) ^ static_cast<uint8_t>(*p)];
+    }
+    return crc ^ 0xffffffff;
+}
+
+/// @returns the CRC32 of the data at @p ptr of size @p size.
+inline uint32_t CRC32(const void* ptr, size_t size) {
+    auto* p = static_cast<const uint8_t*>(ptr);
+    uint32_t crc = 0xffffffff;
+    while (size--) {
+        crc = (crc >> 8) ^ kCRC32LUT[static_cast<uint8_t>(crc) ^ *p++];
     }
     return crc ^ 0xffffffff;
 }

tools/src/cmd/test-runner/main.go

@@ -17,6 +17,8 @@ package main
 
 import (
 	"context"
+	"crypto/sha256"
+	"encoding/json"
 	"flag"
 	"fmt"
 	"io/ioutil"
@@ -190,6 +192,8 @@ func run() error {
 		defaultMSLExe = "metal.exe"
 	}
 
+	toolchainHash := sha256.New()
+
 	// If explicit verification compilers have been specified, check they exist.
 	// Otherwise, look on PATH for them, but don't error if they cannot be found.
 	for _, tool := range []struct {
@@ -222,9 +226,18 @@ func run() error {
 		}
 		color.Unset()
 		fmt.Println()
+
+		toolchainHash.Write([]byte(tool.name))
+		if s, err := os.Stat(*tool.path); err == nil {
+			toolchainHash.Write([]byte(s.ModTime().String()))
+			toolchainHash.Write([]byte(fmt.Sprint(s.Size())))
+		}
 	}
 	fmt.Println()
 
+	validationCache := loadValidationCache(fmt.Sprintf("%x", toolchainHash.Sum(nil)))
+	defer saveValidationCache(validationCache)
+
 	// Build the list of results.
 	// These hold the chans used to report the job results.
 	results := make([]map[outputFormat]chan status, len(files))
@@ -247,6 +260,7 @@ func run() error {
 		xcrunPath:        xcrunPath,
 		generateExpected: generateExpected,
 		generateSkip:     generateSkip,
+		validationCache:  validationCache,
 	}
 	for cpu := 0; cpu < numCPU; cpu++ {
 		go func() {
@@ -326,6 +340,8 @@ func run() error {
 	printFormatsHeader()
 	printHorizontalLine()
 
+	newKnownGood := knownGoodHashes{}
+
 	for i, file := range files {
 		results := results[i]
 
@@ -343,6 +359,11 @@ func run() error {
 		for _, format := range formats {
 			columnWidth := formatWidth(format)
 			result := <-results[format]
+
+			// Update the known-good hashes
+			newKnownGood[fileAndFormat{file, format}] = result.passHashes
+
+			// Update stats
 			stats := statsByFmt[format]
 			stats.numTests++
 			stats.timeTaken += result.timeTaken
@@ -351,6 +372,7 @@ func run() error {
 					file: file, format: format, err: err,
 				})
 			}
+
 			switch result.code {
 			case pass:
 				green.Fprintf(row, alignCenter("PASS", columnWidth))
@@ -375,6 +397,17 @@ func run() error {
 		}
 	}
 
+	// Update the validation cache known-good hashes.
+	// This has to be done after all the results have been collected to avoid
+	// concurrent access on the map.
+	for ff, hashes := range newKnownGood {
+		if len(newKnownGood) > 0 {
+			validationCache.knownGood[ff] = hashes
+		} else {
+			delete(validationCache.knownGood, ff)
+		}
+	}
+
 	printHorizontalLine()
 	printFormatsHeader()
 	printHorizontalLine()
@@ -497,9 +530,10 @@ const (
 )
 
 type status struct {
-	code      statusCode
+	code       statusCode
-	err       error
+	err        error
-	timeTaken time.Duration
+	timeTaken  time.Duration
+	passHashes []string
 }
 
 type job struct {
@@ -517,6 +551,7 @@ type runConfig struct {
 	xcrunPath        string
 	generateExpected bool
 	generateSkip     bool
+	validationCache  validationCache
 }
 
 func (j job) run(cfg runConfig) {
@@ -558,6 +593,14 @@ func (j job) run(cfg runConfig) {
 		args := []string{
 			file,
 			"--format", strings.Split(string(j.format), "-")[0], // 'hlsl-fxc' -> 'hlsl', etc.
+			"--print-hash",
+		}
+
+		// Append any skip-hashes, if they're found.
+		if j.format != "wgsl" { // Don't skip 'wgsl' as this 'toolchain' is ever changing.
+			if skipHashes := cfg.validationCache.knownGood[fileAndFormat{file, j.format}]; len(skipHashes) > 0 {
+				args = append(args, "--skip-hash", strings.Join(skipHashes, ","))
+			}
 		}
 
 		// Can we validate?
@@ -597,6 +640,7 @@ func (j job) run(cfg runConfig) {
 		timeTaken := time.Since(start)
 
 		out = strings.ReplaceAll(out, "\r\n", "\n")
+		out, hashes := extractValidationHashes(out)
 		matched := expected == "" || expected == out
 
 		canEmitPassExpectationFile := true
@@ -627,7 +671,7 @@ func (j job) run(cfg runConfig) {
 		switch {
 		case ok && matched:
 			// Test passed
-			return status{code: pass, timeTaken: timeTaken}
+			return status{code: pass, timeTaken: timeTaken, passHashes: hashes}
 
 			//       --- Below this point the test has failed ---
 
@@ -676,6 +720,24 @@ func (j job) run(cfg runConfig) {
 	}()
 }
 
+var reValidationHash = regexp.MustCompile(`<<HASH: ([^>]*)>>\n`)
+
+// Parses and returns the validation hashes emitted by tint, or an empty string
+// if the hash wasn't found, along with the input string with the validation
+// hashes removed.
+func extractValidationHashes(in string) (out string, hashes []string) {
+	matches := reValidationHash.FindAllStringSubmatch(in, -1)
+	if matches == nil {
+		return in, nil
+	}
+	out = in
+	for _, match := range matches {
+		out = strings.ReplaceAll(out, match[0], "")
+		hashes = append(hashes, match[1])
+	}
+	return out, hashes
+}
+
 // indent returns the string 's' indented with 'n' whitespace characters
 func indent(s string, n int) string {
 	tab := strings.Repeat(" ", n)
@@ -803,3 +865,109 @@ func printDuration(d time.Duration) string {
 	}
 	return sb.String()
 }
+
+// fileAndFormat is a pair of test file path and output format.
+type fileAndFormat struct {
+	file   string
+	format outputFormat
+}
+
+// Used to optimize end-to-end testing of tint
+type validationCache struct {
+	// A hash of all the validation toolchains in use.
+	toolchainHash string
+	// A map of fileAndFormat to known-good (validated) output hashes.
+	knownGood knownGoodHashes
+}
+
+// A map of fileAndFormat to known-good (validated) output hashes.
+type knownGoodHashes map[fileAndFormat][]string
+
+// The serialized form of a known-good validation.cache file
+type ValidationCacheFile struct {
+	ToolchainHash string
+	KnownGood     []ValidationCacheFileKnownGood
+}
+
+type ValidationCacheFileKnownGood struct {
+	File   string
+	Format outputFormat
+	Hashes []string
+}
+
+func validationCachePath() string {
+	return filepath.Join(fileutils.DawnRoot(), "test", "tint", "validation.cache")
+}
+
+// loadValidationCache attempts to load the validation cache.
+// Returns an empty cache if the file could not be loaded, or if toolchains have changed.
+func loadValidationCache(toolchainHash string) validationCache {
+	out := validationCache{
+		toolchainHash: toolchainHash,
+		knownGood:     knownGoodHashes{},
+	}
+
+	file, err := os.Open(validationCachePath())
+	if err != nil {
+		return out
+	}
+	defer file.Close()
+
+	content := ValidationCacheFile{}
+	if err := json.NewDecoder(file).Decode(&content); err != nil {
+		return out
+	}
+
+	if content.ToolchainHash != toolchainHash {
+		color.Set(color.FgYellow)
+		fmt.Println("Toolchains have changed - clearing validation cache")
+		color.Unset()
+		return out
+	}
+
+	for _, knownGood := range content.KnownGood {
+		out.knownGood[fileAndFormat{knownGood.File, knownGood.Format}] = knownGood.Hashes
+	}
+
+	return out
+}
+
+// saveValidationCache saves the validation cache file.
+func saveValidationCache(vc validationCache) error {
+	out := ValidationCacheFile{
+		ToolchainHash: vc.toolchainHash,
+		KnownGood:     make([]ValidationCacheFileKnownGood, 0, len(vc.knownGood)),
+	}
+
+	for ff, hashes := range vc.knownGood {
+		out.KnownGood = append(out.KnownGood, ValidationCacheFileKnownGood{
+			File:   ff.file,
+			Format: ff.format,
+			Hashes: hashes,
+		})
+	}
+
+	sort.Slice(out.KnownGood, func(i, j int) bool {
+		switch {
+		case out.KnownGood[i].File < out.KnownGood[j].File:
+			return true
+		case out.KnownGood[i].File > out.KnownGood[j].File:
+			return false
+		case out.KnownGood[i].Format < out.KnownGood[j].Format:
+			return true
+		case out.KnownGood[i].Format > out.KnownGood[j].Format:
+			return false
+		}
+		return false
+	})
+
+	file, err := os.Create(validationCachePath())
+	if err != nil {
+		return fmt.Errorf("failed to save the validation cache file: %w", err)
+	}
+	defer file.Close()
+
+	enc := json.NewEncoder(file)
+	enc.SetIndent("", "  ")
+	return enc.Encode(&out)
+}