From 2270b434bc2574a307eab2f21fbbbbad1c21a190 Mon Sep 17 00:00:00 2001
From: Ben Clayton <bclayton@google.com>
Date: Thu, 3 Jun 2021 10:47:24 +0000
Subject: [PATCH] [tools] test-runner: Add --generate-skip flag

Generates expected cases that fail, using a SKIP header

Change-Id: I2738fa2cf47c715fe6e978edf51aa928c5e3b216
Reviewed-on: https://dawn-review.googlesource.com/c/tint/+/53049
Kokoro: Kokoro <noreply+kokoro@google.com>
Commit-Queue: Ben Clayton <bclayton@chromium.org>
Reviewed-by: James Price <jrprice@google.com>
---
 tools/src/cmd/test-runner/main.go | 77 ++++++++++++++++++++-----------
 1 file changed, 51 insertions(+), 26 deletions(-)

diff --git a/tools/src/cmd/test-runner/main.go b/tools/src/cmd/test-runner/main.go
index 160f4652df..8345e3ebb1 100644
--- a/tools/src/cmd/test-runner/main.go
+++ b/tools/src/cmd/test-runner/main.go
@@ -70,12 +70,13 @@ optional flags:`)
 func run() error {
 	var formatList, filter, dxcPath, xcrunPath string
 	numCPU := runtime.NumCPU()
-	generateExpected := false
+	generateExpected, generateSkip := false, false
 	flag.StringVar(&formatList, "format", "all", "comma separated list of formats to emit. Possible values are: all, wgsl, spvasm, msl, hlsl")
 	flag.StringVar(&filter, "filter", "**.wgsl, **.spvasm, **.spv", "comma separated list of glob patterns for test files")
 	flag.StringVar(&dxcPath, "dxc", "", "path to DXC executable for validating HLSL output")
 	flag.StringVar(&xcrunPath, "xcrun", "", "path to xcrun executable for validating MSL output")
 	flag.BoolVar(&generateExpected, "generate-expected", false, "create or update all expected outputs")
+	flag.BoolVar(&generateSkip, "generate-skip", false, "create or update all expected outputs that fail with SKIP")
 	flag.IntVar(&numCPU, "j", numCPU, "maximum number of concurrent threads to run tests")
 	flag.Usage = showUsage
 	flag.Parse()
@@ -199,7 +200,7 @@ func run() error {
 		go func() {
 			defer wg.Done()
 			for job := range jobs {
-				job.run(exe, dxcPath, xcrunPath, generateExpected)
+				job.run(exe, dxcPath, xcrunPath, generateExpected, generateSkip)
 			}
 		}()
 	}
@@ -355,12 +356,12 @@ type job struct {
 	result *status
 }
 
-func (j job) run(exe, dxcPath, xcrunPath string, generateExpected bool) {
+func (j job) run(exe, dxcPath, xcrunPath string, generateExpected, generateSkip bool) {
 	// Is there an expected output?
 	expected := loadExpectedFile(j.file, j.format)
+	skipped := false
 	if strings.HasPrefix(expected, "SKIP") { // Special SKIP token
-		*j.result = status{code: skip}
-		return
+		skipped = true
 	}
 
 	expected = strings.ReplaceAll(expected, "\r\n", "\n")
@@ -382,17 +383,49 @@ func (j job) run(exe, dxcPath, xcrunPath string, generateExpected bool) {
 	}
 
 	// Invoke the compiler...
-	var err error
-	if ok, out := invoke(exe, args...); ok {
-		out = strings.ReplaceAll(out, "\r\n", "\n")
-		if generateExpected {
-			// If --generate-expected was passed, write out the output
-			err = saveExpectedFile(j.file, j.format, out)
-		} else if expected != "" && expected != out {
-			// Expected output did not match
-			dmp := diffmatchpatch.New()
-			diff := dmp.DiffPrettyText(dmp.DiffMain(expected, out, true))
-			err = fmt.Errorf(`Output was not as expected
+	ok, out := invoke(exe, args...)
+	out = strings.ReplaceAll(out, "\r\n", "\n")
+	matched := expected == "" || expected == out
+
+	if ok && generateExpected {
+		saveExpectedFile(j.file, j.format, out)
+		matched = true
+	}
+
+	switch {
+	case ok && matched:
+		// Test passed
+		*j.result = status{code: pass}
+		return
+
+		//       --- Below this point the test has failed ---
+
+	case skipped:
+		if generateSkip {
+			saveExpectedFile(j.file, j.format, "SKIP: FAILED\n\n"+out)
+		}
+		*j.result = status{code: skip}
+		return
+
+	case !ok:
+		// Compiler returned non-zero exit code
+		if generateSkip {
+			saveExpectedFile(j.file, j.format, "SKIP: FAILED\n\n"+out)
+		}
+		err := fmt.Errorf("%s", out)
+		*j.result = status{code: fail, err: err}
+		return
+
+	default:
+		// Compiler returned zero exit code, or output was not as expected
+		if generateSkip {
+			saveExpectedFile(j.file, j.format, "SKIP: FAILED\n\n"+out)
+		}
+
+		// Expected output did not match
+		dmp := diffmatchpatch.New()
+		diff := dmp.DiffPrettyText(dmp.DiffMain(expected, out, true))
+		err := fmt.Errorf(`Output was not as expected
 
 --------------------------------------------------------------------------------
 -- Expected:                                                                  --
@@ -408,17 +441,9 @@ func (j job) run(exe, dxcPath, xcrunPath string, generateExpected bool) {
 -- Diff:                                                                      --
 --------------------------------------------------------------------------------
 %s`,
-				expected, out, diff)
-		}
-	} else {
-		// Compiler returned a non-zero exit code
-		err = fmt.Errorf("%s", out)
-	}
-
-	if err != nil {
+			expected, out, diff)
 		*j.result = status{code: fail, err: err}
-	} else {
-		*j.result = status{code: pass}
+		return
 	}
 }