test-runner: Change behavior of 'excluded' directories

For the directories "/test/unittest/" and "/test/vk-gl-cts/", continue to prevent the emission of PASS expectations, but now:
* Generate SKIP expectations in these directories if `--generate-skip` is passed.
* Delete expectation files in these directories if the tests now pass and `--generate-expected` is passed.

There's no change in behaviour for other test directories.

Change-Id: Ibd9e84a51029715e7c3d9e22e76a65770b66f1c7
Reviewed-on: https://dawn-review.googlesource.com/c/tint/+/77442
Kokoro: Kokoro <noreply+kokoro@google.com>
Reviewed-by: James Price <jrprice@google.com>
Commit-Queue: Ben Clayton <bclayton@google.com>

tools/src/cmd/test-runner/main.go

@@ -48,6 +48,14 @@ const (
 	wgsl   = outputFormat("wgsl")
 )
 
+// Directories we don't generate expected PASS result files for.
+// These directories contain large corpora of tests for which the generated code
+// is uninteresting.
+var dirsWithNoPassExpectations = []string{
+	"/test/unittest/",
+	"/test/vk-gl-cts/",
+}
+
 func main() {
 	if err := run(); err != nil {
 		fmt.Println(err)
@@ -489,8 +497,16 @@ type job struct {
 
 func (j job) run(wd, exe string, fxc bool, dxcPath, xcrunPath string, generateExpected, generateSkip bool) {
 	j.result <- func() status {
-		// Is there an expected output?
-		expected := loadExpectedFile(j.file, j.format)
+		// expectedFilePath is the path to the expected output file for the given test
+		expectedFilePath := j.file + ".expected." + string(j.format)
+
+		// Is there an expected output file? If so, load it.
+		expected, expectedFileExists := "", false
+		if content, err := ioutil.ReadFile(expectedFilePath); err == nil {
+			expected = string(content)
+			expectedFileExists = true
+		}
+
 		skipped := false
 		if strings.HasPrefix(expected, "SKIP") { // Special SKIP token
 			skipped = true
@@ -545,9 +561,29 @@ func (j job) run(wd, exe string, fxc bool, dxcPath, xcrunPath string, generateEx
 		out = strings.ReplaceAll(out, "\r\n", "\n")
 		matched := expected == "" || expected == out
 
+		canEmitPassExpectationFile := true
+		for _, noPass := range dirsWithNoPassExpectations {
+			if strings.Contains(j.file, filepath.FromSlash(noPass)) {
+				canEmitPassExpectationFile = false
+				break
+			}
+		}
+
+		saveExpectedFile := func(path string, content string) error {
+			return ioutil.WriteFile(path, []byte(content), 0666)
+		}
+
 		if ok && generateExpected && (validate || !skipped) {
-			saveExpectedFile(j.file, j.format, out)
-			matched = true
+			// User requested to update PASS expectations, and test passed.
+			if canEmitPassExpectationFile {
+				saveExpectedFile(expectedFilePath, out)
+			} else if expectedFileExists {
+				// Test lives in a directory where we do not want to save PASS
+				// files, and there already exists an expectation file. Test has
+				// likely started passing. Delete the old expectation.
+				os.Remove(expectedFilePath)
+			}
+			matched = true // test passed and matched expectations
 		}
 
 		switch {
@@ -559,14 +595,14 @@ func (j job) run(wd, exe string, fxc bool, dxcPath, xcrunPath string, generateEx
 
 		case skipped:
 			if generateSkip {
-				saveExpectedFile(j.file, j.format, "SKIP: FAILED\n\n"+out)
+				saveExpectedFile(expectedFilePath, "SKIP: FAILED\n\n"+out)
 			}
 			return status{code: skip, timeTaken: timeTaken}
 
 		case !ok:
 			// Compiler returned non-zero exit code
 			if generateSkip {
-				saveExpectedFile(j.file, j.format, "SKIP: FAILED\n\n"+out)
+				saveExpectedFile(expectedFilePath, "SKIP: FAILED\n\n"+out)
 			}
 			err := fmt.Errorf("%s", out)
 			return status{code: fail, err: err, timeTaken: timeTaken}
@@ -574,7 +610,7 @@ func (j job) run(wd, exe string, fxc bool, dxcPath, xcrunPath string, generateEx
 		default:
 			// Compiler returned zero exit code, or output was not as expected
 			if generateSkip {
-				saveExpectedFile(j.file, j.format, "SKIP: FAILED\n\n"+out)
+				saveExpectedFile(expectedFilePath, "SKIP: FAILED\n\n"+out)
 			}
 
 			// Expected output did not match
@@ -602,36 +638,6 @@ func (j job) run(wd, exe string, fxc bool, dxcPath, xcrunPath string, generateEx
 	}()
 }
 
-// loadExpectedFile loads the expected output file for the test file at 'path'
-// and the output format 'format'. If the file does not exist, or cannot be
-// read, then an empty string is returned.
-func loadExpectedFile(path string, format outputFormat) string {
-	content, err := ioutil.ReadFile(expectedFilePath(path, format))
-	if err != nil {
-		return ""
-	}
-	return string(content)
-}
-
-// saveExpectedFile writes the expected output file for the test file at 'path'
-// and the output format 'format', with the content 'content'.
-func saveExpectedFile(path string, format outputFormat, content string) error {
-	// Don't generate expected results for certain directories that contain
-	// large corpora of tests for which the generated code is uninteresting.
-	for _, exclude := range []string{"/test/unittest/", "/test/vk-gl-cts/"} {
-		if strings.Contains(path, filepath.FromSlash(exclude)) {
-			return nil
-		}
-	}
-	return ioutil.WriteFile(expectedFilePath(path, format), []byte(content), 0666)
-}
-
-// expectedFilePath returns the expected output file path for the test file at
-// 'path' and the output format 'format'.
-func expectedFilePath(path string, format outputFormat) string {
-	return path + ".expected." + string(format)
-}
-
 // indent returns the string 's' indented with 'n' whitespace characters
 func indent(s string, n int) string {
 	tab := strings.Repeat(" ", n)