test-runner: Print results as soon as they're available

Gives feedback much eariler.

Also move the error messages after the table

Change-Id: If462fd9bbc72ff9428d6ee77c8ec6c9338f7f27e
Reviewed-on: https://dawn-review.googlesource.com/c/tint/+/54000
Kokoro: Kokoro <noreply+kokoro@google.com>
Commit-Queue: Ben Clayton <bclayton@google.com>
Auto-Submit: Ben Clayton <bclayton@google.com>
Reviewed-by: James Price <jrprice@google.com>

tools/src/cmd/test-runner/main.go

@@ -26,7 +26,6 @@ import (
 	"sort"
 	"strconv"
 	"strings"
-	"sync"
 	"unicode/utf8"
 
 	"dawn.googlesource.com/tint/tools/src/fileutils"
@@ -105,7 +104,7 @@ func run() error {
 	}
 
 	// Allow using '/' in the filter on Windows
-	filter = strings.ReplaceAll(filter, "/", string(filepath.Separator));
+	filter = strings.ReplaceAll(filter, "/", string(filepath.Separator))
 
 	// Split the --filter flag up by ',', trimming any whitespace at the start and end
 	globIncludes := strings.Split(filter, ",")
@@ -199,62 +198,51 @@ func run() error {
 	}
 	fmt.Println()
 
-	results := make([]map[outputFormat]*status, len(files))
+	// Build the list of results.
-	jobs := make(chan job, 256)
+	// These hold the chans used to report the job results.
+	results := make([]map[outputFormat]chan status, len(files))
+	for i := range files {
+		fileResults := map[outputFormat]chan status{}
+		for _, format := range formats {
+			fileResults[format] = make(chan status, 1)
+		}
+		results[i] = fileResults
+	}
+
+	pendingJobs := make(chan job, 256)
 
 	// Spawn numCPU job runners...
-	wg := sync.WaitGroup{}
-	wg.Add(numCPU)
 	for cpu := 0; cpu < numCPU; cpu++ {
 		go func() {
-			defer wg.Done()
+			for job := range pendingJobs {
-			for job := range jobs {
 				job.run(dir, exe, dxcPath, xcrunPath, generateExpected, generateSkip)
 			}
 		}()
 	}
 
 	// Issue the jobs...
+	go func() {
 		for i, file := range files { // For each test file...
 			file := filepath.Join(dir, file)
-		fileResults := map[outputFormat]*status{}
 			for _, format := range formats { // For each output format...
-			result := &status{}
+				pendingJobs <- job{
-			jobs <- job{
 					file:   file,
 					format: format,
-				result: result,
+					result: results[i][format],
 				}
-			fileResults[format] = result
 			}
-		results[i] = fileResults
+		}
+		close(pendingJobs)
+	}()
+
+	type failure struct {
+		file   string
+		format outputFormat
+		err    error
 	}
 
-	// Wait for the jobs to all finish...
+	// Print the table of file x format
-	close(jobs)
+	failures := []failure{}
-	wg.Wait()
-
-	// Time to print the outputs
-
-	// Start by printing the error message for any file x format combinations
-	// that failed...
-	for i, file := range files {
-		results := results[i]
-		for _, format := range formats {
-			if err := results[format].err; err != nil {
-				color.Set(color.FgBlue)
-				fmt.Printf("%s ", file)
-				color.Set(color.FgCyan)
-				fmt.Printf("%s ", format)
-				color.Set(color.FgRed)
-				fmt.Println("FAIL")
-				color.Unset()
-				fmt.Println(indent(err.Error(), 4))
-			}
-		}
-	}
-
-	// Now print the table of file x format
 	numTests, numPass, numSkip, numFail := 0, 0, 0, 0
 	filenameFmt := columnFormat(maxStringLen(files), false)
 
@@ -285,8 +273,13 @@ func run() error {
 		fmt.Printf(" ┃ ")
 		for _, format := range formats {
 			formatFmt := columnFormat(formatWidth(format), true)
-			result := results[format]
+			result := <-results[format]
 			numTests++
+			if err := result.err; err != nil {
+				failures = append(failures, failure{
+					file: file, format: format, err: err,
+				})
+			}
 			switch result.code {
 			case pass:
 				color.Set(color.FgGreen)
@@ -310,6 +303,20 @@ func run() error {
 	}
 	fmt.Println()
 
+	for _, f := range failures {
+		color.Set(color.FgBlue)
+		fmt.Printf("%s ", f.file)
+		color.Set(color.FgCyan)
+		fmt.Printf("%s ", f.format)
+		color.Set(color.FgRed)
+		fmt.Println("FAIL")
+		color.Unset()
+		fmt.Println(indent(f.err.Error(), 4))
+	}
+	if len(failures) > 0 {
+		fmt.Println()
+	}
+
 	fmt.Printf("%d tests run", numTests)
 	if numPass > 0 {
 		fmt.Printf(", ")
@@ -362,10 +369,11 @@ type status struct {
 type job struct {
 	file   string
 	format outputFormat
-	result *status
+	result chan status
 }
 
 func (j job) run(wd, exe, dxcPath, xcrunPath string, generateExpected, generateSkip bool) {
+	j.result <- func() status {
 		// Is there an expected output?
 		expected := loadExpectedFile(j.file, j.format)
 		skipped := false
@@ -420,8 +428,7 @@ func (j job) run(wd, exe, dxcPath, xcrunPath string, generateExpected, generateS
 		switch {
 		case ok && matched:
 			// Test passed
-		*j.result = status{code: pass}
+			return status{code: pass}
-		return
 
 			//       --- Below this point the test has failed ---
 
@@ -429,8 +436,7 @@ func (j job) run(wd, exe, dxcPath, xcrunPath string, generateExpected, generateS
 			if generateSkip {
 				saveExpectedFile(j.file, j.format, "SKIP: FAILED\n\n"+out)
 			}
-		*j.result = status{code: skip}
+			return status{code: skip}
-		return
 
 		case !ok:
 			// Compiler returned non-zero exit code
@@ -438,8 +444,7 @@ func (j job) run(wd, exe, dxcPath, xcrunPath string, generateExpected, generateS
 				saveExpectedFile(j.file, j.format, "SKIP: FAILED\n\n"+out)
 			}
 			err := fmt.Errorf("%s", out)
-		*j.result = status{code: fail, err: err}
+			return status{code: fail, err: err}
-		return
 
 		default:
 			// Compiler returned zero exit code, or output was not as expected
@@ -467,9 +472,9 @@ func (j job) run(wd, exe, dxcPath, xcrunPath string, generateExpected, generateS
 --------------------------------------------------------------------------------
 %s`,
 				expected, out, diff)
-		*j.result = status{code: fail, err: err}
+			return status{code: fail, err: err}
-		return
 		}
+	}()
 }
 
 // loadExpectedFile loads the expected output file for the test file at 'path'