test-runner: Don't print PASS rows by default

If a full test row PASSes, then omit this unless the --verbose flag is used.
Add a summary below the table for PASS, SKIP and FAIL for each column.

Change-Id: Ie88003696c59614cf8e5efa3c80dca37a8e4d3f0
Reviewed-on: https://dawn-review.googlesource.com/c/tint/+/56380
Auto-Submit: Ben Clayton <bclayton@google.com>
Commit-Queue: Antonio Maiorano <amaiorano@google.com>
Kokoro: Kokoro <noreply+kokoro@google.com>
Reviewed-by: Antonio Maiorano <amaiorano@google.com>

kokoro/linux/docker.sh

@@ -128,7 +128,7 @@ if [ "$BUILD_SYSTEM" == "cmake" ]; then
 
     status "Testing test/test-all.sh"
     show_cmds
-        ${SRC_DIR}/test/test-all.sh "${BUILD_DIR}/tint"
+        ${SRC_DIR}/test/test-all.sh "${BUILD_DIR}/tint" --verbose
     hide_cmds
 
     status "Checking _other.cc files also build"

kokoro/windows/build.bat

@@ -127,7 +127,7 @@ call :status "Testing test/test-all.sh"
 cd /d %SRC_DIR% || goto :error
 set PATH=C:\Program Files\Metal Developer Tools\macos\bin;%PATH%
 where metal.exe
-git bash -- ./test/test-all.sh ../tint-build/%BUILD_TYPE%/tint.exe
+git bash -- ./test/test-all.sh ../tint-build/%BUILD_TYPE%/tint.exe --verbose
 @echo off
 
 call :status "Done"

tools/src/cmd/test-runner/main.go

@@ -24,7 +24,6 @@ import (
 	"path/filepath"
 	"runtime"
 	"sort"
-	"strconv"
 	"strings"
 	"unicode/utf8"
 
@@ -69,11 +68,12 @@ optional flags:`)
 func run() error {
 	var formatList, filter, dxcPath, xcrunPath string
 	numCPU := runtime.NumCPU()
-	generateExpected, generateSkip := false, false
+	verbose, generateExpected, generateSkip := false, false, false
 	flag.StringVar(&formatList, "format", "all", "comma separated list of formats to emit. Possible values are: all, wgsl, spvasm, msl, hlsl")
 	flag.StringVar(&filter, "filter", "**.wgsl, **.spvasm, **.spv", "comma separated list of glob patterns for test files")
 	flag.StringVar(&dxcPath, "dxc", "", "path to DXC executable for validating HLSL output")
 	flag.StringVar(&xcrunPath, "xcrun", "", "path to xcrun executable for validating MSL output")
+	flag.BoolVar(&verbose, "verbose", false, "print all run tests, including rows that all pass")
 	flag.BoolVar(&generateExpected, "generate-expected", false, "create or update all expected outputs")
 	flag.BoolVar(&generateSkip, "generate-skip", false, "create or update all expected outputs that fail with SKIP")
 	flag.IntVar(&numCPU, "j", numCPU, "maximum number of concurrent threads to run tests")
@@ -158,9 +158,9 @@ func run() error {
 		}
 	}
 
-	default_msl_exe := "xcrun"
+	defaultMSLExe := "xcrun"
 	if runtime.GOOS == "windows" {
-		default_msl_exe = "metal.exe"
+		defaultMSLExe = "metal.exe"
 	}
 
 	// If explicit verification compilers have been specified, check they exist.
@@ -171,7 +171,7 @@ func run() error {
 		path *string
 	}{
 		{"dxc", "hlsl", &dxcPath},
-		{default_msl_exe, "msl", &xcrunPath},
+		{defaultMSLExe, "msl", &xcrunPath},
 	} {
 		if *tool.path == "" {
 			p, err := exec.LookPath(tool.name)
@@ -241,40 +241,63 @@ func run() error {
 		err    error
 	}
 
-	// Print the table of file x format
+	type stats struct {
-	failures := []failure{}
+		numTests, numPass, numSkip, numFail int
-	numTests, numPass, numSkip, numFail := 0, 0, 0, 0
+	}
-	filenameFmt := columnFormat(maxStringLen(files), false)
 
-	fmt.Println()
+	// Statistics per output format
-	fmt.Printf(filenameFmt, "")
+	statsByFmt := map[outputFormat]*stats{}
+	for _, format := range formats {
+		statsByFmt[format] = &stats{}
+
+	}
+
+	// Print the table of file x format and gather per-format stats
+	failures := []failure{}
+	filenameColumnWidth := maxStringLen(files)
+
+	red := color.New(color.FgRed)
+	green := color.New(color.FgGreen)
+	yellow := color.New(color.FgYellow)
+	cyan := color.New(color.FgCyan)
+
+	printFormatsHeader := func() {
+		fmt.Printf(strings.Repeat(" ", filenameColumnWidth))
 		fmt.Printf(" ┃ ")
 		for _, format := range formats {
-		color.Set(color.FgCyan)
+			cyan.Printf(alignCenter(format, formatWidth(format)))
-		fmt.Printf(columnFormat(formatWidth(format), false), format)
-		color.Unset()
 			fmt.Printf(" │ ")
 		}
 		fmt.Println()
+	}
+	printHorizontalLine := func() {
 		fmt.Printf(strings.Repeat("━", maxStringLen(files)))
 		fmt.Printf("━╋━")
 		for _, format := range formats {
 			fmt.Printf(strings.Repeat("━", formatWidth(format)))
-		fmt.Printf("━│━")
+			fmt.Printf("━┿━")
 		}
 		fmt.Println()
+	}
+
+	fmt.Println()
+
+	printFormatsHeader()
+	printHorizontalLine()
 
 	for i, file := range files {
 		results := results[i]
 
-		color.Set(color.FgBlue)
+		row := &strings.Builder{}
-		fmt.Printf(filenameFmt, file)
+		rowAllPassed := true
-		color.Unset()
+
-		fmt.Printf(" ┃ ")
+		fmt.Fprintf(row, alignRight(file, filenameColumnWidth))
+		fmt.Fprintf(row, " ┃ ")
 		for _, format := range formats {
-			formatFmt := columnFormat(formatWidth(format), true)
+			columnWidth := formatWidth(format)
 			result := <-results[format]
-			numTests++
+			stats := statsByFmt[format]
+			stats.numTests++
 			if err := result.err; err != nil {
 				failures = append(failures, failure{
 					file: file, format: format, err: err,
@@ -282,25 +305,72 @@ func run() error {
 			}
 			switch result.code {
 			case pass:
-				color.Set(color.FgGreen)
+				green.Fprintf(row, alignCenter("PASS", columnWidth))
-				fmt.Printf(formatFmt, "PASS")
+				stats.numPass++
-				numPass++
 			case fail:
-				color.Set(color.FgRed)
+				red.Fprintf(row, alignCenter("FAIL", columnWidth))
-				fmt.Printf(formatFmt, "FAIL")
+				rowAllPassed = false
-				numFail++
+				stats.numFail++
 			case skip:
-				color.Set(color.FgYellow)
+				yellow.Fprintf(row, alignCenter("SKIP", columnWidth))
-				fmt.Printf(formatFmt, "SKIP")
+				rowAllPassed = false
-				numSkip++
+				stats.numSkip++
 			default:
-				fmt.Printf(formatFmt, result.code)
+				fmt.Fprintf(row, alignCenter(result.code, columnWidth))
+				rowAllPassed = false
+			}
+			fmt.Fprintf(row, " │ ")
+		}
+
+		if verbose || !rowAllPassed {
+			fmt.Println(row)
+		}
+	}
+
+	printHorizontalLine()
+	printFormatsHeader()
+	printHorizontalLine()
+	printStat := func(col *color.Color, name string, num func(*stats) int) {
+		row := &strings.Builder{}
+		anyNonZero := false
+		for _, format := range formats {
+			columnWidth := formatWidth(format)
+			count := num(statsByFmt[format])
+			if count > 0 {
+				col.Fprintf(row, alignLeft(count, columnWidth))
+				anyNonZero = true
+			} else {
+				fmt.Fprintf(row, alignLeft(count, columnWidth))
+			}
+			fmt.Fprintf(row, " │ ")
+		}
+
+		if !anyNonZero {
+			return
+		}
+		col.Printf(alignRight(name, filenameColumnWidth))
+		fmt.Printf(" ┃ ")
+		fmt.Println(row)
+
+		col.Printf(strings.Repeat(" ", filenameColumnWidth))
+		fmt.Printf(" ┃ ")
+		for _, format := range formats {
+			columnWidth := formatWidth(format)
+			stats := statsByFmt[format]
+			count := num(stats)
+			percent := percentage(count, stats.numTests)
+			if count > 0 {
+				col.Print(alignRight(percent, columnWidth))
+			} else {
+				fmt.Print(alignRight(percent, columnWidth))
 			}
-			color.Unset()
 			fmt.Printf(" │ ")
 		}
 		fmt.Println()
 	}
+	printStat(green, "PASS", func(s *stats) int { return s.numPass })
+	printStat(yellow, "SKIP", func(s *stats) int { return s.numSkip })
+	printStat(red, "FAIL", func(s *stats) int { return s.numFail })
 	fmt.Println()
 
 	for _, f := range failures {
@@ -317,35 +387,44 @@ func run() error {
 		fmt.Println()
 	}
 
-	fmt.Printf("%d tests run", numTests)
+	allStats := stats{}
-	if numPass > 0 {
+	for _, format := range formats {
+		stats := statsByFmt[format]
+		allStats.numTests += stats.numTests
+		allStats.numPass += stats.numPass
+		allStats.numSkip += stats.numSkip
+		allStats.numFail += stats.numFail
+	}
+
+	fmt.Printf("%d tests run", allStats.numTests)
+	if allStats.numPass > 0 {
 		fmt.Printf(", ")
 		color.Set(color.FgGreen)
-		fmt.Printf("%d tests pass", numPass)
+		fmt.Printf("%d tests pass", allStats.numPass)
 		color.Unset()
 	} else {
-		fmt.Printf(", %d tests pass", numPass)
+		fmt.Printf(", %d tests pass", allStats.numPass)
 	}
-	if numSkip > 0 {
+	if allStats.numSkip > 0 {
 		fmt.Printf(", ")
 		color.Set(color.FgYellow)
-		fmt.Printf("%d tests skipped", numSkip)
+		fmt.Printf("%d tests skipped", allStats.numSkip)
 		color.Unset()
 	} else {
-		fmt.Printf(", %d tests skipped", numSkip)
+		fmt.Printf(", %d tests skipped", allStats.numSkip)
 	}
-	if numFail > 0 {
+	if allStats.numFail > 0 {
 		fmt.Printf(", ")
 		color.Set(color.FgRed)
-		fmt.Printf("%d tests failed", numFail)
+		fmt.Printf("%d tests failed", allStats.numFail)
 		color.Unset()
 	} else {
-		fmt.Printf(", %d tests failed", numFail)
+		fmt.Printf(", %d tests failed", allStats.numFail)
 	}
 	fmt.Println()
 	fmt.Println()
 
-	if numFail > 0 {
+	if allStats.numFail > 0 {
 		os.Exit(1)
 	}
 
@@ -508,13 +587,28 @@ func indent(s string, n int) string {
 	return tab + strings.ReplaceAll(s, "\n", "\n"+tab)
 }
 
-// columnFormat returns the printf format string to sprint a string with the
+// alignLeft returns the string of 'val' padded so that it is aligned left in
-// width of 'i' runes.
+// a column of the given width
-func columnFormat(i int, alignLeft bool) string {
+func alignLeft(val interface{}, width int) string {
-	if alignLeft {
+	s := fmt.Sprint(val)
-		return "%-" + strconv.Itoa(i) + "s"
+	padding := width - utf8.RuneCountInString(s)
-	}
+	return s + strings.Repeat(" ", padding)
-	return "%" + strconv.Itoa(i) + "s"
+}
+
+// alignCenter returns the string of 'val' padded so that it is centered in a
+// column of the given width.
+func alignCenter(val interface{}, width int) string {
+	s := fmt.Sprint(val)
+	padding := width - utf8.RuneCountInString(s)
+	return strings.Repeat(" ", padding/2) + s + strings.Repeat(" ", (padding+1)/2)
+}
+
+// alignRight returns the string of 'val' padded so that it is aligned right in
+// a column of the given width
+func alignRight(val interface{}, width int) string {
+	s := fmt.Sprint(val)
+	padding := width - utf8.RuneCountInString(s)
+	return strings.Repeat(" ", padding) + s
 }
 
 // maxStringLen returns the maximum number of runes found in all the strings in
@@ -531,11 +625,21 @@ func maxStringLen(l []string) int {
 
 // formatWidth returns the width in runes for the outputFormat column 'b'
 func formatWidth(b outputFormat) int {
+	const min = 6
 	c := utf8.RuneCountInString(string(b))
-	if c > 4 {
+	if c < min {
-		return c
+		return min
 	}
-	return 4
+	return c
+}
+
+// percentage returns the percentage of n out of total as a string
+func percentage(n, total int) string {
+	if total == 0 {
+		return "-"
+	}
+	f := float64(n) / float64(total)
+	return fmt.Sprintf("%.1f%c", f*100.0, '%')
 }
 
 // invoke runs the executable 'exe' with the provided arguments.