tools/test-all.sh: Reimplement in golang

Makes future development easier.

New features:
* A more compact and cleaner results view
* Concurrent testing, much quicker across multiple cores
* Supports comparing output against an expected file, including a text diff of differences. Also has a flag for updating the expected outputs
* Advanced file-globbing support, including scanning for files in subdirectories
* Skip lists are now no longer hidden away in the tool, but defined as a SKIP header in the *.expected.* file

Change-Id: I4fac80bb084a720ec9a307b4acf9f73792973a1d
Reviewed-on: https://dawn-review.googlesource.com/c/tint/+/50903
Commit-Queue: Ben Clayton <bclayton@google.com>
Reviewed-by: David Neto <dneto@google.com>

tools/src/cmd/test-runner/main.go

@@ -0,0 +1,401 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// test-runner runs tint against a number of test shaders checking for expected behavior
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"unicode/utf8"
+
+	"dawn.googlesource.com/tint/tools/src/fileutils"
+	"dawn.googlesource.com/tint/tools/src/glob"
+	"github.com/fatih/color"
+	"github.com/sergi/go-diff/diffmatchpatch"
+)
+
+type outputFormat string
+
+const (
+	wgsl   = outputFormat("wgsl")
+	spvasm = outputFormat("spvasm")
+	msl    = outputFormat("msl")
+	hlsl   = outputFormat("hlsl")
+)
+
+func main() {
+	if err := run(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func showUsage() {
+	fmt.Println(`
+test-runner runs tint against a number of test shaders checking for expected behavior
+
+usage:
+  test-runner [flags...] <executable> [<directory>]
+
+  <executable> the path to the tint executable
+  <directory>  the root directory of the test files
+
+optional flags:`)
+	flag.PrintDefaults()
+	fmt.Println(``)
+	os.Exit(1)
+}
+
+func run() error {
+	var formatList, filter string
+	generateExpected := false
+	flag.StringVar(&formatList, "format", "all", "comma separated list of formats to emit. Possible values are: all, wgsl, spvasm, msl, hlsl")
+	flag.StringVar(&filter, "filter", "**.wgsl, **.spvasm, **.spv", "comma separated list of glob patterns for test files")
+	flag.BoolVar(&generateExpected, "generate-expected", false, "create or update all expected outputs")
+	flag.Usage = showUsage
+	flag.Parse()
+
+	args := flag.Args()
+	if len(args) == 0 {
+		showUsage()
+	}
+
+	// executable path is the first argument
+	exe, args := args[0], args[1:]
+
+	// (optional) target directory is the second argument
+	dir := "."
+	if len(args) > 0 {
+		dir, args = args[0], args[1:]
+	}
+
+	// Check the executable can be found and actually is executable
+	if !fileutils.IsExe(exe) {
+		return fmt.Errorf("'%s' not found or is not executable", exe)
+	}
+
+	// Split the --filter flag up by ',', trimming any whitespace at the start and end
+	globIncludes := strings.Split(filter, ",")
+	for i, s := range globIncludes {
+		globIncludes[i] = `"` + strings.TrimSpace(s) + `"`
+	}
+
+	// Glob the files to test
+	files, err := glob.Scan(dir, glob.MustParseConfig(`{
+		"paths": [
+			{
+				"include": [ `+strings.Join(globIncludes, ",")+` ]
+			},
+			{
+				"exclude": [
+					"**.expected.wgsl",
+					"**.expected.spvasm",
+					"**.expected.msl",
+					"**.expected.hlsl"
+				]
+			}
+		]
+	}`))
+	if err != nil {
+		return fmt.Errorf("Failed to glob files: %w", err)
+	}
+
+	// Ensure the files are sorted (globbing should do this, but why not)
+	sort.Strings(files)
+
+	// Parse --format into a list of outputFormat
+	formats := []outputFormat{}
+	if formatList == "all" {
+		formats = []outputFormat{wgsl, spvasm, msl, hlsl}
+	} else {
+		for _, f := range strings.Split(formatList, ",") {
+			switch strings.TrimSpace(f) {
+			case "wgsl":
+				formats = append(formats, wgsl)
+			case "spvasm":
+				formats = append(formats, spvasm)
+			case "msl":
+				formats = append(formats, msl)
+			case "hlsl":
+				formats = append(formats, hlsl)
+			default:
+				return fmt.Errorf("unknown format '%s'", f)
+			}
+		}
+	}
+
+	// Structures to hold the results of the tests
+	type statusCode string
+	const (
+		fail statusCode = "FAIL"
+		pass statusCode = "PASS"
+		skip statusCode = "SKIP"
+	)
+	type status struct {
+		code statusCode
+		err  error
+	}
+	type result map[outputFormat]status
+	results := make([]result, len(files))
+
+	// In parallel...
+	wg := sync.WaitGroup{}
+	wg.Add(len(files))
+	for i, file := range files { // For each test file...
+		i, file := i, filepath.Join(dir, file)
+		go func() {
+			defer wg.Done()
+			r := result{}
+			for _, format := range formats { // For each output format...
+
+				// Is there an expected output?
+				expected := loadExpectedFile(file, format)
+				if strings.HasPrefix(expected, "SKIP") { // Special SKIP token
+					r[format] = status{code: skip}
+					continue
+				}
+
+				// Invoke the compiler...
+				var err error
+				if ok, out := invoke(exe, file, "--format", string(format), "--dawn-validation"); ok {
+					if generateExpected {
+						// If --generate-expected was passed, write out the output
+						err = saveExpectedFile(file, format, out)
+					} else if expected != "" && expected != out {
+						// Expected output did not match
+						dmp := diffmatchpatch.New()
+						diff := dmp.DiffPrettyText(dmp.DiffMain(expected, out, true))
+						err = fmt.Errorf(`Output was not as expected
+
+--------------------------------------------------------------------------------
+-- Expected:                                                                  --
+--------------------------------------------------------------------------------
+%s
+
+--------------------------------------------------------------------------------
+-- Got:                                                                       --
+--------------------------------------------------------------------------------
+%s
+
+--------------------------------------------------------------------------------
+-- Diff:                                                                      --
+--------------------------------------------------------------------------------
+%s`,
+							expected, out, diff)
+					}
+				} else {
+					// Compiler returned a non-zero exit code
+					err = fmt.Errorf("%s", out)
+				}
+
+				if err != nil {
+					r[format] = status{code: fail, err: err}
+				} else {
+					r[format] = status{code: pass}
+				}
+			}
+			results[i] = r
+		}()
+	}
+	wg.Wait()
+
+	// At this point all the tests have been run
+	// Time to print the outputs
+
+	// Start by printing the error message for any file x format combinations
+	// that failed...
+	for i, file := range files {
+		results := results[i]
+		for _, format := range formats {
+			if err := results[format].err; err != nil {
+				color.Set(color.FgBlue)
+				fmt.Printf("%s ", file)
+				color.Set(color.FgCyan)
+				fmt.Printf("%s ", format)
+				color.Set(color.FgRed)
+				fmt.Println("FAIL")
+				color.Unset()
+				fmt.Println(indent(err.Error(), 4))
+			}
+		}
+	}
+
+	// Now print the table of file x format
+	numTests, numPass, numSkip, numFail := 0, 0, 0, 0
+	filenameFmt := columnFormat(maxStringLen(files), false)
+
+	fmt.Println()
+	fmt.Printf(filenameFmt, "")
+	fmt.Printf(" ┃ ")
+	for _, format := range formats {
+		color.Set(color.FgCyan)
+		fmt.Printf(columnFormat(formatWidth(format), false), format)
+		color.Unset()
+		fmt.Printf(" │ ")
+	}
+	fmt.Println()
+	fmt.Printf(strings.Repeat("━", maxStringLen(files)))
+	fmt.Printf("━╋━")
+	for _, format := range formats {
+		fmt.Printf(strings.Repeat("━", formatWidth(format)))
+		fmt.Printf("━│━")
+	}
+	fmt.Println()
+
+	for i, file := range files {
+		results := results[i]
+
+		color.Set(color.FgBlue)
+		fmt.Printf(filenameFmt, file)
+		color.Unset()
+		fmt.Printf(" ┃ ")
+		for _, format := range formats {
+			formatFmt := columnFormat(formatWidth(format), true)
+			result := results[format]
+			numTests++
+			switch result.code {
+			case pass:
+				color.Set(color.FgGreen)
+				fmt.Printf(formatFmt, "PASS")
+				numPass++
+			case fail:
+				color.Set(color.FgRed)
+				fmt.Printf(formatFmt, "FAIL")
+				numFail++
+			case skip:
+				color.Set(color.FgYellow)
+				fmt.Printf(formatFmt, "SKIP")
+				numSkip++
+			default:
+				fmt.Printf(formatFmt, result.code)
+			}
+			color.Unset()
+			fmt.Printf(" │ ")
+		}
+		fmt.Println()
+	}
+	fmt.Println()
+
+	fmt.Printf("%d tests run", numTests)
+	if numPass > 0 {
+		fmt.Printf(", ")
+		color.Set(color.FgGreen)
+		fmt.Printf("%d tests pass", numPass)
+		color.Unset()
+	} else {
+		fmt.Printf(", %d tests pass", numPass)
+	}
+	if numSkip > 0 {
+		fmt.Printf(", ")
+		color.Set(color.FgYellow)
+		fmt.Printf("%d tests skipped", numSkip)
+		color.Unset()
+	} else {
+		fmt.Printf(", %d tests skipped", numSkip)
+	}
+	if numFail > 0 {
+		fmt.Printf(", ")
+		color.Set(color.FgRed)
+		fmt.Printf("%d tests failed", numFail)
+		color.Unset()
+	} else {
+		fmt.Printf(", %d tests failed", numFail)
+	}
+	fmt.Println()
+	fmt.Println()
+
+	return nil
+}
+
+// loadExpectedFile loads the expected output file for the test file at 'path'
+// and the output format 'format'. If the file does not exist, or cannot be
+// read, then an empty string is returned.
+func loadExpectedFile(path string, format outputFormat) string {
+	content, err := ioutil.ReadFile(expectedFilePath(path, format))
+	if err != nil {
+		return ""
+	}
+	return string(content)
+}
+
+// saveExpectedFile writes the expected output file for the test file at 'path'
+// and the output format 'format', with the content 'content'.
+func saveExpectedFile(path string, format outputFormat, content string) error {
+	return ioutil.WriteFile(expectedFilePath(path, format), []byte(content), 0666)
+}
+
+// expectedFilePath returns the expected output file path for the test file at
+// 'path' and the output format 'format'.
+func expectedFilePath(path string, format outputFormat) string {
+	return path + ".expected." + string(format)
+}
+
+// indent returns the string 's' indented with 'n' whitespace characters
+func indent(s string, n int) string {
+	tab := strings.Repeat(" ", n)
+	return tab + strings.ReplaceAll(s, "\n", "\n"+tab)
+}
+
+// columnFormat returns the printf format string to sprint a string with the
+// width of 'i' runes.
+func columnFormat(i int, alignLeft bool) string {
+	if alignLeft {
+		return "%-" + strconv.Itoa(i) + "s"
+	}
+	return "%" + strconv.Itoa(i) + "s"
+}
+
+// maxStringLen returns the maximum number of runes found in all the strings in
+// 'l'
+func maxStringLen(l []string) int {
+	max := 0
+	for _, s := range l {
+		if c := utf8.RuneCountInString(s); c > max {
+			max = c
+		}
+	}
+	return max
+}
+
+// formatWidth returns the width in runes for the outputFormat column 'b'
+func formatWidth(b outputFormat) int {
+	c := utf8.RuneCountInString(string(b))
+	if c > 4 {
+		return c
+	}
+	return 4
+}
+
+// invoke runs the executable 'exe' with the provided arguments.
+func invoke(exe string, args ...string) (ok bool, output string) {
+	cmd := exec.Command(exe, args...)
+	out, err := cmd.CombinedOutput()
+	str := string(out)
+	if err != nil {
+		if str != "" {
+			return false, str
+		}
+		return false, err.Error()
+	}
+	return true, str
+}