tools/test-all.sh: Reimplement in golang

Makes future development easier.

New features:
* A more compact and cleaner results view
* Concurrent testing, much quicker across multiple cores
* Supports comparing output against an expected file, including a text diff of differences. Also has a flag for updating the expected outputs
* Advanced file-globbing support, including scanning for files in subdirectories
* Skip lists are now no longer hidden away in the tool, but defined as a SKIP header in the *.expected.* file

Change-Id: I4fac80bb084a720ec9a307b4acf9f73792973a1d
Reviewed-on: https://dawn-review.googlesource.com/c/tint/+/50903
Commit-Queue: Ben Clayton <bclayton@google.com>
Reviewed-by: David Neto <dneto@google.com>

tools/src/cmd/test-runner/main.go

@@ -0,0 +1,401 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// test-runner runs tint against a number of test shaders checking for expected behavior
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"unicode/utf8"
+
+	"dawn.googlesource.com/tint/tools/src/fileutils"
+	"dawn.googlesource.com/tint/tools/src/glob"
+	"github.com/fatih/color"
+	"github.com/sergi/go-diff/diffmatchpatch"
+)
+
+type outputFormat string
+
+const (
+	wgsl   = outputFormat("wgsl")
+	spvasm = outputFormat("spvasm")
+	msl    = outputFormat("msl")
+	hlsl   = outputFormat("hlsl")
+)
+
+func main() {
+	if err := run(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func showUsage() {
+	fmt.Println(`
+test-runner runs tint against a number of test shaders checking for expected behavior
+
+usage:
+  test-runner [flags...] <executable> [<directory>]
+
+  <executable> the path to the tint executable
+  <directory>  the root directory of the test files
+
+optional flags:`)
+	flag.PrintDefaults()
+	fmt.Println(``)
+	os.Exit(1)
+}
+
+func run() error {
+	var formatList, filter string
+	generateExpected := false
+	flag.StringVar(&formatList, "format", "all", "comma separated list of formats to emit. Possible values are: all, wgsl, spvasm, msl, hlsl")
+	flag.StringVar(&filter, "filter", "**.wgsl, **.spvasm, **.spv", "comma separated list of glob patterns for test files")
+	flag.BoolVar(&generateExpected, "generate-expected", false, "create or update all expected outputs")
+	flag.Usage = showUsage
+	flag.Parse()
+
+	args := flag.Args()
+	if len(args) == 0 {
+		showUsage()
+	}
+
+	// executable path is the first argument
+	exe, args := args[0], args[1:]
+
+	// (optional) target directory is the second argument
+	dir := "."
+	if len(args) > 0 {
+		dir, args = args[0], args[1:]
+	}
+
+	// Check the executable can be found and actually is executable
+	if !fileutils.IsExe(exe) {
+		return fmt.Errorf("'%s' not found or is not executable", exe)
+	}
+
+	// Split the --filter flag up by ',', trimming any whitespace at the start and end
+	globIncludes := strings.Split(filter, ",")
+	for i, s := range globIncludes {
+		globIncludes[i] = `"` + strings.TrimSpace(s) + `"`
+	}
+
+	// Glob the files to test
+	files, err := glob.Scan(dir, glob.MustParseConfig(`{
+		"paths": [
+			{
+				"include": [ `+strings.Join(globIncludes, ",")+` ]
+			},
+			{
+				"exclude": [
+					"**.expected.wgsl",
+					"**.expected.spvasm",
+					"**.expected.msl",
+					"**.expected.hlsl"
+				]
+			}
+		]
+	}`))
+	if err != nil {
+		return fmt.Errorf("Failed to glob files: %w", err)
+	}
+
+	// Ensure the files are sorted (globbing should do this, but why not)
+	sort.Strings(files)
+
+	// Parse --format into a list of outputFormat
+	formats := []outputFormat{}
+	if formatList == "all" {
+		formats = []outputFormat{wgsl, spvasm, msl, hlsl}
+	} else {
+		for _, f := range strings.Split(formatList, ",") {
+			switch strings.TrimSpace(f) {
+			case "wgsl":
+				formats = append(formats, wgsl)
+			case "spvasm":
+				formats = append(formats, spvasm)
+			case "msl":
+				formats = append(formats, msl)
+			case "hlsl":
+				formats = append(formats, hlsl)
+			default:
+				return fmt.Errorf("unknown format '%s'", f)
+			}
+		}
+	}
+
+	// Structures to hold the results of the tests
+	type statusCode string
+	const (
+		fail statusCode = "FAIL"
+		pass statusCode = "PASS"
+		skip statusCode = "SKIP"
+	)
+	type status struct {
+		code statusCode
+		err  error
+	}
+	type result map[outputFormat]status
+	results := make([]result, len(files))
+
+	// In parallel...
+	wg := sync.WaitGroup{}
+	wg.Add(len(files))
+	for i, file := range files { // For each test file...
+		i, file := i, filepath.Join(dir, file)
+		go func() {
+			defer wg.Done()
+			r := result{}
+			for _, format := range formats { // For each output format...
+
+				// Is there an expected output?
+				expected := loadExpectedFile(file, format)
+				if strings.HasPrefix(expected, "SKIP") { // Special SKIP token
+					r[format] = status{code: skip}
+					continue
+				}
+
+				// Invoke the compiler...
+				var err error
+				if ok, out := invoke(exe, file, "--format", string(format), "--dawn-validation"); ok {
+					if generateExpected {
+						// If --generate-expected was passed, write out the output
+						err = saveExpectedFile(file, format, out)
+					} else if expected != "" && expected != out {
+						// Expected output did not match
+						dmp := diffmatchpatch.New()
+						diff := dmp.DiffPrettyText(dmp.DiffMain(expected, out, true))
+						err = fmt.Errorf(`Output was not as expected
+
+--------------------------------------------------------------------------------
+-- Expected:                                                                  --
+--------------------------------------------------------------------------------
+%s
+
+--------------------------------------------------------------------------------
+-- Got:                                                                       --
+--------------------------------------------------------------------------------
+%s
+
+--------------------------------------------------------------------------------
+-- Diff:                                                                      --
+--------------------------------------------------------------------------------
+%s`,
+							expected, out, diff)
+					}
+				} else {
+					// Compiler returned a non-zero exit code
+					err = fmt.Errorf("%s", out)
+				}
+
+				if err != nil {
+					r[format] = status{code: fail, err: err}
+				} else {
+					r[format] = status{code: pass}
+				}
+			}
+			results[i] = r
+		}()
+	}
+	wg.Wait()
+
+	// At this point all the tests have been run
+	// Time to print the outputs
+
+	// Start by printing the error message for any file x format combinations
+	// that failed...
+	for i, file := range files {
+		results := results[i]
+		for _, format := range formats {
+			if err := results[format].err; err != nil {
+				color.Set(color.FgBlue)
+				fmt.Printf("%s ", file)
+				color.Set(color.FgCyan)
+				fmt.Printf("%s ", format)
+				color.Set(color.FgRed)
+				fmt.Println("FAIL")
+				color.Unset()
+				fmt.Println(indent(err.Error(), 4))
+			}
+		}
+	}
+
+	// Now print the table of file x format
+	numTests, numPass, numSkip, numFail := 0, 0, 0, 0
+	filenameFmt := columnFormat(maxStringLen(files), false)
+
+	fmt.Println()
+	fmt.Printf(filenameFmt, "")
+	fmt.Printf(" ┃ ")
+	for _, format := range formats {
+		color.Set(color.FgCyan)
+		fmt.Printf(columnFormat(formatWidth(format), false), format)
+		color.Unset()
+		fmt.Printf(" │ ")
+	}
+	fmt.Println()
+	fmt.Printf(strings.Repeat("━", maxStringLen(files)))
+	fmt.Printf("━╋━")
+	for _, format := range formats {
+		fmt.Printf(strings.Repeat("━", formatWidth(format)))
+		fmt.Printf("━│━")
+	}
+	fmt.Println()
+
+	for i, file := range files {
+		results := results[i]
+
+		color.Set(color.FgBlue)
+		fmt.Printf(filenameFmt, file)
+		color.Unset()
+		fmt.Printf(" ┃ ")
+		for _, format := range formats {
+			formatFmt := columnFormat(formatWidth(format), true)
+			result := results[format]
+			numTests++
+			switch result.code {
+			case pass:
+				color.Set(color.FgGreen)
+				fmt.Printf(formatFmt, "PASS")
+				numPass++
+			case fail:
+				color.Set(color.FgRed)
+				fmt.Printf(formatFmt, "FAIL")
+				numFail++
+			case skip:
+				color.Set(color.FgYellow)
+				fmt.Printf(formatFmt, "SKIP")
+				numSkip++
+			default:
+				fmt.Printf(formatFmt, result.code)
+			}
+			color.Unset()
+			fmt.Printf(" │ ")
+		}
+		fmt.Println()
+	}
+	fmt.Println()
+
+	fmt.Printf("%d tests run", numTests)
+	if numPass > 0 {
+		fmt.Printf(", ")
+		color.Set(color.FgGreen)
+		fmt.Printf("%d tests pass", numPass)
+		color.Unset()
+	} else {
+		fmt.Printf(", %d tests pass", numPass)
+	}
+	if numSkip > 0 {
+		fmt.Printf(", ")
+		color.Set(color.FgYellow)
+		fmt.Printf("%d tests skipped", numSkip)
+		color.Unset()
+	} else {
+		fmt.Printf(", %d tests skipped", numSkip)
+	}
+	if numFail > 0 {
+		fmt.Printf(", ")
+		color.Set(color.FgRed)
+		fmt.Printf("%d tests failed", numFail)
+		color.Unset()
+	} else {
+		fmt.Printf(", %d tests failed", numFail)
+	}
+	fmt.Println()
+	fmt.Println()
+
+	return nil
+}
+
+// loadExpectedFile loads the expected output file for the test file at 'path'
+// and the output format 'format'. If the file does not exist, or cannot be
+// read, then an empty string is returned.
+func loadExpectedFile(path string, format outputFormat) string {
+	content, err := ioutil.ReadFile(expectedFilePath(path, format))
+	if err != nil {
+		return ""
+	}
+	return string(content)
+}
+
+// saveExpectedFile writes the expected output file for the test file at 'path'
+// and the output format 'format', with the content 'content'.
+func saveExpectedFile(path string, format outputFormat, content string) error {
+	return ioutil.WriteFile(expectedFilePath(path, format), []byte(content), 0666)
+}
+
+// expectedFilePath returns the expected output file path for the test file at
+// 'path' and the output format 'format'.
+func expectedFilePath(path string, format outputFormat) string {
+	return path + ".expected." + string(format)
+}
+
+// indent returns the string 's' indented with 'n' whitespace characters
+func indent(s string, n int) string {
+	tab := strings.Repeat(" ", n)
+	return tab + strings.ReplaceAll(s, "\n", "\n"+tab)
+}
+
+// columnFormat returns the printf format string to sprint a string with the
+// width of 'i' runes.
+func columnFormat(i int, alignLeft bool) string {
+	if alignLeft {
+		return "%-" + strconv.Itoa(i) + "s"
+	}
+	return "%" + strconv.Itoa(i) + "s"
+}
+
+// maxStringLen returns the maximum number of runes found in all the strings in
+// 'l'
+func maxStringLen(l []string) int {
+	max := 0
+	for _, s := range l {
+		if c := utf8.RuneCountInString(s); c > max {
+			max = c
+		}
+	}
+	return max
+}
+
+// formatWidth returns the width in runes for the outputFormat column 'b'
+func formatWidth(b outputFormat) int {
+	c := utf8.RuneCountInString(string(b))
+	if c > 4 {
+		return c
+	}
+	return 4
+}
+
+// invoke runs the executable 'exe' with the provided arguments.
+func invoke(exe string, args ...string) (ok bool, output string) {
+	cmd := exec.Command(exe, args...)
+	out, err := cmd.CombinedOutput()
+	str := string(out)
+	if err != nil {
+		if str != "" {
+			return false, str
+		}
+		return false, err.Error()
+	}
+	return true, str
+}

tools/src/fileutils/fileutils.go

@@ -0,0 +1,29 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package fileutils contains utility functions for files
+package fileutils
+
+import (
+	"os"
+)
+
+// IsExe returns true if the file at path is an executable
+func IsExe(path string) bool {
+	s, err := os.Stat(path)
+	if err != nil {
+		return false
+	}
+	return s.Mode()&0100 != 0
+}

tools/src/glob/glob.go

@@ -22,6 +22,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"dawn.googlesource.com/tint/tools/src/match"
 )
@@ -92,7 +93,12 @@ func LoadConfig(path string) (Config, error) {
 	if err != nil {
 		return Config{}, err
 	}
-	d := json.NewDecoder(bytes.NewReader(cfgBody))
+	return ParseConfig(string(cfgBody))
+}
+
+// ParseConfig parses the config from a JSON string.
+func ParseConfig(config string) (Config, error) {
+	d := json.NewDecoder(strings.NewReader(config))
 	cfg := Config{}
 	if err := d.Decode(&cfg); err != nil {
 		return Config{}, err
@@ -100,6 +106,17 @@ func LoadConfig(path string) (Config, error) {
 	return cfg, nil
 }
 
+// MustParseConfig parses the config from a JSON string, panicing if the config
+// does not parse
+func MustParseConfig(config string) Config {
+	d := json.NewDecoder(strings.NewReader(config))
+	cfg := Config{}
+	if err := d.Decode(&cfg); err != nil {
+		panic(fmt.Errorf("Failed to parse config: %w\nConfig:\n%v", err, config))
+	}
+	return cfg
+}
+
 // rule is a search path predicate.
 // root is the project relative path.
 // cond is the value to return if the rule doesn't either include or exclude.

tools/src/go.mod

@@ -2,4 +2,7 @@ module dawn.googlesource.com/tint/tools/src
 
 go 1.16
 
-require github.com/sergi/go-diff v1.2.0
+require (
+	github.com/fatih/color v1.10.0
+	github.com/sergi/go-diff v1.2.0
+)

tools/src/go.sum

@@ -1,9 +1,15 @@
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
+github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
+github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
@@ -11,6 +17,9 @@ github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNX
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

tools/test-runner

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Copyright 2021 The Tint Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e # Fail on any error.
+
+if [ ! -x "$(which go)" ] ; then
+    echo "error: go needs to be on \$PATH to use $0"
+    exit 1
+fi
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )"
+ROOT_DIR="$( cd "${SCRIPT_DIR}/.." >/dev/null 2>&1 && pwd )"
+BINARY="${SCRIPT_DIR}/bin/test-runner"
+
+# Rebuild the binary.
+# Note, go caches build artifacts, so this is quick for repeat calls
+pushd "${SCRIPT_DIR}/src/cmd/test-runner" > /dev/null
+    go build -o "${BINARY}" main.go
+popd > /dev/null
+
+"${BINARY}" "$@"