dawn-cmake/tools/src/cmd/test-runner/main.go

// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// test-runner runs tint against a number of test shaders checking for expected behavior
package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"
	"unicode/utf8"

	"dawn.googlesource.com/tint/tools/src/fileutils"
	"dawn.googlesource.com/tint/tools/src/glob"
	"github.com/fatih/color"
	"github.com/sergi/go-diff/diffmatchpatch"
)

type outputFormat string

const (
	wgsl   = outputFormat("wgsl")
	spvasm = outputFormat("spvasm")
	msl    = outputFormat("msl")
	hlsl   = outputFormat("hlsl")
)

func main() {
	if err := run(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}

func showUsage() {
	fmt.Println(`
test-runner runs tint against a number of test shaders checking for expected behavior

usage:
  test-runner [flags...] <executable> [<directory>]

  <executable> the path to the tint executable
  <directory>  the root directory of the test files

optional flags:`)
	flag.PrintDefaults()
	fmt.Println(``)
	os.Exit(1)
}

func run() error {
	var formatList, filter string
	generateExpected := false
	flag.StringVar(&formatList, "format", "all", "comma separated list of formats to emit. Possible values are: all, wgsl, spvasm, msl, hlsl")
	flag.StringVar(&filter, "filter", "**.wgsl, **.spvasm, **.spv", "comma separated list of glob patterns for test files")
	flag.BoolVar(&generateExpected, "generate-expected", false, "create or update all expected outputs")
	flag.Usage = showUsage
	flag.Parse()

	args := flag.Args()
	if len(args) == 0 {
		showUsage()
	}

	// executable path is the first argument
	exe, args := args[0], args[1:]

	// (optional) target directory is the second argument
	dir := "."
	if len(args) > 0 {
		dir, args = args[0], args[1:]
	}

	// Check the executable can be found and actually is executable
	if !fileutils.IsExe(exe) {
		return fmt.Errorf("'%s' not found or is not executable", exe)
	}

	// Split the --filter flag up by ',', trimming any whitespace at the start and end
	globIncludes := strings.Split(filter, ",")
	for i, s := range globIncludes {
		globIncludes[i] = `"` + strings.TrimSpace(s) + `"`
	}

	// Glob the files to test
	files, err := glob.Scan(dir, glob.MustParseConfig(`{
		"paths": [
			{
				"include": [ `+strings.Join(globIncludes, ",")+` ]
			},
			{
				"exclude": [
					"**.expected.wgsl",
					"**.expected.spvasm",
					"**.expected.msl",
					"**.expected.hlsl"
				]
			}
		]
	}`))
	if err != nil {
		return fmt.Errorf("Failed to glob files: %w", err)
	}

	// Ensure the files are sorted (globbing should do this, but why not)
	sort.Strings(files)

	// Parse --format into a list of outputFormat
	formats := []outputFormat{}
	if formatList == "all" {
		formats = []outputFormat{wgsl, spvasm, msl, hlsl}
	} else {
		for _, f := range strings.Split(formatList, ",") {
			switch strings.TrimSpace(f) {
			case "wgsl":
				formats = append(formats, wgsl)
			case "spvasm":
				formats = append(formats, spvasm)
			case "msl":
				formats = append(formats, msl)
			case "hlsl":
				formats = append(formats, hlsl)
			default:
				return fmt.Errorf("unknown format '%s'", f)
			}
		}
	}

	// Structures to hold the results of the tests
	type statusCode string
	const (
		fail statusCode = "FAIL"
		pass statusCode = "PASS"
		skip statusCode = "SKIP"
	)
	type status struct {
		code statusCode
		err  error
	}
	type result map[outputFormat]status
	results := make([]result, len(files))

	// In parallel...
	wg := sync.WaitGroup{}
	wg.Add(len(files))
	for i, file := range files { // For each test file...
		i, file := i, filepath.Join(dir, file)
		go func() {
			defer wg.Done()
			r := result{}
			for _, format := range formats { // For each output format...

				// Is there an expected output?
				expected := loadExpectedFile(file, format)
				if strings.HasPrefix(expected, "SKIP") { // Special SKIP token
					r[format] = status{code: skip}
					continue
				}

				// Invoke the compiler...
				var err error
				if ok, out := invoke(exe, file, "--format", string(format), "--dawn-validation"); ok {
					if generateExpected {
						// If --generate-expected was passed, write out the output
						err = saveExpectedFile(file, format, out)
					} else if expected != "" && expected != out {
						// Expected output did not match
						dmp := diffmatchpatch.New()
						diff := dmp.DiffPrettyText(dmp.DiffMain(expected, out, true))
						err = fmt.Errorf(`Output was not as expected

--------------------------------------------------------------------------------
-- Expected:                                                                  --
--------------------------------------------------------------------------------
%s

--------------------------------------------------------------------------------
-- Got:                                                                       --
--------------------------------------------------------------------------------
%s

--------------------------------------------------------------------------------
-- Diff:                                                                      --
--------------------------------------------------------------------------------
%s`,
							expected, out, diff)
					}
				} else {
					// Compiler returned a non-zero exit code
					err = fmt.Errorf("%s", out)
				}

				if err != nil {
					r[format] = status{code: fail, err: err}
				} else {
					r[format] = status{code: pass}
				}
			}
			results[i] = r
		}()
	}
	wg.Wait()

	// At this point all the tests have been run
	// Time to print the outputs

	// Start by printing the error message for any file x format combinations
	// that failed...
	for i, file := range files {
		results := results[i]
		for _, format := range formats {
			if err := results[format].err; err != nil {
				color.Set(color.FgBlue)
				fmt.Printf("%s ", file)
				color.Set(color.FgCyan)
				fmt.Printf("%s ", format)
				color.Set(color.FgRed)
				fmt.Println("FAIL")
				color.Unset()
				fmt.Println(indent(err.Error(), 4))
			}
		}
	}

	// Now print the table of file x format
	numTests, numPass, numSkip, numFail := 0, 0, 0, 0
	filenameFmt := columnFormat(maxStringLen(files), false)

	fmt.Println()
	fmt.Printf(filenameFmt, "")
	fmt.Printf(" ┃ ")
	for _, format := range formats {
		color.Set(color.FgCyan)
		fmt.Printf(columnFormat(formatWidth(format), false), format)
		color.Unset()
		fmt.Printf(" │ ")
	}
	fmt.Println()
	fmt.Printf(strings.Repeat("━", maxStringLen(files)))
	fmt.Printf("━╋━")
	for _, format := range formats {
		fmt.Printf(strings.Repeat("━", formatWidth(format)))
		fmt.Printf("━│━")
	}
	fmt.Println()

	for i, file := range files {
		results := results[i]

		color.Set(color.FgBlue)
		fmt.Printf(filenameFmt, file)
		color.Unset()
		fmt.Printf(" ┃ ")
		for _, format := range formats {
			formatFmt := columnFormat(formatWidth(format), true)
			result := results[format]
			numTests++
			switch result.code {
			case pass:
				color.Set(color.FgGreen)
				fmt.Printf(formatFmt, "PASS")
				numPass++
			case fail:
				color.Set(color.FgRed)
				fmt.Printf(formatFmt, "FAIL")
				numFail++
			case skip:
				color.Set(color.FgYellow)
				fmt.Printf(formatFmt, "SKIP")
				numSkip++
			default:
				fmt.Printf(formatFmt, result.code)
			}
			color.Unset()
			fmt.Printf(" │ ")
		}
		fmt.Println()
	}
	fmt.Println()

	fmt.Printf("%d tests run", numTests)
	if numPass > 0 {
		fmt.Printf(", ")
		color.Set(color.FgGreen)
		fmt.Printf("%d tests pass", numPass)
		color.Unset()
	} else {
		fmt.Printf(", %d tests pass", numPass)
	}
	if numSkip > 0 {
		fmt.Printf(", ")
		color.Set(color.FgYellow)
		fmt.Printf("%d tests skipped", numSkip)
		color.Unset()
	} else {
		fmt.Printf(", %d tests skipped", numSkip)
	}
	if numFail > 0 {
		fmt.Printf(", ")
		color.Set(color.FgRed)
		fmt.Printf("%d tests failed", numFail)
		color.Unset()
	} else {
		fmt.Printf(", %d tests failed", numFail)
	}
	fmt.Println()
	fmt.Println()

	return nil
}

// loadExpectedFile loads the expected output file for the test file at 'path'
// and the output format 'format'. If the file does not exist, or cannot be
// read, then an empty string is returned.
func loadExpectedFile(path string, format outputFormat) string {
	content, err := ioutil.ReadFile(expectedFilePath(path, format))
	if err != nil {
		return ""
	}
	return string(content)
}

// saveExpectedFile writes the expected output file for the test file at 'path'
// and the output format 'format', with the content 'content'.
func saveExpectedFile(path string, format outputFormat, content string) error {
	return ioutil.WriteFile(expectedFilePath(path, format), []byte(content), 0666)
}

// expectedFilePath returns the expected output file path for the test file at
// 'path' and the output format 'format'.
func expectedFilePath(path string, format outputFormat) string {
	return path + ".expected." + string(format)
}

// indent returns the string 's' indented with 'n' whitespace characters
func indent(s string, n int) string {
	tab := strings.Repeat(" ", n)
	return tab + strings.ReplaceAll(s, "\n", "\n"+tab)
}

// columnFormat returns the printf format string to sprint a string with the
// width of 'i' runes.
func columnFormat(i int, alignLeft bool) string {
	if alignLeft {
		return "%-" + strconv.Itoa(i) + "s"
	}
	return "%" + strconv.Itoa(i) + "s"
}

// maxStringLen returns the maximum number of runes found in all the strings in
// 'l'
func maxStringLen(l []string) int {
	max := 0
	for _, s := range l {
		if c := utf8.RuneCountInString(s); c > max {
			max = c
		}
	}
	return max
}

// formatWidth returns the width in runes for the outputFormat column 'b'
func formatWidth(b outputFormat) int {
	c := utf8.RuneCountInString(string(b))
	if c > 4 {
		return c
	}
	return 4
}

// invoke runs the executable 'exe' with the provided arguments.
func invoke(exe string, args ...string) (ok bool, output string) {
	cmd := exec.Command(exe, args...)
	out, err := cmd.CombinedOutput()
	str := string(out)
	if err != nil {
		if str != "" {
			return false, str
		}
		return false, err.Error()
	}
	return true, str
}