tools: Shuffle go code into an idiomatic tree

main packages usually go under a `cmd` directory.
Hoist utility packages to the root `src` directroy so they can be shared.

Change-Id: I0c221f6cd39980f5c202c030cd5134d775533efa
Reviewed-on: https://dawn-review.googlesource.com/c/tint/+/50901
Commit-Queue: Ben Clayton <bclayton@google.com>
Reviewed-by: David Neto <dneto@google.com>

tools/src/cmd/fix-tests/main.go

@@ -0,0 +1,329 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// fix-tests is a tool to update tests with new expected output.
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"dawn.googlesource.com/tint/tools/src/substr"
+)
+
+func main() {
+	if err := run(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func showUsage() {
+	fmt.Println(`
+fix-tests is a tool to update tests with new expected output.
+
+fix-tests performs string matching and heuristics to fix up expected results of
+tests that use EXPECT_EQ(a, b) and EXPECT_THAT(a, HasSubstr(b))
+
+WARNING: Always thoroughly check the generated output for mistakes.
+This may produce incorrect output
+
+Usage:
+  fix-tests <executable>
+
+  executable         - the path to the test executable to run.`)
+	os.Exit(1)
+}
+
+func run() error {
+	flag.Parse()
+	args := flag.Args()
+	if len(args) < 1 {
+		showUsage()
+	}
+
+	exe := args[0]          // The path to the test executable
+	wd := filepath.Dir(exe) // The directory holding the test exe
+
+	// Create a temporary directory to hold the 'test-results.json' file
+	tmpDir, err := ioutil.TempDir("", "fix-tests")
+	if err != nil {
+		return err
+	}
+	if err := os.MkdirAll(tmpDir, 0666); err != nil {
+		return fmt.Errorf("Failed to create temporary directory: %w", err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	// Full path to the 'test-results.json' in the temporary directory
+	testResultsPath := filepath.Join(tmpDir, "test-results.json")
+
+	// Run the tests
+	switch err := exec.Command(exe, "--gtest_output=json:"+testResultsPath).Run().(type) {
+	default:
+		return err
+	case nil:
+		fmt.Println("All tests passed")
+	case *exec.ExitError:
+	}
+
+	// Read the 'test-results.json' file
+	testResultsFile, err := os.Open(testResultsPath)
+	if err != nil {
+		return err
+	}
+
+	var testResults Results
+	if err := json.NewDecoder(testResultsFile).Decode(&testResults); err != nil {
+		return err
+	}
+
+	// For each failing test...
+	seen := map[string]bool{}
+	numFixed, numFailed := 0, 0
+	for _, group := range testResults.Groups {
+		for _, suite := range group.Testsuites {
+			for _, failure := range suite.Failures {
+				// .. attempt to fix the problem
+				test := testName(group, suite)
+				if seen[test] {
+					continue
+				}
+				seen[test] = true
+
+				if err := processFailure(test, wd, failure.Failure); err != nil {
+					fmt.Println(fmt.Errorf("%v: %w", test, err))
+					numFailed++
+				} else {
+					numFixed++
+				}
+			}
+		}
+	}
+
+	fmt.Println()
+
+	if numFailed > 0 {
+		fmt.Println(numFailed, "tests could not be fixed")
+	}
+	if numFixed > 0 {
+		fmt.Println(numFixed, "tests fixed")
+	}
+	return nil
+}
+
+func testName(group TestsuiteGroup, suite Testsuite) string {
+	groupParts := strings.Split(group.Name, "/")
+	suiteParts := strings.Split(suite.Name, "/")
+	return groupParts[len(groupParts)-1] + "." + suiteParts[0]
+}
+
+var (
+	// Regular expression to match a test declaration
+	reTests = regexp.MustCompile(`TEST(?:_[FP])?\([ \n]*(\w+),[ \n]*(\w+)\)`)
+	// Regular expression to match a `EXPECT_EQ(a, b)` failure for strings
+	reExpectEq = regexp.MustCompile(`([./\\a-z_-]*):(\d+).*\nExpected equality of these values:\n(?:.|\n)*?(?:Which is: |  )"((?:.|\n)*?[^\\])"\n(?:.|\n)*?(?:Which is: |  )"((?:.|\n)*?[^\\])"`)
+	// Regular expression to match a `EXPECT_THAT(a, HasSubstr(b))` failure for strings
+	reExpectHasSubstr = regexp.MustCompile(`([./\\a-z_-]*):(\d+).*\nValue of: .*\nExpected: has substring "((?:.|\n)*?[^\\])"\n  Actual: "((?:.|\n)*?[^\\])"`)
+)
+
+func processFailure(test, wd, failure string) error {
+	// Start by un-escaping newlines in the failure message
+	failure = strings.ReplaceAll(failure, "\\n", "\n")
+	// Matched regex strings will also need to be un-escaped, but do this after
+	// the match, as unescaped quotes may upset the regex patterns
+	unescape := func(s string) string {
+		return strings.ReplaceAll(s, `\"`, `"`)
+	}
+	escape := func(s string) string {
+		s = strings.ReplaceAll(s, "\n", `\n`)
+		s = strings.ReplaceAll(s, "\"", `\"`)
+		return s
+	}
+
+	// Look for a EXPECT_EQ failure pattern
+	var file string
+	var fix func(testSource string) (string, error)
+	if parts := reExpectEq.FindStringSubmatch(failure); len(parts) == 5 {
+		// EXPECT_EQ(a, b)
+		a, b := unescape(parts[3]), unescape(parts[4])
+		file = parts[1]
+		fix = func(testSource string) (string, error) {
+			// We don't know if a or b is the expected, so just try flipping the string
+			// to the other form.
+			switch {
+			case strings.Contains(testSource, a):
+				testSource = strings.Replace(testSource, a, b, -1)
+			case strings.Contains(testSource, b):
+				testSource = strings.Replace(testSource, b, a, -1)
+			default:
+				// Try escaping for R"(...)" strings
+				a, b = escape(a), escape(b)
+				switch {
+				case strings.Contains(testSource, a):
+					testSource = strings.Replace(testSource, a, b, -1)
+				case strings.Contains(testSource, b):
+					testSource = strings.Replace(testSource, b, a, -1)
+				default:
+					return "", fmt.Errorf("Could not fix 'EXPECT_EQ' pattern in '%v'", file)
+				}
+			}
+			return testSource, nil
+		}
+	} else if parts := reExpectHasSubstr.FindStringSubmatch(failure); len(parts) == 5 {
+		// EXPECT_THAT(a, HasSubstr(b))
+		a, b := unescape(parts[4]), unescape(parts[3])
+		file = parts[1]
+		fix = func(testSource string) (string, error) {
+			if fix := substr.Fix(a, b); fix != "" {
+				if !strings.Contains(testSource, b) {
+					// Try escaping for R"(...)" strings
+					b, fix = escape(b), escape(fix)
+				}
+				if strings.Contains(testSource, b) {
+					testSource = strings.Replace(testSource, b, fix, -1)
+					return testSource, nil
+				}
+				return "", fmt.Errorf("Could apply fix for 'HasSubstr' pattern in '%v'", file)
+			}
+
+			return "", fmt.Errorf("Could find fix for 'HasSubstr' pattern in '%v'", file)
+		}
+	} else {
+		return fmt.Errorf("Cannot fix this type of failure")
+	}
+
+	// Get the path to the source file containing the test failure
+	sourcePath := filepath.Join(wd, file)
+
+	// Parse the source file, split into tests
+	sourceFile, err := parseSourceFile(sourcePath)
+	if err != nil {
+		return fmt.Errorf("Couldn't parse tests from file '%v': %w", file, err)
+	}
+
+	// Find the test
+	testIdx, ok := sourceFile.tests[test]
+	if !ok {
+		return fmt.Errorf("Test not found in '%v'", file)
+	}
+
+	// Grab the source for the particular test
+	testSource := sourceFile.parts[testIdx]
+
+	if testSource, err = fix(testSource); err != nil {
+		return err
+	}
+
+	// Replace the part of the source file
+	sourceFile.parts[testIdx] = testSource
+
+	// Write out the source file
+	return writeSourceFile(sourcePath, sourceFile)
+}
+
+// parseSourceFile() reads the file at path, splitting the content into chunks
+// for each TEST.
+func parseSourceFile(path string) (sourceFile, error) {
+	fileBytes, err := ioutil.ReadFile(path)
+	if err != nil {
+		return sourceFile{}, err
+	}
+	fileContent := string(fileBytes)
+
+	out := sourceFile{
+		tests: map[string]int{},
+	}
+
+	pos := 0
+	for _, span := range reTests.FindAllStringIndex(fileContent, -1) {
+		out.parts = append(out.parts, fileContent[pos:span[0]])
+		pos = span[0]
+
+		match := reTests.FindStringSubmatch(fileContent[span[0]:span[1]])
+		group := match[1]
+		suite := match[2]
+		out.tests[group+"."+suite] = len(out.parts)
+	}
+	out.parts = append(out.parts, fileContent[pos:])
+
+	return out, nil
+}
+
+// writeSourceFile() joins the chunks of the file, and writes the content out to
+// path.
+func writeSourceFile(path string, file sourceFile) error {
+	body := strings.Join(file.parts, "")
+	return ioutil.WriteFile(path, []byte(body), 0666)
+}
+
+type sourceFile struct {
+	parts []string
+	tests map[string]int // "X.Y" -> part index
+}
+
+// Results is the root JSON structure of the JSON --gtest_output file .
+type Results struct {
+	Tests     int              `json:"tests"`
+	Failures  int              `json:"failures"`
+	Disabled  int              `json:"disabled"`
+	Errors    int              `json:"errors"`
+	Timestamp string           `json:"timestamp"`
+	Time      string           `json:"time"`
+	Name      string           `json:"name"`
+	Groups    []TestsuiteGroup `json:"testsuites"`
+}
+
+// TestsuiteGroup is a group of test suites in the JSON --gtest_output file .
+type TestsuiteGroup struct {
+	Name       string      `json:"name"`
+	Tests      int         `json:"tests"`
+	Failures   int         `json:"failures"`
+	Disabled   int         `json:"disabled"`
+	Errors     int         `json:"errors"`
+	Timestamp  string      `json:"timestamp"`
+	Time       string      `json:"time"`
+	Testsuites []Testsuite `json:"testsuite"`
+}
+
+// Testsuite is a suite of tests in the JSON --gtest_output file.
+type Testsuite struct {
+	Name       string    `json:"name"`
+	ValueParam string    `json:"value_param,omitempty"`
+	Status     Status    `json:"status"`
+	Result     Result    `json:"result"`
+	Timestamp  string    `json:"timestamp"`
+	Time       string    `json:"time"`
+	Classname  string    `json:"classname"`
+	Failures   []Failure `json:"failures,omitempty"`
+}
+
+// Failure is a reported test failure in the JSON --gtest_output file.
+type Failure struct {
+	Failure string `json:"failure"`
+	Type    string `json:"type"`
+}
+
+// Status is a status code in the JSON --gtest_output file.
+type Status string
+
+// Result is a result code in the JSON --gtest_output file.
+type Result string

tools/src/cmd/run-parallel/main.go

@@ -0,0 +1,133 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// run-parallel is a tool to run an executable with the provided templated
+// arguments across all the hardware threads.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"runtime"
+	"strings"
+	"sync"
+)
+
+func main() {
+	if err := run(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func showUsage() {
+	fmt.Println(`
+run-parallel is a tool to run an executable with the provided templated
+arguments across all the hardware threads.
+
+Usage:
+  run-parallel <executable> [arguments...] -- [per-instance-value...]
+
+  executable         - the path to the executable to run.
+  arguments          - a list of arguments to pass to the executable.
+                       Any occurrance of $ will be substituted with the
+                       per-instance-value for the given invocation.
+  per-instance-value - a list of values. The executable will be invoked for each
+                       value in this list.`)
+	os.Exit(1)
+}
+
+func run() error {
+	onlyPrintFailures := flag.Bool("only-print-failures", false, "Omit output for processes that did not fail")
+	flag.Parse()
+
+	args := flag.Args()
+	if len(args) < 2 {
+		showUsage()
+	}
+	exe := args[0]
+	args = args[1:]
+
+	var perInstanceValues []string
+	for i, arg := range args {
+		if arg == "--" {
+			perInstanceValues = args[i+1:]
+			args = args[:i]
+			break
+		}
+	}
+	if perInstanceValues == nil {
+		showUsage()
+	}
+
+	taskIndices := make(chan int, 64)
+	type result struct {
+		msg    string
+		failed bool
+	}
+	results := make([]result, len(perInstanceValues))
+
+	numCPU := runtime.NumCPU()
+	wg := sync.WaitGroup{}
+	wg.Add(numCPU)
+	for i := 0; i < numCPU; i++ {
+		go func() {
+			defer wg.Done()
+			for idx := range taskIndices {
+				taskArgs := make([]string, len(args))
+				for i, arg := range args {
+					taskArgs[i] = strings.ReplaceAll(arg, "$", perInstanceValues[idx])
+				}
+				success, out := invoke(exe, taskArgs)
+				if !success || !*onlyPrintFailures {
+					results[idx] = result{out, !success}
+				}
+			}
+		}()
+	}
+
+	for i := range perInstanceValues {
+		taskIndices <- i
+	}
+	close(taskIndices)
+
+	wg.Wait()
+
+	failed := false
+	for _, result := range results {
+		if result.msg != "" {
+			fmt.Println(result.msg)
+		}
+		failed = failed || result.failed
+	}
+	if failed {
+		os.Exit(1)
+	}
+	return nil
+}
+
+func invoke(exe string, args []string) (ok bool, output string) {
+	cmd := exec.Command(exe, args...)
+	out, err := cmd.CombinedOutput()
+	str := string(out)
+	if err != nil {
+		if str != "" {
+			return false, str
+		}
+		return false, err.Error()
+	}
+	return true, str
+}

tools/src/cmd/trim-includes/config.cfg

@@ -0,0 +1,6 @@
+{
+  "paths": [
+    { "include": [ "src/**.h", "src/**.cc" ] },
+    { "exclude": [ "src/**_windows.*", "src/**_other.*" ] }
+  ]
+}

tools/src/cmd/trim-includes/main.go

@@ -0,0 +1,295 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// trim-includes is a tool to try removing unnecessary include statements from
+// all .cc and .h files in the tint project.
+//
+// trim-includes removes each #include from each file, then runs the provided
+// build script to determine whether the line was necessary. If the include is
+// required, it is restored, otherwise it is left deleted.
+// After all the #include statements have been tested, the file is
+// clang-formatted and git staged.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"strings"
+	"sync"
+	"time"
+
+	"dawn.googlesource.com/tint/tools/src/glob"
+)
+
+var (
+	// Directory to this .go file
+	toolRoot = getToolRoot()
+	// Root directory of the Tint project
+	projectRoot = getProjectRoot(toolRoot)
+
+	// Path to the build script to run after each attempting to remove each
+	// #include
+	buildScript = ""
+)
+
+func main() {
+	if err := run(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func showUsage() {
+	fmt.Println(`
+trim-includes is a tool to try removing unnecessary include statements from all
+.cc and .h files in the tint project.
+
+trim-includes removes each #include from each file, then runs the provided build
+script to determine whether the line was necessary. If the include is required,
+it is restored, otherwise it is left deleted.
+After all the #include statements have been tested, the file is clang-formatted
+and git staged.
+
+Usage:
+  trim-includes <path-to-build-script>`)
+	os.Exit(1)
+}
+
+func run() error {
+	flag.Parse()
+	args := flag.Args()
+	if len(args) < 1 {
+		showUsage()
+	}
+
+	var err error
+	buildScript, err = exec.LookPath(args[0])
+	if err != nil {
+		return err
+	}
+	buildScript, err = filepath.Abs(buildScript)
+	if err != nil {
+		return err
+	}
+
+	cfg, err := glob.LoadConfig("config.cfg")
+	if err != nil {
+		return err
+	}
+
+	fmt.Println("Checking the project builds with no changes...")
+	ok, err := tryBuild()
+	if err != nil {
+		return err
+	}
+	if !ok {
+		return fmt.Errorf("Project does not build without edits")
+	}
+
+	fmt.Println("Scanning for files...")
+	paths, err := glob.Scan(projectRoot, cfg)
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("Loading %v source files...\n", len(paths))
+	files, err := loadFiles(paths)
+	if err != nil {
+		return err
+	}
+
+	for fileIdx, file := range files {
+		fmt.Printf("[%d/%d]: %v\n", fileIdx+1, len(files), file.path)
+		includeLines := file.includesLineNumbers()
+		enabled := make(map[int]bool, len(file.lines))
+		for i := range file.lines {
+			enabled[i] = true
+		}
+		for includeIdx, line := range includeLines {
+			fmt.Printf("    [%d/%d]: %v", includeIdx+1, len(includeLines), file.lines[line])
+			enabled[line] = false
+			if err := file.save(enabled); err != nil {
+				return err
+			}
+			ok, err := tryBuild()
+			if err != nil {
+				return err
+			}
+			if ok {
+				fmt.Printf(" removed\n")
+				// Wait a bit so file timestamps get an opportunity to change.
+				// Attempting to save too soon after a successful build may
+				// result in a false-positive build.
+				time.Sleep(time.Second)
+			} else {
+				fmt.Printf(" required\n")
+				enabled[line] = true
+			}
+		}
+		if err := file.save(enabled); err != nil {
+			return err
+		}
+		if err := file.format(); err != nil {
+			return err
+		}
+		if err := file.stage(); err != nil {
+			return err
+		}
+	}
+	fmt.Println("Done")
+	return nil
+}
+
+// Attempt to build the project. Returns true on successful build, false if
+// there was a build failure.
+func tryBuild() (bool, error) {
+	cmd := exec.Command("sh", "-c", buildScript)
+	out, err := cmd.CombinedOutput()
+	switch err := err.(type) {
+	case nil:
+		return cmd.ProcessState.Success(), nil
+	case *exec.ExitError:
+		return false, nil
+	default:
+		return false, fmt.Errorf("Test failed with error: %v\n%v", err, string(out))
+	}
+}
+
+type file struct {
+	path  string
+	lines []string
+}
+
+var includeRE = regexp.MustCompile(`^\s*#include (?:\"([^"]*)\"|:?\<([^"]*)\>)`)
+
+// Returns the file path with the extension stripped
+func stripExtension(path string) string {
+	if dot := strings.IndexRune(path, '.'); dot > 0 {
+		return path[:dot]
+	}
+	return path
+}
+
+// Returns the zero-based line numbers of all #include statements in the file
+func (f *file) includesLineNumbers() []int {
+	out := []int{}
+	for i, l := range f.lines {
+		matches := includeRE.FindStringSubmatch(l)
+		if len(matches) == 0 {
+			continue
+		}
+
+		include := matches[1]
+		if include == "" {
+			include = matches[2]
+		}
+
+		if strings.HasSuffix(stripExtension(f.path), stripExtension(include)) {
+			// Don't remove #include for header of cc
+			continue
+		}
+
+		out = append(out, i)
+	}
+	return out
+}
+
+// Saves the file, omitting the lines with the zero-based line number that are
+// either not in `lines` or have a `false` value.
+func (f *file) save(lines map[int]bool) error {
+	content := []string{}
+	for i, l := range f.lines {
+		if lines[i] {
+			content = append(content, l)
+		}
+	}
+	data := []byte(strings.Join(content, "\n"))
+	return ioutil.WriteFile(f.path, data, 0666)
+}
+
+// Runs clang-format on the file
+func (f *file) format() error {
+	err := exec.Command("clang-format", "-i", f.path).Run()
+	if err != nil {
+		return fmt.Errorf("Couldn't format file '%v': %w", f.path, err)
+	}
+	return nil
+}
+
+// Runs git add on the file
+func (f *file) stage() error {
+	err := exec.Command("git", "-C", projectRoot, "add", f.path).Run()
+	if err != nil {
+		return fmt.Errorf("Couldn't stage file '%v': %w", f.path, err)
+	}
+	return nil
+}
+
+// Loads all the files with the given file paths, splitting their content into
+// into lines.
+func loadFiles(paths []string) ([]file, error) {
+	wg := sync.WaitGroup{}
+	wg.Add(len(paths))
+	files := make([]file, len(paths))
+	errs := make([]error, len(paths))
+	for i, path := range paths {
+		i, path := i, filepath.Join(projectRoot, path)
+		go func() {
+			defer wg.Done()
+			body, err := ioutil.ReadFile(path)
+			if err != nil {
+				errs[i] = fmt.Errorf("Failed to open %v: %w", path, err)
+			} else {
+				content := string(body)
+				lines := strings.Split(content, "\n")
+				files[i] = file{path: path, lines: lines}
+			}
+		}()
+	}
+	wg.Wait()
+	for _, err := range errs {
+		if err != nil {
+			return nil, err
+		}
+	}
+	return files, nil
+}
+
+// Returns the path to the directory holding this .go file
+func getToolRoot() string {
+	_, filename, _, ok := runtime.Caller(0)
+	if !ok {
+		panic("No caller information")
+	}
+	mainPath, err := filepath.Abs(filename)
+	if err != nil {
+		panic(err)
+	}
+	return filepath.Dir(mainPath)
+}
+
+// Returns the path to the project root
+func getProjectRoot(toolRoot string) string {
+	root, err := filepath.Abs(filepath.Join(toolRoot, "../.."))
+	if err != nil {
+		panic(err)
+	}
+	return root
+}