Merge remote-tracking branch 'tint/main' into HEAD

Integrates Tint repo into Dawn

KIs:
- Building docs for Tint is turned off, because it fails due to lack
  of annotations in Dawn source files.
- Dawn CQ needs to be updated to run Tint specific tests
- Significant post-merge cleanup needed

R=bclayton,cwallez
BUG=dawn:1339

Change-Id: I6c9714a0030934edd6c51f3cac4684dcd59d1ea3

tools/src/bench/bench.go

@@ -0,0 +1,417 @@
+// Copyright 2022 The Tint Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package bench provides types and methods for parsing Google benchmark results.
+package bench
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+	"unicode/utf8"
+)
+
+// Run holds all the benchmark results for a run, along with the context
+// information for the run.
+type Run struct {
+	Benchmarks []Benchmark
+	Context    *Context
+}
+
+// Context provides information about the environment used to perform the
+// benchmark.
+type Context struct {
+	Date              time.Time
+	HostName          string
+	Executable        string
+	NumCPUs           int
+	MhzPerCPU         int
+	CPUScalingEnabled bool
+	Caches            []ContextCache
+	LoadAvg           []float32
+	LibraryBuildType  string
+}
+
+// ContextCache holds information about one of the system caches.
+type ContextCache struct {
+	Type       string
+	Level      int
+	Size       int
+	NumSharing int
+}
+
+// Benchmark holds the results of a single benchmark test.
+type Benchmark struct {
+	Name          string
+	Duration      time.Duration
+	AggregateType AggregateType
+}
+
+// AggregateType is an enumerator of benchmark aggregate types.
+type AggregateType string
+
+// Enumerator values of AggregateType
+const (
+	NonAggregate AggregateType = "NonAggregate"
+	Mean         AggregateType = "mean"
+	Median       AggregateType = "median"
+	Stddev       AggregateType = "stddev"
+)
+
+// Parse parses the benchmark results from the string s.
+// Parse will handle the json and 'console' formats.
+func Parse(s string) (Run, error) {
+	type Parser = func(s string) (Run, error)
+	for _, parser := range []Parser{parseConsole, parseJSON} {
+		r, err := parser(s)
+		switch err {
+		case nil:
+			return r, nil
+		case errWrongFormat:
+		default:
+			return Run{}, err
+		}
+	}
+
+	return Run{}, errors.New("Unrecognised file format")
+}
+
+var errWrongFormat = errors.New("Wrong format")
+var consoleLineRE = regexp.MustCompile(`([\w/:]+)\s+([0-9]+(?:.[0-9]+)?) ns\s+[0-9]+(?:.[0-9]+) ns\s+([0-9]+)`)
+
+func parseConsole(s string) (Run, error) {
+	blocks := strings.Split(s, "------------------------------------------------------------------------------------------")
+	if len(blocks) != 3 {
+		return Run{}, errWrongFormat
+	}
+
+	lines := strings.Split(blocks[2], "\n")
+	b := make([]Benchmark, 0, len(lines))
+
+	for _, line := range lines {
+		if len(line) == 0 {
+			continue
+		}
+		matches := consoleLineRE.FindStringSubmatch(line)
+		if len(matches) != 4 {
+			return Run{}, fmt.Errorf("Unable to parse the line:\n" + line)
+		}
+		ns, err := strconv.ParseFloat(matches[2], 64)
+		if err != nil {
+			return Run{}, fmt.Errorf("Unable to parse the duration: " + matches[2])
+		}
+
+		b = append(b, Benchmark{
+			Name:     trimAggregateSuffix(matches[1]),
+			Duration: time.Nanosecond * time.Duration(ns),
+		})
+	}
+	return Run{Benchmarks: b}, nil
+}
+
+func parseJSON(s string) (Run, error) {
+	type Data struct {
+		Context struct {
+			Date              time.Time `json:"date"`
+			HostName          string    `json:"host_name"`
+			Executable        string    `json:"executable"`
+			NumCPUs           int       `json:"num_cpus"`
+			MhzPerCPU         int       `json:"mhz_per_cpu"`
+			CPUScalingEnabled bool      `json:"cpu_scaling_enabled"`
+			LoadAvg           []float32 `json:"load_avg"`
+			LibraryBuildType  string    `json:"library_build_type"`
+			Caches            []struct {
+				Type       string `json:"type"`
+				Level      int    `json:"level"`
+				Size       int    `json:"size"`
+				NumSharing int    `json:"num_sharing"`
+			} `json:"caches"`
+		} `json:"context"`
+		Benchmarks []struct {
+			Name          string        `json:"name"`
+			Time          float64       `json:"real_time"`
+			AggregateType AggregateType `json:"aggregate_name"`
+		} `json:"benchmarks"`
+	}
+	data := Data{}
+	d := json.NewDecoder(strings.NewReader(s))
+	if err := d.Decode(&data); err != nil {
+		return Run{}, err
+	}
+
+	out := Run{
+		Benchmarks: make([]Benchmark, len(data.Benchmarks)),
+		Context: &Context{
+			Date:              data.Context.Date,
+			HostName:          data.Context.HostName,
+			Executable:        data.Context.Executable,
+			NumCPUs:           data.Context.NumCPUs,
+			MhzPerCPU:         data.Context.MhzPerCPU,
+			CPUScalingEnabled: data.Context.CPUScalingEnabled,
+			LoadAvg:           data.Context.LoadAvg,
+			LibraryBuildType:  data.Context.LibraryBuildType,
+			Caches:            make([]ContextCache, len(data.Context.Caches)),
+		},
+	}
+	for i, c := range data.Context.Caches {
+		out.Context.Caches[i] = ContextCache{
+			Type:       c.Type,
+			Level:      c.Level,
+			Size:       c.Size,
+			NumSharing: c.NumSharing,
+		}
+	}
+	for i, b := range data.Benchmarks {
+		out.Benchmarks[i] = Benchmark{
+			Name:          trimAggregateSuffix(b.Name),
+			Duration:      time.Nanosecond * time.Duration(int64(b.Time)),
+			AggregateType: b.AggregateType,
+		}
+	}
+
+	return out, nil
+}
+
+// Diff describes the difference between two benchmarks
+type Diff struct {
+	TestName           string
+	Delta              time.Duration // Δ (A → B)
+	PercentChangeAB    float64       // % (A → B)
+	PercentChangeBA    float64       // % (A → B)
+	MultiplierChangeAB float64       // × (A → B)
+	MultiplierChangeBA float64       // × (A → B)
+	TimeA              time.Duration // A
+	TimeB              time.Duration // B
+}
+
+// Diffs is a list of Diff
+type Diffs []Diff
+
+// DiffFormat describes how a list of diffs should be formatted
+type DiffFormat struct {
+	TestName           bool
+	Delta              bool
+	PercentChangeAB    bool
+	PercentChangeBA    bool
+	MultiplierChangeAB bool
+	MultiplierChangeBA bool
+	TimeA              bool
+	TimeB              bool
+}
+
+func (diffs Diffs) Format(f DiffFormat) string {
+	if len(diffs) == 0 {
+		return "<no changes>"
+	}
+
+	type row []string
+
+	header := row{}
+	if f.TestName {
+		header = append(header, "Test name")
+	}
+	if f.Delta {
+		header = append(header, "Δ (A → B)")
+	}
+	if f.PercentChangeAB {
+		header = append(header, "% (A → B)")
+	}
+	if f.PercentChangeBA {
+		header = append(header, "% (B → A)")
+	}
+	if f.MultiplierChangeAB {
+		header = append(header, "× (A → B)")
+	}
+	if f.MultiplierChangeBA {
+		header = append(header, "× (B → A)")
+	}
+	if f.TimeA {
+		header = append(header, "A")
+	}
+	if f.TimeB {
+		header = append(header, "B")
+	}
+	if len(header) == 0 {
+		return ""
+	}
+
+	columns := []row{}
+	for _, d := range diffs {
+		r := make(row, 0, len(header))
+		if f.TestName {
+			r = append(r, d.TestName)
+		}
+		if f.Delta {
+			r = append(r, fmt.Sprintf("%v", d.Delta))
+		}
+		if f.PercentChangeAB {
+			r = append(r, fmt.Sprintf("%+2.1f%%", d.PercentChangeAB))
+		}
+		if f.PercentChangeBA {
+			r = append(r, fmt.Sprintf("%+2.1f%%", d.PercentChangeBA))
+		}
+		if f.MultiplierChangeAB {
+			r = append(r, fmt.Sprintf("%+.4f", d.MultiplierChangeAB))
+		}
+		if f.MultiplierChangeBA {
+			r = append(r, fmt.Sprintf("%+.4f", d.MultiplierChangeBA))
+		}
+		if f.TimeA {
+			r = append(r, fmt.Sprintf("%v", d.TimeA))
+		}
+		if f.TimeB {
+			r = append(r, fmt.Sprintf("%v", d.TimeB))
+		}
+		columns = append(columns, r)
+	}
+
+	// measure
+	widths := make([]int, len(header))
+	for i, h := range header {
+		widths[i] = utf8.RuneCountInString(h)
+	}
+	for _, row := range columns {
+		for i, cell := range row {
+			l := utf8.RuneCountInString(cell)
+			if widths[i] < l {
+				widths[i] = l
+			}
+		}
+	}
+
+	pad := func(s string, i int) string {
+		if n := i - utf8.RuneCountInString(s); n > 0 {
+			return s + strings.Repeat(" ", n)
+		}
+		return s
+	}
+
+	// Draw table
+	b := &strings.Builder{}
+
+	horizontal_bar := func() {
+		for i := range header {
+			fmt.Fprintf(b, "+%v", strings.Repeat("-", 2+widths[i]))
+		}
+		fmt.Fprintln(b, "+")
+	}
+
+	horizontal_bar()
+
+	for i, h := range header {
+		fmt.Fprintf(b, "| %v ", pad(h, widths[i]))
+	}
+	fmt.Fprintln(b, "|")
+
+	horizontal_bar()
+
+	for _, row := range columns {
+		for i, cell := range row {
+			fmt.Fprintf(b, "| %v ", pad(cell, widths[i]))
+		}
+		fmt.Fprintln(b, "|")
+	}
+
+	horizontal_bar()
+
+	return b.String()
+}
+
+// Compare returns a string describing differences in the two benchmarks
+// Absolute benchmark differences less than minDiff are omitted
+// Absolute relative differences between [1, 1+x] are omitted
+func Compare(a, b []Benchmark, minDiff time.Duration, minRelDiff float64) Diffs {
+	type times struct {
+		a time.Duration
+		b time.Duration
+	}
+	byName := map[string]times{}
+	for _, test := range a {
+		byName[test.Name] = times{a: test.Duration}
+	}
+	for _, test := range b {
+		t := byName[test.Name]
+		t.b = test.Duration
+		byName[test.Name] = t
+	}
+
+	type delta struct {
+		name       string
+		times      times
+		relDiff    float64
+		absRelDiff float64
+	}
+	deltas := []delta{}
+	for name, times := range byName {
+		if times.a == 0 || times.b == 0 {
+			continue // Assuming test was missing from a or b
+		}
+		diff := times.b - times.a
+		absDiff := diff
+		if absDiff < 0 {
+			absDiff = -absDiff
+		}
+		if absDiff < minDiff {
+			continue
+		}
+
+		relDiff := float64(times.b) / float64(times.a)
+		absRelDiff := relDiff
+		if absRelDiff < 1 {
+			absRelDiff = 1.0 / absRelDiff
+		}
+		if absRelDiff < (1.0 + minRelDiff) {
+			continue
+		}
+
+		d := delta{
+			name:       name,
+			times:      times,
+			relDiff:    relDiff,
+			absRelDiff: absRelDiff,
+		}
+		deltas = append(deltas, d)
+	}
+
+	sort.Slice(deltas, func(i, j int) bool { return deltas[j].relDiff < deltas[i].relDiff })
+
+	out := make(Diffs, len(deltas))
+
+	for i, delta := range deltas {
+		a2b := delta.times.b - delta.times.a
+		out[i] = Diff{
+			TestName:           delta.name,
+			Delta:              a2b,
+			PercentChangeAB:    100 * float64(a2b) / float64(delta.times.a),
+			PercentChangeBA:    100 * float64(-a2b) / float64(delta.times.b),
+			MultiplierChangeAB: float64(delta.times.b) / float64(delta.times.a),
+			MultiplierChangeBA: float64(delta.times.a) / float64(delta.times.b),
+			TimeA:              delta.times.a,
+			TimeB:              delta.times.b,
+		}
+	}
+	return out
+}
+
+func trimAggregateSuffix(name string) string {
+	name = strings.TrimSuffix(name, "_stddev")
+	name = strings.TrimSuffix(name, "_mean")
+	name = strings.TrimSuffix(name, "_median")
+	return name
+}

tools/src/bench/bench_test.go

@@ -0,0 +1,252 @@
+// Copyright 2022 The Tint Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bench_test
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	"dawn.googlesource.com/tint/tools/src/bench"
+)
+
+func TestParseJson(t *testing.T) {
+	json := `
+	{
+		"context": {
+		  "date": "2022-01-24T10:28:13+00:00",
+		  "host_name": "hostname",
+		  "executable": "./myexe",
+		  "num_cpus": 16,
+		  "mhz_per_cpu": 2400,
+		  "cpu_scaling_enabled": false,
+		  "caches": [
+			{
+			  "type": "Data",
+			  "level": 1,
+			  "size": 32768,
+			  "num_sharing": 2
+			},
+			{
+			  "type": "Instruction",
+			  "level": 1,
+			  "size": 32768,
+			  "num_sharing": 2
+			},
+			{
+			  "type": "Unified",
+			  "level": 2,
+			  "size": 262144,
+			  "num_sharing": 2
+			},
+			{
+			  "type": "Unified",
+			  "level": 3,
+			  "size": 16777216,
+			  "num_sharing": 16
+			}
+		  ],
+		  "load_avg": [2.60938,2.59863,2.55566],
+		  "library_build_type": "release"
+		},
+		"benchmarks": [
+		  {
+			"name": "MyBenchmark",
+			"family_index": 0,
+			"per_family_instance_index": 0,
+			"run_name": "MyBenchmark",
+			"run_type": "iteration",
+			"repetitions": 2,
+			"repetition_index": 0,
+			"threads": 1,
+			"iterations": 402,
+			"real_time": 1.6392272438353568e+06,
+			"cpu_time": 1.6387412935323382e+06,
+			"time_unit": "ns"
+		  },
+		  {
+			"name": "MyBenchmark",
+			"family_index": 0,
+			"per_family_instance_index": 0,
+			"run_name": "MyBenchmark",
+			"run_type": "iteration",
+			"repetitions": 2,
+			"repetition_index": 1,
+			"threads": 1,
+			"iterations": 402,
+			"real_time": 1.7143936117703272e+06,
+			"cpu_time": 1.7124004975124374e+06,
+			"time_unit": "ns"
+		  },
+		  {
+			"name": "MyBenchmark_mean",
+			"family_index": 0,
+			"per_family_instance_index": 0,
+			"run_name": "MyBenchmark",
+			"run_type": "aggregate",
+			"repetitions": 2,
+			"threads": 1,
+			"aggregate_name": "mean",
+			"iterations": 2,
+			"real_time": 1.6768104278028419e+06,
+			"cpu_time": 1.6755708955223879e+06,
+			"time_unit": "ns"
+		  },
+		  {
+			"name": "MyBenchmark_median",
+			"family_index": 0,
+			"per_family_instance_index": 0,
+			"run_name": "MyBenchmark",
+			"run_type": "aggregate",
+			"repetitions": 2,
+			"threads": 1,
+			"aggregate_name": "median",
+			"iterations": 2,
+			"real_time": 1.6768104278028419e+06,
+			"cpu_time": 1.6755708955223879e+06,
+			"time_unit": "ns"
+		  },
+		  {
+			"name": "MyBenchmark_stddev",
+			"family_index": 0,
+			"per_family_instance_index": 0,
+			"run_name": "MyBenchmark",
+			"run_type": "aggregate",
+			"repetitions": 2,
+			"threads": 1,
+			"aggregate_name": "stddev",
+			"iterations": 2,
+			"real_time": 5.3150648483981553e+04,
+			"cpu_time": 5.2084922631119407e+04,
+			"time_unit": "ns"
+		  }
+		]
+	  }
+`
+	got, err := bench.Parse(json)
+	if err != nil {
+		t.Errorf("bench.Parse() returned %v", err)
+		return
+	}
+
+	expectedDate, err := time.Parse(time.RFC1123, "Mon, 24 Jan 2022 10:28:13 GMT")
+	if err != nil {
+		t.Errorf("time.Parse() returned %v", err)
+		return
+	}
+
+	expect := bench.Run{
+		Benchmarks: []bench.Benchmark{
+			{Name: "MyBenchmark", Duration: time.Nanosecond * 1639227, AggregateType: ""},
+			{Name: "MyBenchmark", Duration: time.Nanosecond * 1714393, AggregateType: ""},
+			{Name: "MyBenchmark", Duration: time.Nanosecond * 1676810, AggregateType: "mean"},
+			{Name: "MyBenchmark", Duration: time.Nanosecond * 1676810, AggregateType: "median"},
+			{Name: "MyBenchmark", Duration: time.Nanosecond * 53150, AggregateType: "stddev"},
+		},
+		Context: &bench.Context{
+			Date:       expectedDate,
+			HostName:   "hostname",
+			Executable: "./myexe",
+			NumCPUs:    16,
+			MhzPerCPU:  2400, CPUScalingEnabled: false,
+			Caches: []bench.ContextCache{
+				{Type: "Data", Level: 1, Size: 32768, NumSharing: 2},
+				{Type: "Instruction", Level: 1, Size: 32768, NumSharing: 2},
+				{Type: "Unified", Level: 2, Size: 262144, NumSharing: 2},
+				{Type: "Unified", Level: 3, Size: 16777216, NumSharing: 16},
+			},
+			LoadAvg: []float32{2.60938, 2.59863, 2.55566}, LibraryBuildType: "release"},
+	}
+
+	expectEqual(t, "bench.Parse().Benchmarks", got.Benchmarks, expect.Benchmarks)
+	expectEqual(t, "bench.Parse().Context", got.Benchmarks, expect.Benchmarks)
+}
+
+func TestCompare(t *testing.T) {
+	a := []bench.Benchmark{
+		{Name: "MyBenchmark1", Duration: time.Nanosecond * 1714393},
+		{Name: "MyBenchmark0", Duration: time.Nanosecond * 1639227},
+		{Name: "MyBenchmark3", Duration: time.Nanosecond * 1676810},
+		{Name: "MyBenchmark4", Duration: time.Nanosecond * 53150},
+		{Name: "MyBenchmark2", Duration: time.Nanosecond * 1676810},
+	}
+	b := []bench.Benchmark{
+		{Name: "MyBenchmark1", Duration: time.Nanosecond * 56747654},
+		{Name: "MyBenchmark0", Duration: time.Nanosecond * 236246},
+		{Name: "MyBenchmark2", Duration: time.Nanosecond * 675865},
+		{Name: "MyBenchmark4", Duration: time.Nanosecond * 2352336},
+		{Name: "MyBenchmark3", Duration: time.Nanosecond * 87657868},
+	}
+
+	minDiff := time.Millisecond * 2
+	minRelDiff := 35.0
+
+	cmp := bench.Compare(a, b, minDiff, minRelDiff)
+
+	expectEqual(t, "bench.Compare().Format", cmp.Format(bench.DiffFormat{}), "")
+	expectEqual(t, "bench.Compare().Format", "\n"+cmp.Format(bench.DiffFormat{TimeA: true}), `
++-----------+
+| A         |
++-----------+
+| 1.67681ms |
+| 53.15µs   |
++-----------+
+`)
+	expectEqual(t, "bench.Compare().Format", "\n"+cmp.Format(bench.DiffFormat{TimeA: true, TimeB: true}), `
++-----------+-------------+
+| A         | B           |
++-----------+-------------+
+| 1.67681ms | 87.657868ms |
+| 53.15µs   | 2.352336ms  |
++-----------+-------------+
+`)
+	expectEqual(t, "bench.Compare().Format", "\n"+cmp.Format(bench.DiffFormat{
+		TestName:        true,
+		Delta:           true,
+		PercentChangeAB: true,
+		TimeA:           true,
+		TimeB:           true,
+	}), `
++--------------+-------------+-----------+-----------+-------------+
+| Test name    | Δ (A → B)   | % (A → B) | A         | B           |
++--------------+-------------+-----------+-----------+-------------+
+| MyBenchmark3 | 85.981058ms | +5127.7%  | 1.67681ms | 87.657868ms |
+| MyBenchmark4 | 2.299186ms  | +4325.8%  | 53.15µs   | 2.352336ms  |
++--------------+-------------+-----------+-----------+-------------+
+`)
+	expectEqual(t, "bench.Compare().Format", "\n"+cmp.Format(bench.DiffFormat{
+		TestName:           true,
+		Delta:              true,
+		PercentChangeAB:    true,
+		PercentChangeBA:    true,
+		MultiplierChangeAB: true,
+		MultiplierChangeBA: true,
+		TimeA:              true,
+		TimeB:              true,
+	}), `
++--------------+-------------+-----------+-----------+-----------+-----------+-----------+-------------+
+| Test name    | Δ (A → B)   | % (A → B) | % (B → A) | × (A → B) | × (B → A) | A         | B           |
++--------------+-------------+-----------+-----------+-----------+-----------+-----------+-------------+
+| MyBenchmark3 | 85.981058ms | +5127.7%  | -98.1%    | +52.2766  | +0.0191   | 1.67681ms | 87.657868ms |
+| MyBenchmark4 | 2.299186ms  | +4325.8%  | -97.7%    | +44.2584  | +0.0226   | 53.15µs   | 2.352336ms  |
++--------------+-------------+-----------+-----------+-----------+-----------+-----------+-------------+
+`)
+}
+
+func expectEqual(t *testing.T, desc string, got, expect interface{}) {
+	if !reflect.DeepEqual(got, expect) {
+		t.Errorf("%v was not expected:\nGot:\n%+v\nExpected:\n%+v", desc, got, expect)
+	}
+}

tools/src/cmd/benchdiff/main.go

@@ -0,0 +1,96 @@
+// Copyright 2022 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// benchdiff is a tool that compares two Google benchmark results and displays
+// sorted performance differences.
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"time"
+
+	"dawn.googlesource.com/tint/tools/src/bench"
+)
+
+var (
+	minDiff    = flag.Duration("min-diff", time.Microsecond*10, "Filter away time diffs less than this duration")
+	minRelDiff = flag.Float64("min-rel-diff", 0.01, "Filter away absolute relative diffs between [1, 1+x]")
+)
+
+func main() {
+	flag.ErrHelp = errors.New("benchdiff is a tool to compare two benchmark results")
+	flag.Parse()
+	flag.Usage = func() {
+		fmt.Fprintln(os.Stderr, "benchdiff <benchmark-a> <benchmark-b>")
+		flag.PrintDefaults()
+	}
+
+	args := flag.Args()
+	if len(args) < 2 {
+		flag.Usage()
+		os.Exit(1)
+	}
+
+	pathA, pathB := args[0], args[1]
+
+	if err := run(pathA, pathB); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(-1)
+	}
+}
+
+func run(pathA, pathB string) error {
+	fileA, err := ioutil.ReadFile(pathA)
+	if err != nil {
+		return err
+	}
+	benchA, err := bench.Parse(string(fileA))
+	if err != nil {
+		return err
+	}
+
+	fileB, err := ioutil.ReadFile(pathB)
+	if err != nil {
+		return err
+	}
+	benchB, err := bench.Parse(string(fileB))
+	if err != nil {
+		return err
+	}
+
+	cmp := bench.Compare(benchA.Benchmarks, benchB.Benchmarks, *minDiff, *minRelDiff)
+	diff := cmp.Format(bench.DiffFormat{
+		TestName:        true,
+		Delta:           true,
+		PercentChangeAB: true,
+		TimeA:           true,
+		TimeB:           true,
+	})
+
+	fmt.Println("A:", pathA, "  B:", pathB)
+	fmt.Println()
+	fmt.Println(diff)
+
+	return nil
+}
+
+func fileName(path string) string {
+	_, name := filepath.Split(path)
+	return name
+}

tools/src/cmd/builtin-gen/ast/ast.go

@@ -0,0 +1,307 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package ast defines AST nodes that are produced by the Tint intrinsic
+// definition parser
+package ast
+
+import (
+	"fmt"
+	"strings"
+
+	"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/tok"
+)
+
+// AST is the parsed syntax tree of the intrinsic definition file
+type AST struct {
+	Enums     []EnumDecl
+	Types     []TypeDecl
+	Matchers  []MatcherDecl
+	Functions []FunctionDecl
+}
+
+func (a AST) String() string {
+	sb := strings.Builder{}
+	for _, e := range a.Enums {
+		fmt.Fprintf(&sb, "%v", e)
+		fmt.Fprintln(&sb)
+	}
+	for _, p := range a.Types {
+		fmt.Fprintf(&sb, "%v", p)
+		fmt.Fprintln(&sb)
+	}
+	for _, m := range a.Matchers {
+		fmt.Fprintf(&sb, "%v", m)
+		fmt.Fprintln(&sb)
+	}
+	for _, f := range a.Functions {
+		fmt.Fprintf(&sb, "%v", f)
+		fmt.Fprintln(&sb)
+	}
+	return sb.String()
+}
+
+// EnumDecl describes an enumerator
+type EnumDecl struct {
+	Source  tok.Source
+	Name    string
+	Entries []EnumEntry
+}
+
+// Format implements the fmt.Formatter interface
+func (e EnumDecl) Format(w fmt.State, verb rune) {
+	fmt.Fprintf(w, "enum %v {\n", e.Name)
+	for _, e := range e.Entries {
+		fmt.Fprintf(w, "  %v\n", e)
+	}
+	fmt.Fprintf(w, "}\n")
+}
+
+// EnumEntry describes an entry in a enumerator
+type EnumEntry struct {
+	Source      tok.Source
+	Name        string
+	Decorations Decorations
+}
+
+// Format implements the fmt.Formatter interface
+func (e EnumEntry) Format(w fmt.State, verb rune) {
+	if len(e.Decorations) > 0 {
+		fmt.Fprintf(w, "%v %v", e.Decorations, e.Name)
+	} else {
+		fmt.Fprint(w, e.Name)
+	}
+}
+
+// MatcherDecl describes a matcher declaration
+type MatcherDecl struct {
+	Source  tok.Source
+	Name    string
+	Options MatcherOptions
+}
+
+// Format implements the fmt.Formatter interface
+func (m MatcherDecl) Format(w fmt.State, verb rune) {
+	fmt.Fprintf(w, "match %v", m.Name)
+	fmt.Fprintf(w, ": ")
+	m.Options.Format(w, verb)
+}
+
+// FunctionDecl describes a function declaration
+type FunctionDecl struct {
+	Source         tok.Source
+	Name           string
+	Decorations    Decorations
+	TemplateParams TemplateParams
+	Parameters     Parameters
+	ReturnType     *TemplatedName
+}
+
+// Format implements the fmt.Formatter interface
+func (f FunctionDecl) Format(w fmt.State, verb rune) {
+	fmt.Fprintf(w, "fn %v", f.Name)
+	f.TemplateParams.Format(w, verb)
+	f.Parameters.Format(w, verb)
+	if f.ReturnType != nil {
+		fmt.Fprintf(w, " -> ")
+		f.ReturnType.Format(w, verb)
+	}
+}
+
+// Parameters is a list of parameter
+type Parameters []Parameter
+
+// Format implements the fmt.Formatter interface
+func (l Parameters) Format(w fmt.State, verb rune) {
+	fmt.Fprintf(w, "(")
+	for i, p := range l {
+		if i > 0 {
+			fmt.Fprintf(w, ", ")
+		}
+		p.Format(w, verb)
+	}
+	fmt.Fprintf(w, ")")
+}
+
+// Parameter describes a single parameter of a function
+type Parameter struct {
+	Source tok.Source
+	Name   string // Optional
+	Type   TemplatedName
+}
+
+// Format implements the fmt.Formatter interface
+func (p Parameter) Format(w fmt.State, verb rune) {
+	if p.Name != "" {
+		fmt.Fprintf(w, "%v: ", p.Name)
+	}
+	p.Type.Format(w, verb)
+}
+
+// MatcherOptions is a list of TemplatedName
+type MatcherOptions TemplatedNames
+
+// Format implements the fmt.Formatter interface
+func (o MatcherOptions) Format(w fmt.State, verb rune) {
+	for i, mo := range o {
+		if i > 0 {
+			fmt.Fprintf(w, " | ")
+		}
+		mo.Format(w, verb)
+	}
+}
+
+// TemplatedNames is a list of TemplatedName
+// Example:
+//   a<b>, c<d, e>
+type TemplatedNames []TemplatedName
+
+// Format implements the fmt.Formatter interface
+func (l TemplatedNames) Format(w fmt.State, verb rune) {
+	for i, n := range l {
+		if i > 0 {
+			fmt.Fprintf(w, ", ")
+		}
+		n.Format(w, verb)
+	}
+}
+
+// TemplatedName is an identifier with optional templated arguments
+// Example:
+//  vec<N, T>
+type TemplatedName struct {
+	Source       tok.Source
+	Name         string
+	TemplateArgs TemplatedNames
+}
+
+// Format implements the fmt.Formatter interface
+func (t TemplatedName) Format(w fmt.State, verb rune) {
+	fmt.Fprintf(w, "%v", t.Name)
+	if len(t.TemplateArgs) > 0 {
+		fmt.Fprintf(w, "<")
+		t.TemplateArgs.Format(w, verb)
+		fmt.Fprintf(w, ">")
+	}
+}
+
+// TypeDecl describes a type declaration
+type TypeDecl struct {
+	Source         tok.Source
+	Decorations    Decorations
+	Name           string
+	TemplateParams TemplateParams
+}
+
+// Format implements the fmt.Formatter interface
+func (p TypeDecl) Format(w fmt.State, verb rune) {
+	if len(p.Decorations) > 0 {
+		p.Decorations.Format(w, verb)
+		fmt.Fprintf(w, " type %v", p.Name)
+	}
+	fmt.Fprintf(w, "type %v", p.Name)
+	p.TemplateParams.Format(w, verb)
+}
+
+// TemplateParams is a list of TemplateParam
+// Example:
+//   <A, B : TyB>
+type TemplateParams []TemplateParam
+
+// Format implements the fmt.Formatter interface
+func (p TemplateParams) Format(w fmt.State, verb rune) {
+	if len(p) > 0 {
+		fmt.Fprintf(w, "<")
+		for i, tp := range p {
+			if i > 0 {
+				fmt.Fprintf(w, ", ")
+			}
+			tp.Format(w, verb)
+		}
+		fmt.Fprintf(w, ">")
+	}
+}
+
+// TemplateParam describes a template parameter with optional type
+// Example:
+//   <Name>
+//   <Name: Type>
+type TemplateParam struct {
+	Source tok.Source
+	Name   string
+	Type   TemplatedName // Optional
+}
+
+// Format implements the fmt.Formatter interface
+func (t TemplateParam) Format(w fmt.State, verb rune) {
+	fmt.Fprintf(w, "%v", t.Name)
+	if t.Type.Name != "" {
+		fmt.Fprintf(w, " : ")
+		t.Type.Format(w, verb)
+	}
+}
+
+// Decorations is a list of Decoration
+// Example:
+//   [[a(x), b(y)]]
+type Decorations []Decoration
+
+// Format implements the fmt.Formatter interface
+func (l Decorations) Format(w fmt.State, verb rune) {
+	fmt.Fprint(w, "[[")
+	for i, d := range l {
+		if i > 0 {
+			fmt.Fprintf(w, ", ")
+		}
+		d.Format(w, verb)
+	}
+	fmt.Fprint(w, "]]")
+}
+
+// Take looks up the decoration with the given name. If the decoration is found
+// it is removed from the Decorations list and returned, otherwise nil is
+// returned and the Decorations are not altered.
+func (l *Decorations) Take(name string) *Decoration {
+	for i, d := range *l {
+		if d.Name == name {
+			*l = append((*l)[:i], (*l)[i+1:]...)
+			return &d
+		}
+	}
+	return nil
+}
+
+// Decoration describes a single decoration
+// Example:
+//   a(x)
+type Decoration struct {
+	Source tok.Source
+	Name   string
+	Values []string
+}
+
+// Format implements the fmt.Formatter interface
+func (d Decoration) Format(w fmt.State, verb rune) {
+	fmt.Fprintf(w, "%v", d.Name)
+	if len(d.Values) > 0 {
+		fmt.Fprintf(w, "(")
+		for i, v := range d.Values {
+			if i > 0 {
+				fmt.Fprint(w, ", ")
+			}
+			fmt.Fprintf(w, "%v", v)
+		}
+		fmt.Fprintf(w, ")")
+	}
+}

tools/src/cmd/builtin-gen/gen/builtin_table.go

@@ -0,0 +1,387 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gen
+
+import (
+	"fmt"
+
+	"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/sem"
+	"dawn.googlesource.com/tint/tools/src/list"
+	"dawn.googlesource.com/tint/tools/src/lut"
+)
+
+// BuiltinTable holds data specific to the intrinsic_table.inl.tmpl template
+type BuiltinTable struct {
+	// The semantic info
+	Sem *sem.Sem
+
+	// TMatchers are all the sem.OpenType, sem.Type and sem.TypeMatchers.
+	// These are all implemented by classes deriving from tint::TypeMatcher
+	TMatchers     []sem.Named
+	TMatcherIndex map[sem.Named]int // [object -> index] in TMatcher
+
+	// NMatchers are all the sem.OpenNumber and sem.EnumMatchers.
+	// These are all implemented by classes deriving from tint::NumberMatcher
+	NMatchers     []sem.Named
+	NMatcherIndex map[sem.Named]int // [object -> index] in NMatchers
+
+	MatcherIndices []int        // kMatcherIndices table content
+	OpenTypes      []OpenType   // kOpenTypes table content
+	OpenNumbers    []OpenNumber // kOpenNumbers table content
+	Parameters     []Parameter  // kParameters table content
+	Overloads      []Overload   // kOverloads table content
+	Functions      []Function   // kBuiltins table content
+}
+
+// OpenType is used to create the C++ OpenTypeInfo structure
+type OpenType struct {
+	// Name of the open type (e.g. 'T')
+	Name string
+	// Optional type matcher constraint.
+	// Either an index in Matchers::type, or -1
+	MatcherIndex int
+}
+
+// OpenNumber is used to create the C++ OpenNumberInfo structure
+type OpenNumber struct {
+	// Name of the open number (e.g. 'N')
+	Name string
+	// Optional type matcher constraint.
+	// Either an index in Matchers::type, or -1
+	MatcherIndex int
+}
+
+// Parameter is used to create the C++ ParameterInfo structure
+type Parameter struct {
+	// The parameter usage (parameter name)
+	Usage string
+
+	// Index into BuiltinTable.MatcherIndices, beginning the list of matchers
+	// required to match the parameter type. The matcher indices index
+	// into BuiltinTable::TMatchers and / or BuiltinTable::NMatchers.
+	// These indices are consumed by the matchers themselves.
+	// The first index is always a TypeMatcher.
+	MatcherIndicesOffset *int
+}
+
+// Overload is used to create the C++ OverloadInfo structure
+type Overload struct {
+	// Total number of parameters for the overload
+	NumParameters int
+	// Total number of open types for the overload
+	NumOpenTypes int
+	// Total number of open numbers for the overload
+	NumOpenNumbers int
+	// Index to the first open type in BuiltinTable.OpenTypes
+	OpenTypesOffset *int
+	// Index to the first open number in BuiltinTable.OpenNumbers
+	OpenNumbersOffset *int
+	// Index to the first parameter in BuiltinTable.Parameters
+	ParametersOffset *int
+	// Index into BuiltinTable.MatcherIndices, beginning the list of matchers
+	// required to match the return type. The matcher indices index
+	// into BuiltinTable::TMatchers and / or BuiltinTable::NMatchers.
+	// These indices are consumed by the matchers themselves.
+	// The first index is always a TypeMatcher.
+	ReturnMatcherIndicesOffset *int
+	// StageUses describes the stages an overload can be used in
+	CanBeUsedInStage sem.StageUses
+	// True if the overload is marked as deprecated
+	IsDeprecated bool
+}
+
+// Function is used to create the C++ IntrinsicInfo structure
+type Function struct {
+	OverloadDescriptions []string
+	NumOverloads         int
+	OverloadsOffset      *int
+}
+
+// Helper for building the BuiltinTable
+type BuiltinTableBuilder struct {
+	// The output of the builder
+	BuiltinTable
+
+	// Lookup tables.
+	// These are packed (compressed) once all the entries have been added.
+	lut struct {
+		matcherIndices lut.LUT
+		openTypes      lut.LUT
+		openNumbers    lut.LUT
+		parameters     lut.LUT
+		overloads      lut.LUT
+	}
+}
+
+// Helper for building a single overload
+type overloadBuilder struct {
+	*BuiltinTableBuilder
+	// Maps TemplateParam to index in openTypes
+	openTypeIndex map[sem.TemplateParam]int
+	// Maps TemplateParam to index in openNumbers
+	openNumberIndex map[sem.TemplateParam]int
+	// Open types used by the overload
+	openTypes []OpenType
+	// Open numbers used by the overload
+	openNumbers []OpenNumber
+	// All parameters declared by the overload
+	parameters []Parameter
+	// Index into BuiltinTable.MatcherIndices, beginning the list of matchers
+	// required to match the return type. The matcher indices index
+	// into BuiltinTable::TMatchers and / or BuiltinTable::NMatchers.
+	// These indices are consumed by the matchers themselves.
+	// The first index is always a TypeMatcher.
+	returnTypeMatcherIndicesOffset *int
+}
+
+// layoutMatchers assigns each of the TMatchers and NMatchers a unique index
+// in the C++ Matchers::type and Matchers::number arrays, respectively.
+func (b *BuiltinTableBuilder) layoutMatchers(s *sem.Sem) {
+	// First MaxOpenTypes of TMatchers are open types
+	b.TMatchers = make([]sem.Named, s.MaxOpenTypes)
+	for _, m := range s.Types {
+		b.TMatcherIndex[m] = len(b.TMatchers)
+		b.TMatchers = append(b.TMatchers, m)
+	}
+	for _, m := range s.TypeMatchers {
+		b.TMatcherIndex[m] = len(b.TMatchers)
+		b.TMatchers = append(b.TMatchers, m)
+	}
+
+	// First MaxOpenNumbers of NMatchers are open numbers
+	b.NMatchers = make([]sem.Named, s.MaxOpenNumbers)
+	for _, m := range s.EnumMatchers {
+		b.NMatcherIndex[m] = len(b.NMatchers)
+		b.NMatchers = append(b.NMatchers, m)
+	}
+}
+
+// buildOverload constructs an Overload for a sem.Overload
+func (b *BuiltinTableBuilder) buildOverload(o *sem.Overload) (Overload, error) {
+	ob := overloadBuilder{
+		BuiltinTableBuilder: b,
+		openTypeIndex:       map[sem.TemplateParam]int{},
+		openNumberIndex:     map[sem.TemplateParam]int{},
+	}
+
+	if err := ob.buildOpenTypes(o); err != nil {
+		return Overload{}, err
+	}
+	if err := ob.buildOpenNumbers(o); err != nil {
+		return Overload{}, err
+	}
+	if err := ob.buildParameters(o); err != nil {
+		return Overload{}, err
+	}
+	if err := ob.buildReturnType(o); err != nil {
+		return Overload{}, err
+	}
+
+	return Overload{
+		NumParameters:              len(ob.parameters),
+		NumOpenTypes:               len(ob.openTypes),
+		NumOpenNumbers:             len(ob.openNumbers),
+		OpenTypesOffset:            b.lut.openTypes.Add(ob.openTypes),
+		OpenNumbersOffset:          b.lut.openNumbers.Add(ob.openNumbers),
+		ParametersOffset:           b.lut.parameters.Add(ob.parameters),
+		ReturnMatcherIndicesOffset: ob.returnTypeMatcherIndicesOffset,
+		CanBeUsedInStage:           o.CanBeUsedInStage,
+		IsDeprecated:               o.IsDeprecated,
+	}, nil
+}
+
+// buildOpenTypes constructs the OpenTypes used by the overload, populating
+// b.openTypes
+func (b *overloadBuilder) buildOpenTypes(o *sem.Overload) error {
+	b.openTypes = make([]OpenType, len(o.OpenTypes))
+	for i, t := range o.OpenTypes {
+		b.openTypeIndex[t] = i
+		matcherIndex := -1
+		if t.Type != nil {
+			var err error
+			matcherIndex, err = b.matcherIndex(t.Type)
+			if err != nil {
+				return err
+			}
+		}
+		b.openTypes[i] = OpenType{
+			Name:         t.Name,
+			MatcherIndex: matcherIndex,
+		}
+	}
+	return nil
+}
+
+// buildOpenNumbers constructs the OpenNumbers used by the overload, populating
+// b.openNumbers
+func (b *overloadBuilder) buildOpenNumbers(o *sem.Overload) error {
+	b.openNumbers = make([]OpenNumber, len(o.OpenNumbers))
+	for i, t := range o.OpenNumbers {
+		b.openNumberIndex[t] = i
+		matcherIndex := -1
+		if e, ok := t.(*sem.TemplateEnumParam); ok && e.Matcher != nil {
+			var err error
+			matcherIndex, err = b.matcherIndex(e.Matcher)
+			if err != nil {
+				return err
+			}
+		}
+		b.openNumbers[i] = OpenNumber{
+			Name:         t.GetName(),
+			MatcherIndex: matcherIndex,
+		}
+	}
+	return nil
+}
+
+// buildParameters constructs the Parameters used by the overload, populating
+// b.parameters
+func (b *overloadBuilder) buildParameters(o *sem.Overload) error {
+	b.parameters = make([]Parameter, len(o.Parameters))
+	for i, p := range o.Parameters {
+		indices, err := b.collectMatcherIndices(p.Type)
+		if err != nil {
+			return err
+		}
+
+		b.parameters[i] = Parameter{
+			Usage:                p.Name,
+			MatcherIndicesOffset: b.lut.matcherIndices.Add(indices),
+		}
+	}
+	return nil
+}
+
+// buildParameters calculates the matcher indices required to match the
+// overload's return type (if the overload has a return value), possibly
+// populating b.returnTypeMatcherIndicesOffset
+func (b *overloadBuilder) buildReturnType(o *sem.Overload) error {
+	if o.ReturnType != nil {
+		indices, err := b.collectMatcherIndices(*o.ReturnType)
+		if err != nil {
+			return err
+		}
+		b.returnTypeMatcherIndicesOffset = b.lut.matcherIndices.Add(indices)
+	}
+	return nil
+}
+
+// matcherIndex returns the index of TMatcher or NMatcher in
+// BuiltinTable.TMatcher or BuiltinTable.NMatcher, respectively.
+func (b *overloadBuilder) matcherIndex(n sem.Named) (int, error) {
+	switch n := n.(type) {
+	case *sem.Type, *sem.TypeMatcher:
+		if i, ok := b.TMatcherIndex[n]; ok {
+			return i, nil
+		}
+		return 0, fmt.Errorf("matcherIndex missing entry for %v %T", n.GetName(), n)
+	case *sem.TemplateTypeParam:
+		if i, ok := b.openTypeIndex[n]; ok {
+			return i, nil
+		}
+		return 0, fmt.Errorf("openTypeIndex missing entry for %v %T", n.Name, n)
+	case *sem.EnumMatcher:
+		if i, ok := b.NMatcherIndex[n]; ok {
+			return i, nil
+		}
+		return 0, fmt.Errorf("matcherIndex missing entry for %v %T", n.GetName(), n)
+	case *sem.TemplateEnumParam:
+		if i, ok := b.openNumberIndex[n]; ok {
+			return i, nil
+		}
+		return 0, fmt.Errorf("openNumberIndex missing entry for %v %T", n, n)
+	case *sem.TemplateNumberParam:
+		if i, ok := b.openNumberIndex[n]; ok {
+			return i, nil
+		}
+		return 0, fmt.Errorf("openNumberIndex missing entry for %v %T", n, n)
+	default:
+		return 0, fmt.Errorf("overload.matcherIndex() does not handle %v %T", n, n)
+	}
+}
+
+// collectMatcherIndices returns the full list of matcher indices required to
+// match the fully-qualified-name. For names that have do not have templated
+// arguments, collectMatcherIndices() will return a single TMatcher index.
+// For names that do have templated arguments, collectMatcherIndices() returns
+// a list of type matcher indices, starting with the target of the fully
+// qualified name, then followed by each of the template arguments from left to
+// right. Note that template arguments may themselves have template arguments,
+// and so collectMatcherIndices() may call itself.
+// The order of returned matcher indices is always the order of the fully
+// qualified name as read from left to right.
+// For example, calling collectMatcherIndices() for the fully qualified name:
+//    A<B<C, D>, E<F, G<H>, I>
+// Would return the matcher indices:
+//    A, B, C, D, E, F, G, H, I
+func (b *overloadBuilder) collectMatcherIndices(fqn sem.FullyQualifiedName) ([]int, error) {
+	idx, err := b.matcherIndex(fqn.Target)
+	if err != nil {
+		return nil, err
+	}
+	out := []int{idx}
+	for _, arg := range fqn.TemplateArguments {
+		indices, err := b.collectMatcherIndices(arg.(sem.FullyQualifiedName))
+		if err != nil {
+			return nil, err
+		}
+		out = append(out, indices...)
+	}
+	return out, nil
+}
+
+// buildBuiltinTable builds the BuiltinTable from the semantic info
+func buildBuiltinTable(s *sem.Sem) (*BuiltinTable, error) {
+	b := BuiltinTableBuilder{
+		BuiltinTable: BuiltinTable{
+			Sem:           s,
+			TMatcherIndex: map[sem.Named]int{},
+			NMatcherIndex: map[sem.Named]int{},
+		},
+	}
+	b.lut.matcherIndices = lut.New(list.Wrap(&b.MatcherIndices))
+	b.lut.openTypes = lut.New(list.Wrap(&b.OpenTypes))
+	b.lut.openNumbers = lut.New(list.Wrap(&b.OpenNumbers))
+	b.lut.parameters = lut.New(list.Wrap(&b.Parameters))
+	b.lut.overloads = lut.New(list.Wrap(&b.Overloads))
+
+	b.layoutMatchers(s)
+
+	for _, f := range s.Functions {
+		overloads := make([]Overload, len(f.Overloads))
+		overloadDescriptions := make([]string, len(f.Overloads))
+		for i, o := range f.Overloads {
+			overloadDescriptions[i] = fmt.Sprint(o.Decl)
+			var err error
+			if overloads[i], err = b.buildOverload(o); err != nil {
+				return nil, err
+			}
+		}
+
+		b.Functions = append(b.Functions, Function{
+			OverloadDescriptions: overloadDescriptions,
+			NumOverloads:         len(overloads),
+			OverloadsOffset:      b.lut.overloads.Add(overloads),
+		})
+	}
+
+	b.lut.matcherIndices.Compact()
+	b.lut.openTypes.Compact()
+	b.lut.openNumbers.Compact()
+	b.lut.parameters.Compact()
+	b.lut.overloads.Compact()
+
+	return &b.BuiltinTable, nil
+}

tools/src/cmd/builtin-gen/gen/generate.go

@@ -0,0 +1,271 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gen
+
+import (
+	"fmt"
+	"io"
+	"reflect"
+	"strings"
+	"text/template"
+	"unicode"
+
+	"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/sem"
+)
+
+type generator struct {
+	s      *sem.Sem
+	t      *template.Template
+	cached struct {
+		builtinTable *BuiltinTable // lazily built by builtinTable()
+		permuter     *Permuter     // lazily built by permute()
+	}
+}
+
+// WriteFile is a function that Generate() may call to emit a new file from a
+// template.
+// relpath is the relative path from the currently executing template.
+// content is the file content to write.
+type WriteFile func(relpath, content string) error
+
+// Generate executes the template tmpl using the provided semantic
+// information, writing the output to w.
+// See https://golang.org/pkg/text/template/ for documentation on the template
+// syntax.
+func Generate(s *sem.Sem, tmpl string, w io.Writer, writeFile WriteFile) error {
+	g := generator{s: s}
+	return g.generate(tmpl, w, writeFile)
+}
+
+func (g *generator) generate(tmpl string, w io.Writer, writeFile WriteFile) error {
+	t, err := template.New("<template>").Funcs(map[string]interface{}{
+		"Map":                   newMap,
+		"Iterate":               iterate,
+		"Title":                 strings.Title,
+		"PascalCase":            pascalCase,
+		"SplitDisplayName":      splitDisplayName,
+		"HasPrefix":             strings.HasPrefix,
+		"HasSuffix":             strings.HasSuffix,
+		"TrimPrefix":            strings.TrimPrefix,
+		"TrimSuffix":            strings.TrimSuffix,
+		"TrimLeft":              strings.TrimLeft,
+		"TrimRight":             strings.TrimRight,
+		"IsEnumEntry":           is(sem.EnumEntry{}),
+		"IsEnumMatcher":         is(sem.EnumMatcher{}),
+		"IsFQN":                 is(sem.FullyQualifiedName{}),
+		"IsInt":                 is(1),
+		"IsTemplateEnumParam":   is(sem.TemplateEnumParam{}),
+		"IsTemplateNumberParam": is(sem.TemplateNumberParam{}),
+		"IsTemplateTypeParam":   is(sem.TemplateTypeParam{}),
+		"IsType":                is(sem.Type{}),
+		"IsDeclarable":          isDeclarable,
+		"IsFirstIn":             isFirstIn,
+		"IsLastIn":              isLastIn,
+		"BuiltinTable":          g.builtinTable,
+		"Permute":               g.permute,
+		"Eval":                  g.eval,
+		"WriteFile":             func(relpath, content string) (string, error) { return "", writeFile(relpath, content) },
+	}).Option("missingkey=error").
+		Parse(tmpl)
+	if err != nil {
+		return err
+	}
+	g.t = t
+	return t.Execute(w, map[string]interface{}{
+		"Sem": g.s,
+	})
+}
+
+// eval executes the sub-template with the given name and argument, returning
+// the generated output
+func (g *generator) eval(template string, args ...interface{}) (string, error) {
+	target := g.t.Lookup(template)
+	if target == nil {
+		return "", fmt.Errorf("template '%v' not found", template)
+	}
+	sb := strings.Builder{}
+
+	var err error
+	if len(args) == 1 {
+		err = target.Execute(&sb, args[0])
+	} else {
+		m := newMap()
+		if len(args)%2 != 0 {
+			return "", fmt.Errorf("Eval expects a single argument or list name-value pairs")
+		}
+		for i := 0; i < len(args); i += 2 {
+			name, ok := args[i].(string)
+			if !ok {
+				return "", fmt.Errorf("Eval argument %v is not a string", i)
+			}
+			m.Put(name, args[i+1])
+		}
+		err = target.Execute(&sb, m)
+	}
+
+	if err != nil {
+		return "", fmt.Errorf("while evaluating '%v': %v", template, err)
+	}
+	return sb.String(), nil
+}
+
+// builtinTable lazily calls and returns the result of buildBuiltinTable(),
+// caching the result for repeated calls.
+func (g *generator) builtinTable() (*BuiltinTable, error) {
+	if g.cached.builtinTable == nil {
+		var err error
+		g.cached.builtinTable, err = buildBuiltinTable(g.s)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return g.cached.builtinTable, nil
+}
+
+// permute lazily calls buildPermuter(), caching the result for repeated
+// calls, then passes the argument to Permutator.Permute()
+func (g *generator) permute(overload *sem.Overload) ([]Permutation, error) {
+	if g.cached.permuter == nil {
+		var err error
+		g.cached.permuter, err = buildPermuter(g.s)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return g.cached.permuter.Permute(overload)
+}
+
+// Map is a simple generic key-value map, which can be used in the template
+type Map map[interface{}]interface{}
+
+func newMap() Map { return Map{} }
+
+// Put adds the key-value pair into the map.
+// Put always returns an empty string so nothing is printed in the template.
+func (m Map) Put(key, value interface{}) string {
+	m[key] = value
+	return ""
+}
+
+// Get looks up and returns the value with the given key. If the map does not
+// contain the given key, then nil is returned.
+func (m Map) Get(key interface{}) interface{} {
+	return m[key]
+}
+
+// is returns a function that returns true if the value passed to the function
+// matches the type of 'ty'.
+func is(ty interface{}) func(interface{}) bool {
+	rty := reflect.TypeOf(ty)
+	return func(v interface{}) bool {
+		ty := reflect.TypeOf(v)
+		return ty == rty || ty == reflect.PtrTo(rty)
+	}
+}
+
+// isFirstIn returns true if v is the first element of the given slice.
+func isFirstIn(v, slice interface{}) bool {
+	s := reflect.ValueOf(slice)
+	count := s.Len()
+	if count == 0 {
+		return false
+	}
+	return s.Index(0).Interface() == v
+}
+
+// isFirstIn returns true if v is the last element of the given slice.
+func isLastIn(v, slice interface{}) bool {
+	s := reflect.ValueOf(slice)
+	count := s.Len()
+	if count == 0 {
+		return false
+	}
+	return s.Index(count-1).Interface() == v
+}
+
+// iterate returns a slice of length 'n', with each element equal to its index.
+// Useful for: {{- range Iterate $n -}}<this will be looped $n times>{{end}}
+func iterate(n int) []int {
+	out := make([]int, n)
+	for i := range out {
+		out[i] = i
+	}
+	return out
+}
+
+// isDeclarable returns false if the FullyQualifiedName starts with a
+// leading underscore. These are undeclarable as WGSL does not allow identifers
+// to have a leading underscore.
+func isDeclarable(fqn sem.FullyQualifiedName) bool {
+	return !strings.HasPrefix(fqn.Target.GetName(), "_")
+}
+
+// pascalCase returns the snake-case string s transformed into 'PascalCase',
+// Rules:
+// * The first letter of the string is capitalized
+// * Characters following an underscore or number are capitalized
+// * Underscores are removed from the returned string
+// See: https://en.wikipedia.org/wiki/Camel_case
+func pascalCase(s string) string {
+	b := strings.Builder{}
+	upper := true
+	for _, r := range s {
+		if r == '_' {
+			upper = true
+			continue
+		}
+		if upper {
+			b.WriteRune(unicode.ToUpper(r))
+			upper = false
+		} else {
+			b.WriteRune(r)
+		}
+		if unicode.IsNumber(r) {
+			upper = true
+		}
+	}
+	return b.String()
+}
+
+// splitDisplayName splits displayName into parts, where text wrapped in {}
+// braces are not quoted and the rest is quoted. This is used to help process
+// the string value of the [[display()]] decoration. For example:
+//   splitDisplayName("vec{N}<{T}>")
+// would return the strings:
+//   [`"vec"`, `N`, `"<"`, `T`, `">"`]
+func splitDisplayName(displayName string) []string {
+	parts := []string{}
+	pending := strings.Builder{}
+	for _, r := range displayName {
+		switch r {
+		case '{':
+			if pending.Len() > 0 {
+				parts = append(parts, fmt.Sprintf(`"%v"`, pending.String()))
+				pending.Reset()
+			}
+		case '}':
+			if pending.Len() > 0 {
+				parts = append(parts, pending.String())
+				pending.Reset()
+			}
+		default:
+			pending.WriteRune(r)
+		}
+	}
+	if pending.Len() > 0 {
+		parts = append(parts, fmt.Sprintf(`"%v"`, pending.String()))
+	}
+	return parts
+}

tools/src/cmd/builtin-gen/gen/permutate.go

@@ -0,0 +1,380 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gen
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"strings"
+
+	"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/sem"
+	"dawn.googlesource.com/tint/tools/src/fileutils"
+)
+
+// Permuter generates permutations of intrinsic overloads
+type Permuter struct {
+	sem      *sem.Sem
+	allTypes []sem.FullyQualifiedName
+}
+
+// buildPermuter returns a new initialized Permuter
+func buildPermuter(s *sem.Sem) (*Permuter, error) {
+	// allTypes are the list of FQNs that are used for open, unconstrained types
+	allTypes := []sem.FullyQualifiedName{}
+	for _, ty := range s.Types {
+		if len(ty.TemplateParams) > 0 {
+			// Ignore aggregate types for now.
+			// TODO(bclayton): Support a limited set of aggregate types
+			continue
+		}
+		allTypes = append(allTypes, sem.FullyQualifiedName{Target: ty})
+	}
+	return &Permuter{
+		sem:      s,
+		allTypes: allTypes,
+	}, nil
+}
+
+// Permutation describes a single permutation of an overload
+type Permutation struct {
+	sem.Overload        // The permutated overload signature
+	Desc         string // Description of the overload
+	Hash         string // Hash of the overload
+}
+
+// Permute generates a set of permutations for the given intrinsic overload
+func (p *Permuter) Permute(overload *sem.Overload) ([]Permutation, error) {
+	state := permutationState{
+		Permuter:      p,
+		closedTypes:   map[sem.TemplateParam]sem.FullyQualifiedName{},
+		closedNumbers: map[sem.TemplateParam]interface{}{},
+		parameters:    map[int]sem.FullyQualifiedName{},
+	}
+
+	out := []Permutation{}
+
+	// Map of hash to permutation description. Used to detect collisions.
+	hashes := map[string]string{}
+
+	// permutate appends a permutation to out.
+	// permutate may be chained to generate N-dimensional permutations.
+	permutate := func() error {
+		o := sem.Overload{
+			Decl:             overload.Decl,
+			Function:         overload.Function,
+			CanBeUsedInStage: overload.CanBeUsedInStage,
+		}
+		for i, p := range overload.Parameters {
+			ty := state.parameters[i]
+			if !validate(ty, &o.CanBeUsedInStage) {
+				return nil
+			}
+			o.Parameters = append(o.Parameters, sem.Parameter{
+				Name: p.Name,
+				Type: ty,
+			})
+		}
+		if overload.ReturnType != nil {
+			retTys, err := state.permutateFQN(*overload.ReturnType)
+			if err != nil {
+				return fmt.Errorf("while permutating return type: %w", err)
+			}
+			if len(retTys) != 1 {
+				return fmt.Errorf("result type not pinned")
+			}
+			o.ReturnType = &retTys[0]
+		}
+		desc := fmt.Sprint(o)
+		hash := sha256.Sum256([]byte(desc))
+		const hashLength = 6
+		shortHash := hex.EncodeToString(hash[:])[:hashLength]
+		out = append(out, Permutation{
+			Overload: o,
+			Desc:     desc,
+			Hash:     shortHash,
+		})
+
+		// Check for hash collisions
+		if existing, collision := hashes[shortHash]; collision {
+			return fmt.Errorf("hash '%v' collision between %v and %v\nIncrease hashLength in %v",
+				shortHash, existing, desc, fileutils.GoSourcePath())
+		}
+		hashes[shortHash] = desc
+		return nil
+	}
+	for i, param := range overload.Parameters {
+		i, param := i, param // Capture iterator values for anonymous function
+		next := permutate    // Permutation chaining
+		permutate = func() error {
+			permutations, err := state.permutateFQN(param.Type)
+			if err != nil {
+				return fmt.Errorf("while processing parameter %v: %w", i, err)
+			}
+			if len(permutations) == 0 {
+				return fmt.Errorf("parameter %v has no permutations", i)
+			}
+			for _, fqn := range permutations {
+				state.parameters[i] = fqn
+				if err := next(); err != nil {
+					return err
+				}
+			}
+			return nil
+		}
+	}
+	for _, t := range overload.TemplateParams {
+		next := permutate // Permutation chaining
+		switch t := t.(type) {
+		case *sem.TemplateTypeParam:
+			types := p.allTypes
+			if t.Type != nil {
+				var err error
+				types, err = state.permutateFQN(sem.FullyQualifiedName{Target: t.Type})
+				if err != nil {
+					return nil, fmt.Errorf("while permutating open types: %w", err)
+				}
+			}
+			if len(types) == 0 {
+				return nil, fmt.Errorf("open type %v has no permutations", t.Name)
+			}
+			permutate = func() error {
+				for _, ty := range types {
+					state.closedTypes[t] = ty
+					if err := next(); err != nil {
+						return err
+					}
+				}
+				return nil
+			}
+		case *sem.TemplateEnumParam:
+			var permutations []sem.FullyQualifiedName
+			var err error
+			if t.Matcher != nil {
+				permutations, err = state.permutateFQN(sem.FullyQualifiedName{Target: t.Matcher})
+			} else {
+				permutations, err = state.permutateFQN(sem.FullyQualifiedName{Target: t.Enum})
+			}
+			if err != nil {
+				return nil, fmt.Errorf("while permutating open numbers: %w", err)
+			}
+			if len(permutations) == 0 {
+				return nil, fmt.Errorf("open type %v has no permutations", t.Name)
+			}
+			permutate = func() error {
+				for _, n := range permutations {
+					state.closedNumbers[t] = n
+					if err := next(); err != nil {
+						return err
+					}
+				}
+				return nil
+			}
+		case *sem.TemplateNumberParam:
+			// Currently all open numbers are used for vector / matrices
+			permutations := []int{2, 3, 4}
+			permutate = func() error {
+				for _, n := range permutations {
+					state.closedNumbers[t] = n
+					if err := next(); err != nil {
+						return err
+					}
+				}
+				return nil
+			}
+		}
+	}
+
+	if err := permutate(); err != nil {
+		return nil, fmt.Errorf("%v %v %w\nState: %v", overload.Decl.Source, overload.Decl, err, state)
+	}
+
+	return out, nil
+}
+
+type permutationState struct {
+	*Permuter
+	closedTypes   map[sem.TemplateParam]sem.FullyQualifiedName
+	closedNumbers map[sem.TemplateParam]interface{}
+	parameters    map[int]sem.FullyQualifiedName
+}
+
+func (s permutationState) String() string {
+	sb := &strings.Builder{}
+	sb.WriteString("Closed types:\n")
+	for ct, ty := range s.closedTypes {
+		fmt.Fprintf(sb, "  %v: %v\n", ct.GetName(), ty)
+	}
+	sb.WriteString("Closed numbers:\n")
+	for cn, v := range s.closedNumbers {
+		fmt.Fprintf(sb, "  %v: %v\n", cn.GetName(), v)
+	}
+	return sb.String()
+}
+
+func (s *permutationState) permutateFQN(in sem.FullyQualifiedName) ([]sem.FullyQualifiedName, error) {
+	args := append([]interface{}{}, in.TemplateArguments...)
+	out := []sem.FullyQualifiedName{}
+
+	// permutate appends a permutation to out.
+	// permutate may be chained to generate N-dimensional permutations.
+	var permutate func() error
+
+	switch target := in.Target.(type) {
+	case *sem.Type:
+		permutate = func() error {
+			out = append(out, sem.FullyQualifiedName{Target: in.Target, TemplateArguments: args})
+			args = append([]interface{}{}, in.TemplateArguments...)
+			return nil
+		}
+	case sem.TemplateParam:
+		if ty, ok := s.closedTypes[target]; ok {
+			permutate = func() error {
+				out = append(out, ty)
+				return nil
+			}
+		} else {
+			return nil, fmt.Errorf("'%v' was not found in closedTypes", target.GetName())
+		}
+	case *sem.TypeMatcher:
+		permutate = func() error {
+			for _, ty := range target.Types {
+				out = append(out, sem.FullyQualifiedName{Target: ty})
+			}
+			return nil
+		}
+	case *sem.EnumMatcher:
+		permutate = func() error {
+			for _, o := range target.Options {
+				if !o.IsInternal {
+					out = append(out, sem.FullyQualifiedName{Target: o})
+				}
+			}
+			return nil
+		}
+	case *sem.Enum:
+		permutate = func() error {
+			for _, e := range target.Entries {
+				if !e.IsInternal {
+					out = append(out, sem.FullyQualifiedName{Target: e})
+				}
+			}
+			return nil
+		}
+	default:
+		return nil, fmt.Errorf("unhandled target type: %T", in.Target)
+	}
+
+	for i, arg := range in.TemplateArguments {
+		i := i            // Capture iterator value for anonymous functions
+		next := permutate // Permutation chaining
+		switch arg := arg.(type) {
+		case sem.FullyQualifiedName:
+			switch target := arg.Target.(type) {
+			case sem.TemplateParam:
+				if ty, ok := s.closedTypes[target]; ok {
+					args[i] = ty
+				} else if num, ok := s.closedNumbers[target]; ok {
+					args[i] = num
+				} else {
+					return nil, fmt.Errorf("'%v' was not found in closedTypes or closedNumbers", target.GetName())
+				}
+			default:
+				perms, err := s.permutateFQN(arg)
+				if err != nil {
+					return nil, fmt.Errorf("while processing template argument %v: %v", i, err)
+				}
+				if len(perms) == 0 {
+					return nil, fmt.Errorf("template argument %v has no permutations", i)
+				}
+				permutate = func() error {
+					for _, f := range perms {
+						args[i] = f
+						if err := next(); err != nil {
+							return err
+						}
+					}
+					return nil
+				}
+			}
+		default:
+			return nil, fmt.Errorf("permutateFQN() unhandled template argument type: %T", arg)
+		}
+	}
+
+	if err := permutate(); err != nil {
+		return nil, fmt.Errorf("while processing fully qualified name '%v': %w", in.Target.GetName(), err)
+	}
+
+	return out, nil
+}
+
+func validate(fqn sem.FullyQualifiedName, uses *sem.StageUses) bool {
+	switch fqn.Target.GetName() {
+	case "array":
+		elTy := fqn.TemplateArguments[0].(sem.FullyQualifiedName)
+		elTyName := elTy.Target.GetName()
+		switch {
+		case elTyName == "bool" ||
+			strings.Contains(elTyName, "sampler"),
+			strings.Contains(elTyName, "texture"):
+			return false // Not storable
+		}
+	case "ptr":
+		// https://gpuweb.github.io/gpuweb/wgsl/#storage-class
+		access := fqn.TemplateArguments[2].(sem.FullyQualifiedName).Target.(*sem.EnumEntry).Name
+		storageClass := fqn.TemplateArguments[0].(sem.FullyQualifiedName).Target.(*sem.EnumEntry).Name
+		switch storageClass {
+		case "function", "private":
+			if access != "read_write" {
+				return false
+			}
+		case "workgroup":
+			uses.Vertex = false
+			uses.Fragment = false
+			if access != "read_write" {
+				return false
+			}
+		case "uniform":
+			if access != "read" {
+				return false
+			}
+		case "storage":
+			if access != "read_write" && access != "read" {
+				return false
+			}
+		case "handle":
+			if access != "read" {
+				return false
+			}
+		default:
+			return false
+		}
+	}
+
+	if !isDeclarable(fqn) {
+		return false
+	}
+
+	for _, arg := range fqn.TemplateArguments {
+		if argFQN, ok := arg.(sem.FullyQualifiedName); ok {
+			if !validate(argFQN, uses) {
+				return false
+			}
+		}
+	}
+
+	return true
+}

tools/src/cmd/builtin-gen/lexer/lexer.go

@@ -0,0 +1,200 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package lexer provides a basic lexer for the Tint intrinsic definition
+// language
+package lexer
+
+import (
+	"fmt"
+	"unicode"
+
+	"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/tok"
+)
+
+// Lex produces a list of tokens for the given source code
+func Lex(src []rune, filepath string) ([]tok.Token, error) {
+	l := lexer{
+		tok.Location{Line: 1, Column: 1, Rune: 0, Filepath: filepath},
+		src,
+		[]tok.Token{},
+	}
+	if err := l.lex(); err != nil {
+		return nil, err
+	}
+	return l.tokens, nil
+}
+
+type lexer struct {
+	loc    tok.Location
+	runes  []rune
+	tokens []tok.Token
+}
+
+// lex() lexes the source, populating l.tokens
+func (l *lexer) lex() error {
+	for {
+		switch l.peek(0) {
+		case 0:
+			return nil
+		case ' ', '\t':
+			l.next()
+		case '\n':
+			l.next()
+		case '<':
+			l.tok(1, tok.Lt)
+		case '>':
+			l.tok(1, tok.Gt)
+		case '(':
+			l.tok(1, tok.Lparen)
+		case ')':
+			l.tok(1, tok.Rparen)
+		case '{':
+			l.tok(1, tok.Lbrace)
+		case '}':
+			l.tok(1, tok.Rbrace)
+		case ':':
+			l.tok(1, tok.Colon)
+		case ',':
+			l.tok(1, tok.Comma)
+		case '|':
+			l.tok(1, tok.Or)
+		case '"':
+			start := l.loc
+			l.next() // Skip opening quote
+			n := l.count(toFirst('\n', '"'))
+			if l.peek(n) != '"' {
+				return fmt.Errorf("%v unterminated string", start)
+			}
+			l.tok(n, tok.String)
+			l.next() // Skip closing quote
+		default:
+			switch {
+			case l.peek(1) == '/':
+				l.skip(l.count(toFirst('\n')))
+				l.next() // Consume newline
+			case l.match("[[", tok.Ldeco):
+			case l.match("]]", tok.Rdeco):
+			case l.match("->", tok.Arrow):
+			case l.match("fn", tok.Function):
+			case l.match("enum", tok.Enum):
+			case l.match("type", tok.Type):
+			case l.match("match", tok.Match):
+			case unicode.IsLetter(l.peek(0)) || l.peek(0) == '_':
+				l.tok(l.count(alphaNumericOrUnderscore), tok.Identifier)
+			case unicode.IsNumber(l.peek(0)):
+				l.tok(l.count(unicode.IsNumber), tok.Integer)
+			default:
+				return fmt.Errorf("%v: unexpected '%v'", l.loc, string(l.runes[0]))
+			}
+		}
+	}
+}
+
+// next() consumes and returns the next rune in the source, or 0 if reached EOF
+func (l *lexer) next() rune {
+	if len(l.runes) > 0 {
+		r := l.runes[0]
+		l.runes = l.runes[1:]
+		l.loc.Rune++
+		if r == '\n' {
+			l.loc.Line++
+			l.loc.Column = 1
+		} else {
+			l.loc.Column++
+		}
+		return r
+	}
+	return 0
+}
+
+// skip() consumes the next `n` runes in the source
+func (l *lexer) skip(n int) {
+	for i := 0; i < n; i++ {
+		l.next()
+	}
+}
+
+// peek() returns the rune `i` runes ahead of the current position
+func (l *lexer) peek(i int) rune {
+	if i >= len(l.runes) {
+		return 0
+	}
+	return l.runes[i]
+}
+
+// predicate is a function that can be passed to count()
+type predicate func(r rune) bool
+
+// count() returns the number of sequential runes from the current position that
+// match the predicate `p`
+func (l *lexer) count(p predicate) int {
+	for i := 0; i < len(l.runes); i++ {
+		if !p(l.peek(i)) {
+			return i
+		}
+	}
+	return len(l.runes)
+}
+
+// tok() appends a new token of kind `k` using the next `n` runes.
+// The next `n` runes are consumed by tok().
+func (l *lexer) tok(n int, k tok.Kind) {
+	start := l.loc
+	runes := l.runes[:n]
+	l.skip(n)
+	end := l.loc
+
+	src := tok.Source{S: start, E: end}
+	l.tokens = append(l.tokens, tok.Token{Kind: k, Source: src, Runes: runes})
+}
+
+// match() checks whether the next runes are equal to `s`. If they are, then
+// these runes are used to append a new token of kind `k`, and match() returns
+// true. If the next runes are not equal to `s` then false is returned, and no
+// runes are consumed.
+func (l *lexer) match(s string, kind tok.Kind) bool {
+	runes := []rune(s)
+	if len(l.runes) < len(runes) {
+		return false
+	}
+	for i, r := range runes {
+		if l.runes[i] != r {
+			return false
+		}
+	}
+	l.tok(len(runes), kind)
+	return true
+}
+
+// toFirst() returns a predicate that returns true if the rune is not in `runes`
+// toFirst() is intended to be used with count(), so `count(toFirst('x'))` will
+// count up to, but not including the number of consecutive runes that are not
+// 'x'.
+func toFirst(runes ...rune) predicate {
+	return func(r rune) bool {
+		for _, t := range runes {
+			if t == r {
+				return false
+			}
+		}
+		return true
+	}
+}
+
+// alphaNumericOrUnderscore() returns true if the rune `r` is a number, letter
+// or underscore.
+func alphaNumericOrUnderscore(r rune) bool {
+	return r == '_' || unicode.IsLetter(r) || unicode.IsNumber(r)
+}

tools/src/cmd/builtin-gen/lexer/lexer_test.go

@@ -0,0 +1,147 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package lexer_test
+
+import (
+	"fmt"
+	"testing"
+
+	"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/lexer"
+	"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/tok"
+)
+
+func TestLexTokens(t *testing.T) {
+	type test struct {
+		src    string
+		expect tok.Token
+	}
+
+	filepath := "test.txt"
+	loc := func(l, c, r int) tok.Location {
+		return tok.Location{Line: l, Column: c, Rune: r, Filepath: filepath}
+	}
+
+	for _, test := range []test{
+		{"ident", tok.Token{Kind: tok.Identifier, Runes: []rune("ident"), Source: tok.Source{
+			S: loc(1, 1, 0), E: loc(1, 6, 5),
+		}}},
+		{"ident_123", tok.Token{Kind: tok.Identifier, Runes: []rune("ident_123"), Source: tok.Source{
+			S: loc(1, 1, 0), E: loc(1, 10, 9),
+		}}},
+		{"_ident_", tok.Token{Kind: tok.Identifier, Runes: []rune("_ident_"), Source: tok.Source{
+			S: loc(1, 1, 0), E: loc(1, 8, 7),
+		}}},
+		{"123456789", tok.Token{Kind: tok.Integer, Runes: []rune("123456789"), Source: tok.Source{
+			S: loc(1, 1, 0), E: loc(1, 10, 9),
+		}}},
+		{"match", tok.Token{Kind: tok.Match, Runes: []rune("match"), Source: tok.Source{
+			S: loc(1, 1, 0), E: loc(1, 6, 5),
+		}}},
+		{"fn", tok.Token{Kind: tok.Function, Runes: []rune("fn"), Source: tok.Source{
+			S: loc(1, 1, 0), E: loc(1, 3, 2),
+		}}},
+		{"type", tok.Token{Kind: tok.Type, Runes: []rune("type"), Source: tok.Source{
+			S: loc(1, 1, 0), E: loc(1, 5, 4),
+		}}},
+		{"enum", tok.Token{Kind: tok.Enum, Runes: []rune("enum"), Source: tok.Source{
+			S: loc(1, 1, 0), E: loc(1, 5, 4),
+		}}},
+		{":", tok.Token{Kind: tok.Colon, Runes: []rune(":"), Source: tok.Source{
+			S: loc(1, 1, 0), E: loc(1, 2, 1),
+		}}},
+		{",", tok.Token{Kind: tok.Comma, Runes: []rune(","), Source: tok.Source{
+			S: loc(1, 1, 0), E: loc(1, 2, 1),
+		}}},
+		{"<", tok.Token{Kind: tok.Lt, Runes: []rune("<"), Source: tok.Source{
+			S: loc(1, 1, 0), E: loc(1, 2, 1),
+		}}},
+		{">", tok.Token{Kind: tok.Gt, Runes: []rune(">"), Source: tok.Source{
+			S: loc(1, 1, 0), E: loc(1, 2, 1),
+		}}},
+		{"{", tok.Token{Kind: tok.Lbrace, Runes: []rune("{"), Source: tok.Source{
+			S: loc(1, 1, 0), E: loc(1, 2, 1),
+		}}},
+		{"}", tok.Token{Kind: tok.Rbrace, Runes: []rune("}"), Source: tok.Source{
+			S: loc(1, 1, 0), E: loc(1, 2, 1),
+		}}},
+		{"[[", tok.Token{Kind: tok.Ldeco, Runes: []rune("[["), Source: tok.Source{
+			S: loc(1, 1, 0), E: loc(1, 3, 2),
+		}}},
+		{"]]", tok.Token{Kind: tok.Rdeco, Runes: []rune("]]"), Source: tok.Source{
+			S: loc(1, 1, 0), E: loc(1, 3, 2),
+		}}},
+		{"(", tok.Token{Kind: tok.Lparen, Runes: []rune("("), Source: tok.Source{
+			S: loc(1, 1, 0), E: loc(1, 2, 1),
+		}}},
+		{")", tok.Token{Kind: tok.Rparen, Runes: []rune(")"), Source: tok.Source{
+			S: loc(1, 1, 0), E: loc(1, 2, 1),
+		}}},
+		{"|", tok.Token{Kind: tok.Or, Runes: []rune("|"), Source: tok.Source{
+			S: loc(1, 1, 0), E: loc(1, 2, 1),
+		}}},
+		{"->", tok.Token{Kind: tok.Arrow, Runes: []rune("->"), Source: tok.Source{
+			S: loc(1, 1, 0), E: loc(1, 3, 2),
+		}}},
+		{"x // y ", tok.Token{Kind: tok.Identifier, Runes: []rune("x"), Source: tok.Source{
+			S: loc(1, 1, 0), E: loc(1, 2, 1),
+		}}},
+		{`"abc"`, tok.Token{Kind: tok.String, Runes: []rune("abc"), Source: tok.Source{
+			S: loc(1, 2, 1), E: loc(1, 5, 4),
+		}}},
+		{`
+   //
+   ident
+
+   `, tok.Token{Kind: tok.Identifier, Runes: []rune("ident"), Source: tok.Source{
+			S: loc(3, 4, 10), E: loc(3, 9, 15),
+		}}},
+	} {
+		got, err := lexer.Lex([]rune(test.src), filepath)
+		name := fmt.Sprintf(`Lex("%v")`, test.src)
+		switch {
+		case err != nil:
+			t.Errorf("%v returned error: %v", name, err)
+		case len(got) != 1:
+			t.Errorf("%v returned %d tokens: %v", name, len(got), got)
+		case got[0].Kind != test.expect.Kind:
+			t.Errorf(`%v returned unexpected token kind: got "%+v", expected "%+v"`, name, got[0], test.expect)
+		case string(got[0].Runes) != string(test.expect.Runes):
+			t.Errorf(`%v returned unexpected token runes: got "%+v", expected "%+v"`, name, string(got[0].Runes), string(test.expect.Runes))
+		case got[0].Source != test.expect.Source:
+			t.Errorf(`%v returned unexpected token source: got %+v, expected %+v`, name, got[0].Source, test.expect.Source)
+		}
+	}
+}
+
+func TestErrors(t *testing.T) {
+	type test struct {
+		src    string
+		expect string
+	}
+
+	for _, test := range []test{
+		{" \"abc", "test.txt:1:2 unterminated string"},
+		{" \"abc\n", "test.txt:1:2 unterminated string"},
+		{"*", "test.txt:1:1: unexpected '*'"},
+	} {
+		got, err := lexer.Lex([]rune(test.src), "test.txt")
+		if gotErr := err.Error(); test.expect != gotErr {
+			t.Errorf(`Lex() returned error "%+v", expected error "%+v"`, gotErr, test.expect)
+		}
+		if got != nil {
+			t.Errorf("Lex() returned non-nil for error")
+		}
+	}
+}

tools/src/cmd/builtin-gen/main.go

@@ -0,0 +1,173 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// builtin-gen parses the <tint>/src/tint/builtins.def file, then scans the
+// project directory for '<file>.tmpl' files, to produce '<file>' source code
+// files.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/gen"
+	"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/parser"
+	"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/resolver"
+	"dawn.googlesource.com/tint/tools/src/fileutils"
+	"dawn.googlesource.com/tint/tools/src/glob"
+)
+
+const defProjectRelPath = "src/tint/builtins.def"
+
+func main() {
+	if err := run(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func showUsage() {
+	fmt.Println(`
+builtin-gen generates the builtin table for the Tint compiler
+
+builtin-gen parses the <tint>/src/tint/builtins.def file, then scans the project
+directory for '<file>.tmpl' files, to produce '<file>' source code files.
+
+usage:
+  builtin-gen
+
+optional flags:`)
+	flag.PrintDefaults()
+	fmt.Println(``)
+	os.Exit(1)
+}
+
+func run() error {
+	// Load the builtins definition file
+	projectRoot := fileutils.ProjectRoot()
+	defPath := filepath.Join(projectRoot, defProjectRelPath)
+
+	defSource, err := ioutil.ReadFile(defPath)
+	if err != nil {
+		return err
+	}
+
+	// Parse the definition file to produce an AST
+	ast, err := parser.Parse(string(defSource), defProjectRelPath)
+	if err != nil {
+		return err
+	}
+
+	// Resolve the AST to produce the semantic info
+	sem, err := resolver.Resolve(ast)
+	if err != nil {
+		return err
+	}
+
+	// Recursively find all the template files in the <tint>/src directory
+	files, err := glob.Scan(projectRoot, glob.MustParseConfig(`{
+		"paths": [{"include": [
+			"src/**.tmpl",
+			"test/**.tmpl"
+		]}]
+	}`))
+	if err != nil {
+		return err
+	}
+
+	// For each template file...
+	for _, relTmplPath := range files {
+		// Make tmplPath absolute
+		tmplPath := filepath.Join(projectRoot, relTmplPath)
+
+		// Read the template file
+		tmpl, err := ioutil.ReadFile(tmplPath)
+		if err != nil {
+			return fmt.Errorf("failed to open '%v': %w", tmplPath, err)
+		}
+
+		// Create or update the file at relpath if the file content has changed
+		// relpath is a path relative to the template
+		writeFile := func(relpath, body string) error {
+			// Write the common file header
+			sb := strings.Builder{}
+			sb.WriteString(fmt.Sprintf(header, filepath.ToSlash(relTmplPath), filepath.ToSlash(defProjectRelPath)))
+			sb.WriteString(body)
+			content := sb.String()
+			abspath := filepath.Join(filepath.Dir(tmplPath), relpath)
+			return writeFileIfChanged(abspath, content)
+		}
+
+		// Write the content generated using the template and semantic info
+		sb := strings.Builder{}
+		if err := gen.Generate(sem, string(tmpl), &sb, writeFile); err != nil {
+			return fmt.Errorf("while processing '%v': %w", tmplPath, err)
+		}
+
+		if body := sb.String(); body != "" {
+			_, tmplFileName := filepath.Split(tmplPath)
+			outFileName := strings.TrimSuffix(tmplFileName, ".tmpl")
+			if err := writeFile(outFileName, body); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// writes content to path if the file has changed
+func writeFileIfChanged(path, content string) error {
+	existing, err := ioutil.ReadFile(path)
+	if err == nil && string(existing) == content {
+		return nil // Not changed
+	}
+	if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
+		return fmt.Errorf("failed to create directory for '%v': %w", path, err)
+	}
+	if err := ioutil.WriteFile(path, []byte(content), 0666); err != nil {
+		return fmt.Errorf("failed to write file '%v': %w", path, err)
+	}
+	return nil
+}
+
+const header = `// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+////////////////////////////////////////////////////////////////////////////////
+// File generated by tools/builtin-gen
+// using the template:
+//   %v
+// and the builtin defintion file:
+//   %v
+//
+// Do not modify this file directly
+////////////////////////////////////////////////////////////////////////////////
+
+`

tools/src/cmd/builtin-gen/parser/parser.go

@@ -0,0 +1,312 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package parser provides a basic parser for the Tint builtin definition
+// language
+package parser
+
+import (
+	"fmt"
+
+	"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/ast"
+	"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/lexer"
+	"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/tok"
+)
+
+// Parse produces a list of tokens for the given source code
+func Parse(source, filepath string) (*ast.AST, error) {
+	runes := []rune(source)
+	tokens, err := lexer.Lex(runes, filepath)
+	if err != nil {
+		return nil, err
+	}
+
+	p := parser{tokens: tokens}
+	return p.parse()
+}
+
+type parser struct {
+	tokens []tok.Token
+	err    error
+}
+
+func (p *parser) parse() (*ast.AST, error) {
+	out := ast.AST{}
+	var decorations ast.Decorations
+	for p.err == nil {
+		t := p.peek(0)
+		if t == nil {
+			break
+		}
+		switch t.Kind {
+		case tok.Ldeco:
+			decorations = append(decorations, p.decorations()...)
+		case tok.Enum:
+			if len(decorations) > 0 {
+				p.err = fmt.Errorf("%v unexpected decoration", decorations[0].Source)
+			}
+			out.Enums = append(out.Enums, p.enumDecl())
+		case tok.Match:
+			if len(decorations) > 0 {
+				p.err = fmt.Errorf("%v unexpected decoration", decorations[0].Source)
+			}
+			out.Matchers = append(out.Matchers, p.matcherDecl())
+		case tok.Type:
+			out.Types = append(out.Types, p.typeDecl(decorations))
+			decorations = nil
+		case tok.Function:
+			out.Functions = append(out.Functions, p.functionDecl(decorations))
+			decorations = nil
+		default:
+			p.err = fmt.Errorf("%v unexpected token '%v'", t.Source, t.Kind)
+		}
+		if p.err != nil {
+			return nil, p.err
+		}
+	}
+	return &out, nil
+}
+
+func (p *parser) enumDecl() ast.EnumDecl {
+	p.expect(tok.Enum, "enum declaration")
+	name := p.expect(tok.Identifier, "enum name")
+	e := ast.EnumDecl{Source: name.Source, Name: string(name.Runes)}
+	p.expect(tok.Lbrace, "enum declaration")
+	for p.err == nil && p.match(tok.Rbrace) == nil {
+		e.Entries = append(e.Entries, p.enumEntry())
+	}
+	return e
+}
+
+func (p *parser) enumEntry() ast.EnumEntry {
+	decos := p.decorations()
+	name := p.expect(tok.Identifier, "enum entry")
+	return ast.EnumEntry{Source: name.Source, Decorations: decos, Name: string(name.Runes)}
+}
+
+func (p *parser) matcherDecl() ast.MatcherDecl {
+	p.expect(tok.Match, "matcher declaration")
+	name := p.expect(tok.Identifier, "matcher name")
+	m := ast.MatcherDecl{Source: name.Source, Name: string(name.Runes)}
+	p.expect(tok.Colon, "matcher declaration")
+	for p.err == nil {
+		m.Options = append(m.Options, p.templatedName())
+		if p.match(tok.Or) == nil {
+			break
+		}
+	}
+	return m
+}
+
+func (p *parser) typeDecl(decos ast.Decorations) ast.TypeDecl {
+	p.expect(tok.Type, "type declaration")
+	name := p.expect(tok.Identifier, "type name")
+	m := ast.TypeDecl{
+		Source:      name.Source,
+		Decorations: decos,
+		Name:        string(name.Runes),
+	}
+	if p.peekIs(0, tok.Lt) {
+		m.TemplateParams = p.templateParams()
+	}
+	return m
+}
+
+func (p *parser) decorations() ast.Decorations {
+	if p.match(tok.Ldeco) == nil {
+		return nil
+	}
+	out := ast.Decorations{}
+	for p.err == nil {
+		name := p.expect(tok.Identifier, "decoration name")
+		values := []string{}
+		if p.match(tok.Lparen) != nil {
+			for p.err == nil {
+				values = append(values, p.string())
+				if p.match(tok.Comma) == nil {
+					break
+				}
+			}
+			p.expect(tok.Rparen, "decoration values")
+		}
+		out = append(out, ast.Decoration{
+			Source: name.Source,
+			Name:   string(name.Runes),
+			Values: values,
+		})
+		if !p.peekIs(0, tok.Comma) {
+			break
+		}
+	}
+	p.expect(tok.Rdeco, "decoration list")
+	return out
+}
+
+func (p *parser) functionDecl(decos ast.Decorations) ast.FunctionDecl {
+	p.expect(tok.Function, "function declaration")
+	name := p.expect(tok.Identifier, "function name")
+	f := ast.FunctionDecl{
+		Source:      name.Source,
+		Decorations: decos,
+		Name:        string(name.Runes),
+	}
+	if p.peekIs(0, tok.Lt) {
+		f.TemplateParams = p.templateParams()
+	}
+	f.Parameters = p.parameters()
+	if p.match(tok.Arrow) != nil {
+		ret := p.templatedName()
+		f.ReturnType = &ret
+	}
+	return f
+}
+
+func (p *parser) parameters() ast.Parameters {
+	l := ast.Parameters{}
+	p.expect(tok.Lparen, "function parameter list")
+	if p.match(tok.Rparen) == nil {
+		for p.err == nil {
+			l = append(l, p.parameter())
+			if p.match(tok.Comma) == nil {
+				break
+			}
+		}
+		p.expect(tok.Rparen, "function parameter list")
+	}
+	return l
+}
+
+func (p *parser) parameter() ast.Parameter {
+	if p.peekIs(1, tok.Colon) {
+		// name type
+		name := p.expect(tok.Identifier, "parameter name")
+		p.expect(tok.Colon, "parameter type")
+		return ast.Parameter{
+			Source: name.Source,
+			Name:   string(name.Runes),
+			Type:   p.templatedName(),
+		}
+	}
+	// type
+	ty := p.templatedName()
+	return ast.Parameter{
+		Source: ty.Source,
+		Type:   ty,
+	}
+}
+
+func (p *parser) string() string {
+	s := p.expect(tok.String, "string")
+	return string(s.Runes)
+}
+
+func (p *parser) templatedName() ast.TemplatedName {
+	name := p.expect(tok.Identifier, "type name")
+	m := ast.TemplatedName{Source: name.Source, Name: string(name.Runes)}
+	if p.match(tok.Lt) != nil {
+		for p.err == nil {
+			m.TemplateArgs = append(m.TemplateArgs, p.templatedName())
+			if p.match(tok.Comma) == nil {
+				break
+			}
+		}
+		p.expect(tok.Gt, "template argument type list")
+	}
+	return m
+}
+
+func (p *parser) templateParams() ast.TemplateParams {
+	t := ast.TemplateParams{}
+	p.expect(tok.Lt, "template parameter list")
+	for p.err == nil && p.peekIs(0, tok.Identifier) {
+		t = append(t, p.templateParam())
+	}
+	p.expect(tok.Gt, "template parameter list")
+	return t
+}
+
+func (p *parser) templateParam() ast.TemplateParam {
+	name := p.match(tok.Identifier)
+	t := ast.TemplateParam{
+		Source: name.Source,
+		Name:   string(name.Runes),
+	}
+	if p.match(tok.Colon) != nil {
+		t.Type = p.templatedName()
+	}
+	p.match(tok.Comma)
+	return t
+}
+
+func (p *parser) expect(kind tok.Kind, use string) tok.Token {
+	if p.err != nil {
+		return tok.Invalid
+	}
+	t := p.match(kind)
+	if t == nil {
+		if len(p.tokens) > 0 {
+			p.err = fmt.Errorf("%v expected '%v' for %v, got '%v'",
+				p.tokens[0].Source, kind, use, p.tokens[0].Kind)
+		} else {
+			p.err = fmt.Errorf("expected '%v' for %v, but reached end of file", kind, use)
+		}
+		return tok.Invalid
+	}
+	return *t
+}
+
+func (p *parser) ident(use string) string {
+	return string(p.expect(tok.Identifier, use).Runes)
+}
+
+// TODO(bclayton): Currently unused, but will be needed for integer bounds
+// func (p *parser) integer(use string) int {
+// 	t := p.expect(tok.Integer, use)
+// 	if t.Kind != tok.Integer {
+// 		return 0
+// 	}
+// 	i, err := strconv.Atoi(string(t.Runes))
+// 	if err != nil {
+// 		p.err = err
+// 		return 0
+// 	}
+// 	return i
+// }
+
+func (p *parser) match(kind tok.Kind) *tok.Token {
+	if p.err != nil || len(p.tokens) == 0 {
+		return nil
+	}
+	t := p.tokens[0]
+	if t.Kind != kind {
+		return nil
+	}
+	p.tokens = p.tokens[1:]
+	return &t
+}
+
+func (p *parser) peekIs(i int, kind tok.Kind) bool {
+	t := p.peek(i)
+	if t == nil {
+		return false
+	}
+	return t.Kind == kind
+}
+
+func (p *parser) peek(i int) *tok.Token {
+	if len(p.tokens) <= i {
+		return nil
+	}
+	return &p.tokens[i]
+}

tools/src/cmd/builtin-gen/parser/parser_test.go

@@ -0,0 +1,210 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package parser_test
+
+import (
+	"testing"
+
+	"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/ast"
+	"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/parser"
+)
+
+func TestParser(t *testing.T) {
+	type test struct {
+		src    string
+		expect ast.AST
+	}
+
+	for _, test := range []test{
+		{"enum E {}", ast.AST{
+			Enums: []ast.EnumDecl{{Name: "E"}},
+		}},
+		{"enum E { A [[deco]] B C }", ast.AST{
+			Enums: []ast.EnumDecl{{
+				Name: "E",
+				Entries: []ast.EnumEntry{
+					{Name: "A"},
+					{
+						Decorations: ast.Decorations{{Name: "deco"}},
+						Name:        "B",
+					},
+					{Name: "C"},
+				},
+			}},
+		}},
+		{"type T", ast.AST{
+			Types: []ast.TypeDecl{{Name: "T"}},
+		}},
+		{"type T<A, B, C>", ast.AST{
+			Types: []ast.TypeDecl{{
+				Name: "T",
+				TemplateParams: ast.TemplateParams{
+					{Name: "A"},
+					{Name: "B"},
+					{Name: "C"},
+				},
+			}},
+		}},
+		{"[[deco]] type T", ast.AST{
+			Types: []ast.TypeDecl{{
+				Decorations: ast.Decorations{
+					{Name: "deco"},
+				},
+				Name: "T",
+			}},
+		}},
+		{`[[deco("a", "b")]] type T`, ast.AST{
+			Types: []ast.TypeDecl{{
+				Decorations: ast.Decorations{
+					{Name: "deco", Values: []string{"a", "b"}},
+				},
+				Name: "T",
+			}},
+		}},
+		{"match M : A", ast.AST{
+			Matchers: []ast.MatcherDecl{{
+				Name: "M",
+				Options: ast.MatcherOptions{
+					ast.TemplatedName{Name: "A"},
+				},
+			}},
+		}},
+		{"match M : A | B", ast.AST{
+			Matchers: []ast.MatcherDecl{{
+				Name: "M",
+				Options: ast.MatcherOptions{
+					ast.TemplatedName{Name: "A"},
+					ast.TemplatedName{Name: "B"},
+				},
+			}},
+		}},
+		{"fn F()", ast.AST{
+			Functions: []ast.FunctionDecl{{
+				Name: "F",
+			}},
+		}},
+		{"[[deco]] fn F()", ast.AST{
+			Functions: []ast.FunctionDecl{{
+				Name: "F",
+				Decorations: ast.Decorations{
+					{Name: "deco"},
+				},
+			}},
+		}},
+		{"fn F(a)", ast.AST{
+			Functions: []ast.FunctionDecl{{
+				Name: "F",
+				Parameters: ast.Parameters{
+					{Type: ast.TemplatedName{Name: "a"}},
+				},
+			}},
+		}},
+		{"fn F(a: T)", ast.AST{
+			Functions: []ast.FunctionDecl{{
+				Name: "F",
+				Parameters: ast.Parameters{
+					{Name: "a", Type: ast.TemplatedName{Name: "T"}},
+				},
+			}},
+		}},
+		{"fn F(a, b)", ast.AST{
+			Functions: []ast.FunctionDecl{{
+				Name: "F",
+				Parameters: ast.Parameters{
+					{Type: ast.TemplatedName{Name: "a"}},
+					{Type: ast.TemplatedName{Name: "b"}},
+				},
+			}},
+		}},
+		{"fn F<A : B<C>>()", ast.AST{
+			Functions: []ast.FunctionDecl{{
+				Name: "F",
+				TemplateParams: ast.TemplateParams{
+					{
+						Name: "A", Type: ast.TemplatedName{
+							Name: "B",
+							TemplateArgs: ast.TemplatedNames{
+								{Name: "C"},
+							},
+						},
+					},
+				},
+			}},
+		}},
+		{"fn F<T>(a: X, b: Y<T>)", ast.AST{
+			Functions: []ast.FunctionDecl{{
+				Name: "F",
+				TemplateParams: ast.TemplateParams{
+					{Name: "T"},
+				},
+				Parameters: ast.Parameters{
+					{Name: "a", Type: ast.TemplatedName{Name: "X"}},
+					{Name: "b", Type: ast.TemplatedName{
+						Name:         "Y",
+						TemplateArgs: []ast.TemplatedName{{Name: "T"}},
+					}},
+				},
+			}},
+		}},
+		{"fn F() -> X", ast.AST{
+			Functions: []ast.FunctionDecl{{
+				Name:       "F",
+				ReturnType: &ast.TemplatedName{Name: "X"},
+			}},
+		}},
+		{"fn F() -> X<T>", ast.AST{
+			Functions: []ast.FunctionDecl{{
+				Name: "F",
+				ReturnType: &ast.TemplatedName{
+					Name:         "X",
+					TemplateArgs: []ast.TemplatedName{{Name: "T"}},
+				},
+			}},
+		}},
+	} {
+		got, err := parser.Parse(test.src, "file.txt")
+		if err != nil {
+			t.Errorf("While parsing:\n%s\nParse() returned error: %v", test.src, err)
+			continue
+		}
+
+		gotStr, expectStr := got.String(), test.expect.String()
+		if gotStr != expectStr {
+			t.Errorf("While parsing:\n%s\nGot:\n%s\nExpected:\n%s", test.src, gotStr, expectStr)
+		}
+	}
+}
+
+func TestErrors(t *testing.T) {
+	type test struct {
+		src    string
+		expect string
+	}
+
+	for _, test := range []test{
+		{"+", "test.txt:1:1: unexpected '+'"},
+		{"123", "test.txt:1:1 unexpected token 'integer'"},
+		{"[[123]]", "test.txt:1:3 expected 'ident' for decoration name, got 'integer'"},
+		{"[[abc", "expected ']]' for decoration list, but reached end of file"},
+	} {
+		got, err := parser.Parse(test.src, "test.txt")
+		if gotErr := err.Error(); test.expect != gotErr {
+			t.Errorf(`Parse() returned error "%+v", expected error "%+v"`, gotErr, test.expect)
+		}
+		if got != nil {
+			t.Errorf("Lex() returned non-nil for error")
+		}
+	}
+}

tools/src/cmd/builtin-gen/resolver/resolve.go

@@ -0,0 +1,653 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package resolver
+
+import (
+	"fmt"
+	"sort"
+
+	"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/ast"
+	"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/sem"
+	"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/tok"
+)
+
+type resolver struct {
+	a *ast.AST
+	s *sem.Sem
+
+	globals           scope
+	functions         map[string]*sem.Function
+	enumEntryMatchers map[*sem.EnumEntry]*sem.EnumMatcher
+}
+
+// Resolve processes the AST
+func Resolve(a *ast.AST) (*sem.Sem, error) {
+	r := resolver{
+		a:                 a,
+		s:                 sem.New(),
+		globals:           newScope(nil),
+		functions:         map[string]*sem.Function{},
+		enumEntryMatchers: map[*sem.EnumEntry]*sem.EnumMatcher{},
+	}
+	// Declare and resolve all the enumerators
+	for _, e := range a.Enums {
+		if err := r.enum(e); err != nil {
+			return nil, err
+		}
+	}
+	// Declare and resolve all the ty types
+	for _, p := range a.Types {
+		if err := r.ty(p); err != nil {
+			return nil, err
+		}
+	}
+	// Declare and resolve the type matchers
+	for _, m := range a.Matchers {
+		if err := r.matcher(m); err != nil {
+			return nil, err
+		}
+	}
+	// Declare and resolve the functions
+	for _, f := range a.Functions {
+		if err := r.function(f); err != nil {
+			return nil, err
+		}
+	}
+
+	// Calculate the unique parameter names
+	r.s.UniqueParameterNames = r.calculateUniqueParameterNames()
+
+	return r.s, nil
+}
+
+// enum() resolves an enum declaration.
+// The resulting sem.Enum is appended to Sem.Enums, and the enum and all its
+// entries are registered with the global scope.
+func (r *resolver) enum(e ast.EnumDecl) error {
+	s := &sem.Enum{
+		Decl: e,
+		Name: e.Name,
+	}
+
+	// Register the enum
+	r.s.Enums = append(r.s.Enums, s)
+	if err := r.globals.declare(s, e.Source); err != nil {
+		return err
+	}
+
+	// Register each of the enum entries
+	for _, ast := range e.Entries {
+		entry := &sem.EnumEntry{
+			Name: ast.Name,
+			Enum: s,
+		}
+		if internal := ast.Decorations.Take("internal"); internal != nil {
+			entry.IsInternal = true
+			if len(internal.Values) != 0 {
+				return fmt.Errorf("%v unexpected value for internal decoration", ast.Source)
+			}
+		}
+		if len(ast.Decorations) != 0 {
+			return fmt.Errorf("%v unknown decoration", ast.Decorations[0].Source)
+		}
+		if err := r.globals.declare(entry, e.Source); err != nil {
+			return err
+		}
+		s.Entries = append(s.Entries, entry)
+	}
+
+	return nil
+}
+
+// ty() resolves a type declaration.
+// The resulting sem.Type is appended to Sem.Types, and the type is registered
+// with the global scope.
+func (r *resolver) ty(a ast.TypeDecl) error {
+	t := &sem.Type{
+		Decl: a,
+		Name: a.Name,
+	}
+
+	// Register the type
+	r.s.Types = append(r.s.Types, t)
+	if err := r.globals.declare(t, a.Source); err != nil {
+		return err
+	}
+
+	// Create a new scope for resolving template parameters
+	s := newScope(&r.globals)
+
+	// Resolve the type template parameters
+	templateParams, err := r.templateParams(&s, a.TemplateParams)
+	if err != nil {
+		return err
+	}
+	t.TemplateParams = templateParams
+
+	// Scan for decorations
+	if d := a.Decorations.Take("display"); d != nil {
+		if len(d.Values) != 1 {
+			return fmt.Errorf("%v expected a single value for 'display' decoration", d.Source)
+		}
+		t.DisplayName = d.Values[0]
+	}
+	if len(a.Decorations) != 0 {
+		return fmt.Errorf("%v unknown decoration", a.Decorations[0].Source)
+	}
+
+	return nil
+}
+
+// matcher() resolves a match declaration to either a sem.TypeMatcher or
+// sem.EnumMatcher.
+// The resulting matcher is appended to either Sem.TypeMatchers or
+// Sem.EnumMatchers, and is registered with the global scope.
+func (r *resolver) matcher(a ast.MatcherDecl) error {
+	// Determine whether this is a type matcher or enum matcher by resolving the
+	// first option
+	firstOption, err := r.lookupNamed(&r.globals, a.Options[0])
+	if err != nil {
+		return err
+	}
+
+	// Resolve to a sem.TypeMatcher or a sem.EnumMatcher
+	switch firstOption := firstOption.(type) {
+	case *sem.Type:
+		options := map[sem.Named]tok.Source{}
+		m := &sem.TypeMatcher{
+			Decl: a,
+			Name: a.Name,
+		}
+
+		// Register the matcher
+		r.s.TypeMatchers = append(r.s.TypeMatchers, m)
+		if err := r.globals.declare(m, a.Source); err != nil {
+			return err
+		}
+
+		// Resolve each of the types in the options list
+		for _, ast := range m.Decl.Options {
+			ty, err := r.lookupType(&r.globals, ast)
+			if err != nil {
+				return err
+			}
+			m.Types = append(m.Types, ty)
+			if s, dup := options[ty]; dup {
+				return fmt.Errorf("%v duplicate option '%v' in matcher\nFirst declared here: %v", ast.Source, ast.Name, s)
+			}
+			options[ty] = ast.Source
+		}
+
+		return nil
+
+	case *sem.EnumEntry:
+		enum := firstOption.Enum
+		m := &sem.EnumMatcher{
+			Decl: a,
+			Name: a.Name,
+			Enum: enum,
+		}
+
+		// Register the matcher
+		r.s.EnumMatchers = append(r.s.EnumMatchers, m)
+		if err := r.globals.declare(m, a.Source); err != nil {
+			return err
+		}
+
+		// Resolve each of the enums in the options list
+		for _, ast := range m.Decl.Options {
+			entry := enum.FindEntry(ast.Name)
+			if entry == nil {
+				return fmt.Errorf("%v enum '%v' does not contain '%v'", ast.Source, enum.Name, ast.Name)
+			}
+			m.Options = append(m.Options, entry)
+		}
+
+		return nil
+	}
+	return fmt.Errorf("'%v' cannot be used for matcher", a.Name)
+}
+
+// function() resolves a function overload declaration.
+// The the first overload for the function creates and appends the sem.Function
+// to Sem.Functions. Subsequent overloads append their resolved overload to the
+// sem.Function.Overloads list.
+func (r *resolver) function(a ast.FunctionDecl) error {
+	// If this is the first overload of the function, create and register the
+	// semantic function.
+	f := r.functions[a.Name]
+	if f == nil {
+		f = &sem.Function{Name: a.Name}
+		r.functions[a.Name] = f
+		r.s.Functions = append(r.s.Functions, f)
+	}
+
+	// Create a new scope for resolving template parameters
+	s := newScope(&r.globals)
+
+	// Resolve the declared template parameters
+	templateParams, err := r.templateParams(&s, a.TemplateParams)
+	if err != nil {
+		return err
+	}
+
+	// Construct the semantic overload
+	overload := &sem.Overload{
+		Decl:           a,
+		Function:       f,
+		Parameters:     make([]sem.Parameter, len(a.Parameters)),
+		TemplateParams: templateParams,
+	}
+
+	// Process overload decorations
+	if stageDeco := a.Decorations.Take("stage"); stageDeco != nil {
+		for stageDeco != nil {
+			for _, stage := range stageDeco.Values {
+				switch stage {
+				case "vertex":
+					overload.CanBeUsedInStage.Vertex = true
+				case "fragment":
+					overload.CanBeUsedInStage.Fragment = true
+				case "compute":
+					overload.CanBeUsedInStage.Compute = true
+				default:
+					return fmt.Errorf("%v unknown stage '%v'", stageDeco.Source, stage)
+				}
+			}
+			stageDeco = a.Decorations.Take("stage")
+		}
+	} else {
+		overload.CanBeUsedInStage = sem.StageUses{
+			Vertex:   true,
+			Fragment: true,
+			Compute:  true,
+		}
+	}
+	if deprecated := a.Decorations.Take("deprecated"); deprecated != nil {
+		overload.IsDeprecated = true
+		if len(deprecated.Values) != 0 {
+			return fmt.Errorf("%v unexpected value for deprecated decoration", deprecated.Source)
+		}
+	}
+	if len(a.Decorations) != 0 {
+		return fmt.Errorf("%v unknown decoration", a.Decorations[0].Source)
+	}
+
+	// Append the overload to the function
+	f.Overloads = append(f.Overloads, overload)
+
+	// Sort the template parameters by resolved type. Append these to
+	// sem.Overload.OpenTypes or sem.Overload.OpenNumbers based on their kind.
+	for _, param := range templateParams {
+		switch param := param.(type) {
+		case *sem.TemplateTypeParam:
+			overload.OpenTypes = append(overload.OpenTypes, param)
+		case *sem.TemplateEnumParam, *sem.TemplateNumberParam:
+			overload.OpenNumbers = append(overload.OpenNumbers, param)
+		}
+	}
+
+	// Update high-water marks of open types / numbers
+	if r.s.MaxOpenTypes < len(overload.OpenTypes) {
+		r.s.MaxOpenTypes = len(overload.OpenTypes)
+	}
+	if r.s.MaxOpenNumbers < len(overload.OpenNumbers) {
+		r.s.MaxOpenNumbers = len(overload.OpenNumbers)
+	}
+
+	// Resolve the parameters
+	for i, p := range a.Parameters {
+		usage, err := r.fullyQualifiedName(&s, p.Type)
+		if err != nil {
+			return err
+		}
+		overload.Parameters[i] = sem.Parameter{
+			Name: p.Name,
+			Type: usage,
+		}
+	}
+
+	// Resolve the return type
+	if a.ReturnType != nil {
+		usage, err := r.fullyQualifiedName(&s, *a.ReturnType)
+		if err != nil {
+			return err
+		}
+		switch usage.Target.(type) {
+		case *sem.Type, *sem.TemplateTypeParam:
+			overload.ReturnType = &usage
+		default:
+			return fmt.Errorf("%v cannot use '%v' as return type. Must be a type or template type", a.ReturnType.Source, a.ReturnType.Name)
+		}
+	}
+
+	return nil
+}
+
+// fullyQualifiedName() resolves the ast.TemplatedName to a sem.FullyQualifiedName.
+func (r *resolver) fullyQualifiedName(s *scope, arg ast.TemplatedName) (sem.FullyQualifiedName, error) {
+	target, err := r.lookupNamed(s, arg)
+	if err != nil {
+		return sem.FullyQualifiedName{}, err
+	}
+
+	if entry, ok := target.(*sem.EnumEntry); ok {
+		// The target resolved to an enum entry.
+		// Automagically transform this into a synthetic matcher with a single
+		// option. i.e.
+		// This:
+		//   enum E{ a b c }
+		//   fn F(b)
+		// Becomes:
+		//   enum E{ a b c }
+		//   matcher b
+		//   fn F(b)
+		// We don't really care right now that we have a symbol collision
+		// between E.b and b, as the generators return different names for
+		// these.
+		matcher, ok := r.enumEntryMatchers[entry]
+		if !ok {
+			matcher = &sem.EnumMatcher{
+				Name:    entry.Name,
+				Enum:    entry.Enum,
+				Options: []*sem.EnumEntry{entry},
+			}
+			r.enumEntryMatchers[entry] = matcher
+			r.s.EnumMatchers = append(r.s.EnumMatchers, matcher)
+		}
+		target = matcher
+	}
+
+	fqn := sem.FullyQualifiedName{
+		Target:            target,
+		TemplateArguments: make([]interface{}, len(arg.TemplateArgs)),
+	}
+	for i, a := range arg.TemplateArgs {
+		arg, err := r.fullyQualifiedName(s, a)
+		if err != nil {
+			return sem.FullyQualifiedName{}, err
+		}
+		fqn.TemplateArguments[i] = arg
+	}
+	return fqn, nil
+}
+
+// templateParams() resolves the ast.TemplateParams into list of sem.TemplateParam.
+// Each sem.TemplateParam is registered with the scope s.
+func (r *resolver) templateParams(s *scope, l ast.TemplateParams) ([]sem.TemplateParam, error) {
+	out := []sem.TemplateParam{}
+	for _, ast := range l {
+		param, err := r.templateParam(ast)
+		if err != nil {
+			return nil, err
+		}
+		s.declare(param, ast.Source)
+		out = append(out, param)
+	}
+	return out, nil
+}
+
+// templateParams() resolves the ast.TemplateParam into sem.TemplateParam, which
+// is either a sem.TemplateEnumParam or a sem.TemplateTypeParam.
+func (r *resolver) templateParam(a ast.TemplateParam) (sem.TemplateParam, error) {
+	if a.Type.Name == "num" {
+		return &sem.TemplateNumberParam{Name: a.Name}, nil
+	}
+
+	if a.Type.Name != "" {
+		resolved, err := r.lookupNamed(&r.globals, a.Type)
+		if err != nil {
+			return nil, err
+		}
+		switch r := resolved.(type) {
+		case *sem.Enum:
+			return &sem.TemplateEnumParam{Name: a.Name, Enum: r}, nil
+		case *sem.EnumMatcher:
+			return &sem.TemplateEnumParam{Name: a.Name, Enum: r.Enum, Matcher: r}, nil
+		case *sem.TypeMatcher:
+			return &sem.TemplateTypeParam{Name: a.Name, Type: r}, nil
+		default:
+			return nil, fmt.Errorf("%v invalid template parameter type '%v'", a.Source, a.Type.Name)
+		}
+	}
+
+	return &sem.TemplateTypeParam{Name: a.Name}, nil
+}
+
+// lookupType() searches the scope `s` and its ancestors for the sem.Type with
+// the given name.
+func (r *resolver) lookupType(s *scope, a ast.TemplatedName) (*sem.Type, error) {
+	resolved, err := r.lookupNamed(s, a)
+	if err != nil {
+		return nil, err
+	}
+	// Something with the given name was found...
+	if ty, ok := resolved.(*sem.Type); ok {
+		return ty, nil
+	}
+	// ... but that something was not a sem.Type
+	return nil, fmt.Errorf("%v '%v' resolves to %v but type is expected", a.Source, a.Name, describe(resolved))
+}
+
+// lookupNamed() searches `s` and its ancestors for the sem.Named object with
+// the given name. If there are template arguments for the name `a`, then
+// lookupNamed() performs basic validation that those arguments can be passed
+// to the named object.
+func (r *resolver) lookupNamed(s *scope, a ast.TemplatedName) (sem.Named, error) {
+	target := s.lookup(a.Name)
+	if target == nil {
+		return nil, fmt.Errorf("%v cannot resolve '%v'", a.Source, a.Name)
+	}
+
+	// Something with the given name was found...
+	var params []sem.TemplateParam
+	var ty sem.ResolvableType
+	switch target := target.object.(type) {
+	case *sem.Type:
+		ty = target
+		params = target.TemplateParams
+	case *sem.TypeMatcher:
+		ty = target
+		params = target.TemplateParams
+	case sem.TemplateParam:
+		if len(a.TemplateArgs) != 0 {
+			return nil, fmt.Errorf("%v '%v' template parameters do not accept template arguments", a.Source, a.Name)
+		}
+		return target.(sem.Named), nil
+	case sem.Named:
+		return target, nil
+	default:
+		panic(fmt.Errorf("Unknown resolved type %T", target))
+	}
+	// ... and that something takes template parameters
+	// Check the number of templated name template arguments match the number of
+	// templated parameters for the target.
+	args := a.TemplateArgs
+	if len(params) != len(args) {
+		return nil, fmt.Errorf("%v '%v' requires %d template arguments, but %d were provided", a.Source, a.Name, len(params), len(args))
+	}
+
+	// Check templated name template argument kinds match the parameter kinds
+	for i, ast := range args {
+		param := params[i]
+		arg, err := r.lookupNamed(s, args[i])
+		if err != nil {
+			return nil, err
+		}
+
+		if err := checkCompatible(arg, param); err != nil {
+			return nil, fmt.Errorf("%v %w", ast.Source, err)
+		}
+	}
+	return ty, nil
+}
+
+// calculateUniqueParameterNames() iterates over all the parameters of all
+// overloads, calculating the list of unique parameter names
+func (r *resolver) calculateUniqueParameterNames() []string {
+	set := map[string]struct{}{"": {}}
+	names := []string{}
+	for _, f := range r.s.Functions {
+		for _, o := range f.Overloads {
+			for _, p := range o.Parameters {
+				if _, dup := set[p.Name]; !dup {
+					set[p.Name] = struct{}{}
+					names = append(names, p.Name)
+				}
+			}
+		}
+	}
+	sort.Strings(names)
+	return names
+}
+
+// describe() returns a string describing a sem.Named
+func describe(n sem.Named) string {
+	switch n := n.(type) {
+	case *sem.Type:
+		return "type '" + n.Name + "'"
+	case *sem.TypeMatcher:
+		return "type matcher '" + n.Name + "'"
+	case *sem.Enum:
+		return "enum '" + n.Name + "'"
+	case *sem.EnumMatcher:
+		return "enum matcher '" + n.Name + "'"
+	case *sem.TemplateTypeParam:
+		return "template type"
+	case *sem.TemplateEnumParam:
+		return "template enum '" + n.Enum.Name + "'"
+	case *sem.EnumEntry:
+		return "enum entry '" + n.Enum.Name + "." + n.Name + "'"
+	case *sem.TemplateNumberParam:
+		return "template number"
+	default:
+		panic(fmt.Errorf("unhandled type %T", n))
+	}
+}
+
+// checkCompatible() returns an error if `arg` cannot be used as an argument for
+// a parameter of `param`.
+func checkCompatible(arg, param sem.Named) error {
+	// asEnum() returns the underlying sem.Enum if n is a enum matcher,
+	// templated enum parameter or an enum entry, otherwise nil
+	asEnum := func(n sem.Named) *sem.Enum {
+		switch n := n.(type) {
+		case *sem.EnumMatcher:
+			return n.Enum
+		case *sem.TemplateEnumParam:
+			return n.Enum
+		case *sem.EnumEntry:
+			return n.Enum
+		default:
+			return nil
+		}
+	}
+
+	if arg := asEnum(arg); arg != nil {
+		param := asEnum(param)
+		if arg == param {
+			return nil
+		}
+	}
+
+	anyNumber := "any number"
+	// asNumber() returns anyNumber if n is a TemplateNumberParam.
+	// TODO(bclayton): Once we support number ranges [e.g.: fn F<N: 1..4>()], we
+	// should check number ranges are compatible
+	asNumber := func(n sem.Named) interface{} {
+		switch n.(type) {
+		case *sem.TemplateNumberParam:
+			return anyNumber
+		default:
+			return nil
+		}
+	}
+
+	if arg := asNumber(arg); arg != nil {
+		param := asNumber(param)
+		if arg == param {
+			return nil
+		}
+	}
+
+	anyType := &sem.Type{}
+	// asNumber() returns the sem.Type, sem.TypeMatcher if the named object
+	// resolves to one of these, or anyType if n is a unconstrained template
+	// type parameter.
+	asResolvableType := func(n sem.Named) sem.ResolvableType {
+		switch n := n.(type) {
+		case *sem.TemplateTypeParam:
+			if n.Type != nil {
+				return n.Type
+			}
+			return anyType
+		case *sem.Type:
+			return n
+		case *sem.TypeMatcher:
+			return n
+		default:
+			return nil
+		}
+	}
+
+	if arg := asResolvableType(arg); arg != nil {
+		param := asResolvableType(param)
+		if arg == param || param == anyType {
+			return nil
+		}
+	}
+
+	return fmt.Errorf("cannot use %v as %v", describe(arg), describe(param))
+}
+
+// scope is a basic hierarchical name to object table
+type scope struct {
+	objects map[string]objectAndSource
+	parent  *scope
+}
+
+// objectAndSource is a sem.Named object with a source
+type objectAndSource struct {
+	object sem.Named
+	source tok.Source
+}
+
+// newScope returns a newly initalized scope
+func newScope(parent *scope) scope {
+	return scope{objects: map[string]objectAndSource{}, parent: parent}
+}
+
+// lookup() searches the scope and then its parents for the symbol with the
+// given name.
+func (s *scope) lookup(name string) *objectAndSource {
+	if o, found := s.objects[name]; found {
+		return &o
+	}
+	if s.parent == nil {
+		return nil
+	}
+	return s.parent.lookup(name)
+}
+
+// declare() declares the symbol with the given name, erroring on symbol
+// collision.
+func (s *scope) declare(object sem.Named, source tok.Source) error {
+	name := object.GetName()
+	if existing := s.lookup(name); existing != nil {
+		return fmt.Errorf("%v '%v' already declared\nFirst declared here: %v", source, name, existing.source)
+	}
+	s.objects[name] = objectAndSource{object, source}
+	return nil
+}

tools/src/cmd/builtin-gen/resolver/resolver_test.go

@@ -0,0 +1,330 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package resolver_test
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/parser"
+	"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/resolver"
+)
+
+func TestResolver(t *testing.T) {
+	type test struct {
+		src string
+		err string
+	}
+
+	success := ""
+	for _, test := range []test{
+		{
+			`type X`,
+			success,
+		}, {
+			`enum E {}`,
+			success,
+		}, {
+			`enum E {A B C}`,
+			success,
+		}, {
+			`type X`,
+			success,
+		}, {
+			`[[display("Y")]] type X`,
+			success,
+		}, {
+			`
+type x
+match y: x`,
+			success,
+		}, {
+			`
+enum e {a b c}
+match y: c | a | b`,
+			success,
+		}, {
+			`fn f()`,
+			success,
+		}, {
+			`fn f<T>()`,
+			success,
+		}, {
+			`
+type f32
+fn f<N: num>()`,
+			success,
+		}, {
+			`
+enum e { a b c }
+fn f<N: e>()`,
+			success,
+		}, {
+			`
+type f32
+fn f<T>(T) -> f32`,
+			success,
+		}, {
+			`
+type f32
+type P<T>
+match m: f32
+fn f<T: m>(P<T>) -> T`,
+			success,
+		}, {
+			`
+type f32
+type P<T>
+match m: f32
+fn f(P<m>)`,
+			success,
+		}, {
+			`
+enum e { a }
+fn f(a)`,
+			success,
+		}, {
+			`
+enum e { a b }
+type T<E: e>
+match m: a
+fn f<E: m>(T<E>)`,
+			success,
+		}, {
+			`
+enum e { a b }
+type T<E: e>
+match m: a
+fn f(T<m>)`,
+			success,
+		}, {
+			`
+enum e { a }
+type T<E: e>
+fn f(T<a>)`,
+			success,
+		}, {
+			`
+type T<E: num>
+fn f<E: num>(T<E>)`,
+			success,
+		}, {
+			`fn f<T>(T)`,
+			success,
+		}, {
+			`
+enum e { a b }
+fn f<E: e>()`,
+			success,
+		}, {
+			`
+enum e { a b }
+match m: a | b
+fn f<E: m>()`,
+			success,
+		}, {
+			`
+type f32
+type T<x>
+fn f(T<T<f32>>)`,
+			success,
+		}, {
+			`enum E {A A}`,
+			`
+file.txt:1:6 'A' already declared
+First declared here: file.txt:1:6
+`,
+		},
+		{
+			`type X type X`,
+			`
+file.txt:1:13 'X' already declared
+First declared here: file.txt:1:6`,
+		}, {
+			`[[meow]] type X`,
+			`
+file.txt:1:3 unknown decoration
+`,
+		}, {
+			`[[display("Y", "Z")]] type X`,
+			`
+file.txt:1:3 expected a single value for 'display' decoration`,
+		}, {
+			`
+enum e { a }
+enum e { b }`,
+			`
+file.txt:2:6 'e' already declared
+First declared here: file.txt:1:6`,
+		}, {
+			`
+type X
+match X : X`,
+			`
+file.txt:2:7 'X' already declared
+First declared here: file.txt:1:6`,
+		}, {
+			`type T<X>
+match M : T`,
+			`file.txt:2:11 'T' requires 1 template arguments, but 0 were provided`,
+		}, {
+			`
+match x: y`,
+			`
+file.txt:1:10 cannot resolve 'y'
+`,
+		}, {
+			`
+type a
+match x: a | b`,
+			`
+file.txt:2:14 cannot resolve 'b'
+`,
+		}, {
+			`
+type a
+enum e { b }
+match x: a | b`,
+			`
+file.txt:3:14 'b' resolves to enum entry 'e.b' but type is expected
+`,
+		}, {
+			`
+type a
+type b
+match x: a | b | a`,
+			`
+file.txt:3:18 duplicate option 'a' in matcher
+First declared here: file.txt:3:10
+`,
+		}, {
+			`
+enum e { a c }
+match x: a | b | c`,
+			`
+file.txt:2:14 enum 'e' does not contain 'b'
+`,
+		}, {
+			`
+enum e { a }
+match x: a
+match x: a`,
+			`
+file.txt:3:7 'x' already declared
+First declared here: file.txt:2:7
+`,
+		}, {
+			`
+type t
+match x: t
+match y: x`,
+			`
+'y' cannot be used for matcher
+`,
+		}, {
+			`fn f(u)`,
+			`file.txt:1:6 cannot resolve 'u'`,
+		}, {
+			`fn f() -> u`,
+			`file.txt:1:11 cannot resolve 'u'`,
+		}, {
+			`fn f<T: u>()`,
+			`file.txt:1:9 cannot resolve 'u'`,
+		}, {
+			`
+enum e { a }
+fn f() -> e`,
+			`file.txt:2:11 cannot use 'e' as return type. Must be a type or template type`,
+		}, {
+			`
+type T<x>
+fn f(T<u>)`,
+			`file.txt:2:8 cannot resolve 'u'`,
+		}, {
+			`
+type x
+fn f<T>(T<x>)`,
+			`file.txt:2:9 'T' template parameters do not accept template arguments`,
+		}, {
+			`
+type A<N: num>
+type B
+fn f(A<B>)`,
+			`file.txt:3:8 cannot use type 'B' as template number`,
+		}, {
+			`
+type A<N>
+enum E { b }
+fn f(A<b>)`,
+			`file.txt:3:8 cannot use enum entry 'E.b' as template type`,
+		}, {
+			`
+type T
+type P<N: num>
+match m: T
+fn f(P<m>)`,
+			`file.txt:4:8 cannot use type matcher 'm' as template number`,
+		}, {
+			`
+type P<N: num>
+enum E { b }
+fn f(P<E>)`,
+			`file.txt:3:8 cannot use enum 'E' as template number`,
+		}, {
+			`
+type P<N: num>
+enum E { a b }
+match m: a | b
+fn f(P<m>)`,
+			`file.txt:4:8 cannot use enum matcher 'm' as template number`,
+		}, {
+			`
+type P<N: num>
+enum E { a b }
+match m: a | b
+fn f<M: m>(P<M>)`,
+			`file.txt:4:14 cannot use template enum 'E' as template number`,
+		}, {
+			`
+enum E { a }
+type T<X: a>`,
+			`file.txt:2:8 invalid template parameter type 'a'`,
+		}, {
+			`
+enum E { a }
+fn f<M: a>()`,
+			`file.txt:2:6 invalid template parameter type 'a'`,
+		},
+	} {
+
+		ast, err := parser.Parse(strings.TrimSpace(string(test.src)), "file.txt")
+		if err != nil {
+			t.Errorf("Unexpected parser error: %v", err)
+			continue
+		}
+
+		expectErr := strings.TrimSpace(test.err)
+		_, err = resolver.Resolve(ast)
+		if err != nil {
+			gotErr := strings.TrimSpace(fmt.Sprint(err))
+			if gotErr != expectErr {
+				t.Errorf("While parsing:\n%s\nGot error:\n%s\nExpected:\n%s", test.src, gotErr, expectErr)
+			}
+		} else if expectErr != success {
+			t.Errorf("While parsing:\n%s\nGot no error, expected error:\n%s", test.src, expectErr)
+		}
+	}
+}

tools/src/cmd/builtin-gen/sem/sem.go

@@ -0,0 +1,283 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sem
+
+import (
+	"fmt"
+
+	"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/ast"
+)
+
+// Sem is the root of the semantic tree
+type Sem struct {
+	Enums        []*Enum
+	Types        []*Type
+	TypeMatchers []*TypeMatcher
+	EnumMatchers []*EnumMatcher
+	Functions    []*Function
+	// Maximum number of open-types used across all builtins
+	MaxOpenTypes int
+	// Maximum number of open-numbers used across all builtins
+	MaxOpenNumbers int
+	// The alphabetically sorted list of unique parameter names
+	UniqueParameterNames []string
+}
+
+// New returns a new Sem
+func New() *Sem {
+	return &Sem{
+		Enums:        []*Enum{},
+		Types:        []*Type{},
+		TypeMatchers: []*TypeMatcher{},
+		EnumMatchers: []*EnumMatcher{},
+		Functions:    []*Function{},
+	}
+}
+
+// Enum describes an enumerator
+type Enum struct {
+	Decl    ast.EnumDecl
+	Name    string
+	Entries []*EnumEntry
+}
+
+// FindEntry returns the enum entry with the given name
+func (e *Enum) FindEntry(name string) *EnumEntry {
+	for _, entry := range e.Entries {
+		if entry.Name == name {
+			return entry
+		}
+	}
+	return nil
+}
+
+// EnumEntry is an entry in an enumerator
+type EnumEntry struct {
+	Enum       *Enum
+	Name       string
+	IsInternal bool // True if this entry is not part of the WGSL grammar
+}
+
+// Format implements the fmt.Formatter interface
+func (e EnumEntry) Format(w fmt.State, verb rune) {
+	if e.IsInternal {
+		fmt.Fprint(w, "[[internal]] ")
+	}
+	fmt.Fprint(w, e.Name)
+}
+
+// Type declares a type
+type Type struct {
+	TemplateParams []TemplateParam
+	Decl           ast.TypeDecl
+	Name           string
+	DisplayName    string
+}
+
+// TypeMatcher declares a type matcher
+type TypeMatcher struct {
+	TemplateParams []TemplateParam
+	Decl           ast.MatcherDecl
+	Name           string
+	Types          []*Type
+}
+
+// EnumMatcher declares a enum matcher
+type EnumMatcher struct {
+	TemplateParams []TemplateParam
+	Decl           ast.MatcherDecl
+	Name           string
+	Enum           *Enum
+	Options        []*EnumEntry
+}
+
+// TemplateEnumParam is a template enum parameter
+type TemplateEnumParam struct {
+	Name    string
+	Enum    *Enum
+	Matcher *EnumMatcher // Optional
+}
+
+// TemplateTypeParam is a template type parameter
+type TemplateTypeParam struct {
+	Name string
+	Type ResolvableType
+}
+
+// TemplateNumberParam is a template type parameter
+type TemplateNumberParam struct {
+	Name string
+}
+
+// Function describes the overloads of a builtin function
+type Function struct {
+	Name      string
+	Overloads []*Overload
+}
+
+// Overload describes a single overload of a function
+type Overload struct {
+	Decl             ast.FunctionDecl
+	Function         *Function
+	TemplateParams   []TemplateParam
+	OpenTypes        []*TemplateTypeParam
+	OpenNumbers      []TemplateParam
+	ReturnType       *FullyQualifiedName
+	Parameters       []Parameter
+	CanBeUsedInStage StageUses
+	IsDeprecated     bool // True if this overload is deprecated
+}
+
+// StageUses describes the stages an overload can be used in
+type StageUses struct {
+	Vertex   bool
+	Fragment bool
+	Compute  bool
+}
+
+// List returns the stage uses as a string list
+func (u StageUses) List() []string {
+	out := []string{}
+	if u.Vertex {
+		out = append(out, "vertex")
+	}
+	if u.Fragment {
+		out = append(out, "fragment")
+	}
+	if u.Compute {
+		out = append(out, "compute")
+	}
+	return out
+}
+
+// Format implements the fmt.Formatter interface
+func (o Overload) Format(w fmt.State, verb rune) {
+	fmt.Fprintf(w, "fn %v", o.Function.Name)
+	if len(o.TemplateParams) > 0 {
+		fmt.Fprintf(w, "<")
+		for i, t := range o.TemplateParams {
+			if i > 0 {
+				fmt.Fprint(w, ", ")
+			}
+			fmt.Fprintf(w, "%v", t)
+		}
+		fmt.Fprintf(w, ">")
+	}
+	fmt.Fprint(w, "(")
+	for i, p := range o.Parameters {
+		if i > 0 {
+			fmt.Fprint(w, ", ")
+		}
+		fmt.Fprintf(w, "%v", p)
+	}
+	fmt.Fprint(w, ")")
+	if o.ReturnType != nil {
+		fmt.Fprintf(w, " -> %v", o.ReturnType)
+	}
+}
+
+// Parameter describes a single parameter of a function overload
+type Parameter struct {
+	Name string
+	Type FullyQualifiedName
+}
+
+// Format implements the fmt.Formatter interface
+func (p Parameter) Format(w fmt.State, verb rune) {
+	if p.Name != "" {
+		fmt.Fprintf(w, "%v: ", p.Name)
+	}
+	fmt.Fprintf(w, "%v", p.Type)
+}
+
+// FullyQualifiedName is the usage of a Type, TypeMatcher or TemplateTypeParam
+type FullyQualifiedName struct {
+	Target            Named
+	TemplateArguments []interface{}
+}
+
+// Format implements the fmt.Formatter interface
+func (f FullyQualifiedName) Format(w fmt.State, verb rune) {
+	fmt.Fprint(w, f.Target.GetName())
+	if len(f.TemplateArguments) > 0 {
+		fmt.Fprintf(w, "<")
+		for i, t := range f.TemplateArguments {
+			if i > 0 {
+				fmt.Fprint(w, ", ")
+			}
+			fmt.Fprintf(w, "%v", t)
+		}
+		fmt.Fprintf(w, ">")
+	}
+}
+
+// TemplateParam is a TemplateEnumParam, TemplateTypeParam or TemplateNumberParam
+type TemplateParam interface {
+	Named
+	isTemplateParam()
+}
+
+func (*TemplateEnumParam) isTemplateParam()   {}
+func (*TemplateTypeParam) isTemplateParam()   {}
+func (*TemplateNumberParam) isTemplateParam() {}
+
+// ResolvableType is a Type, TypeMatcher or TemplateTypeParam
+type ResolvableType interface {
+	Named
+	isResolvableType()
+}
+
+func (*Type) isResolvableType()              {}
+func (*TypeMatcher) isResolvableType()       {}
+func (*TemplateTypeParam) isResolvableType() {}
+
+// Named is something that can be looked up by name
+type Named interface {
+	isNamed()
+	GetName() string
+}
+
+func (*Enum) isNamed()                {}
+func (*EnumEntry) isNamed()           {}
+func (*Type) isNamed()                {}
+func (*TypeMatcher) isNamed()         {}
+func (*EnumMatcher) isNamed()         {}
+func (*TemplateTypeParam) isNamed()   {}
+func (*TemplateEnumParam) isNamed()   {}
+func (*TemplateNumberParam) isNamed() {}
+
+// GetName returns the name of the Enum
+func (e *Enum) GetName() string { return e.Name }
+
+// GetName returns the name of the EnumEntry
+func (e *EnumEntry) GetName() string { return e.Name }
+
+// GetName returns the name of the Type
+func (t *Type) GetName() string { return t.Name }
+
+// GetName returns the name of the TypeMatcher
+func (t *TypeMatcher) GetName() string { return t.Name }
+
+// GetName returns the name of the EnumMatcher
+func (e *EnumMatcher) GetName() string { return e.Name }
+
+// GetName returns the name of the TemplateTypeParam
+func (t *TemplateTypeParam) GetName() string { return t.Name }
+
+// GetName returns the name of the TemplateEnumParam
+func (t *TemplateEnumParam) GetName() string { return t.Name }
+
+// GetName returns the name of the TemplateNumberParam
+func (t *TemplateNumberParam) GetName() string { return t.Name }

tools/src/cmd/builtin-gen/tok/tok.go

@@ -0,0 +1,119 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package tok defines tokens that are produced by the Tint intrinsic definition
+// lexer
+package tok
+
+import "fmt"
+
+// Kind is an enumerator of token kinds
+type Kind string
+
+// Token enumerator types
+const (
+	InvalidToken Kind = "<invalid>"
+	Identifier   Kind = "ident"
+	Integer      Kind = "integer"
+	String       Kind = "string"
+	Match        Kind = "match"
+	Function     Kind = "fn"
+	Type         Kind = "type"
+	Enum         Kind = "enum"
+	Colon        Kind = ":"
+	Comma        Kind = ","
+	Lt           Kind = "<"
+	Gt           Kind = ">"
+	Lbrace       Kind = "{"
+	Rbrace       Kind = "}"
+	Ldeco        Kind = "[["
+	Rdeco        Kind = "]]"
+	Lparen       Kind = "("
+	Rparen       Kind = ")"
+	Or           Kind = "|"
+	Arrow        Kind = "->"
+)
+
+// Invalid represents an invalid token
+var Invalid = Token{Kind: InvalidToken}
+
+// Location describes a rune location in the source code
+type Location struct {
+	// 1-based line index
+	Line int
+	// 1-based column index
+	Column int
+	// 0-based rune index
+	Rune int
+	// Optional file path
+	Filepath string
+}
+
+// Format implements the fmt.Formatter interface
+func (l Location) Format(w fmt.State, verb rune) {
+	if w.Flag('+') {
+		if l.Filepath != "" {
+			fmt.Fprintf(w, "%v:%v:%v[%v]", l.Filepath, l.Line, l.Column, l.Rune)
+		} else {
+			fmt.Fprintf(w, "%v:%v[%v]", l.Line, l.Column, l.Rune)
+		}
+	} else {
+		if l.Filepath != "" {
+			fmt.Fprintf(w, "%v:%v:%v", l.Filepath, l.Line, l.Column)
+		} else {
+			fmt.Fprintf(w, "%v:%v", l.Line, l.Column)
+		}
+	}
+}
+
+// Source describes a start and end range in the source code
+type Source struct {
+	S, E Location
+}
+
+// IsValid returns true if the source is valid
+func (s Source) IsValid() bool {
+	return s.S.Line != 0 && s.S.Column != 0 && s.E.Line != 0 && s.E.Column != 0
+}
+
+// Format implements the fmt.Formatter interface
+func (s Source) Format(w fmt.State, verb rune) {
+	if w.Flag('+') {
+		fmt.Fprint(w, "[")
+		s.S.Format(w, verb)
+		fmt.Fprint(w, " - ")
+		s.E.Format(w, verb)
+		fmt.Fprint(w, "]")
+	} else {
+		s.S.Format(w, verb)
+	}
+}
+
+// Token describes a parsed token
+type Token struct {
+	Kind   Kind
+	Runes  []rune
+	Source Source
+}
+
+// Format implements the fmt.Formatter interface
+func (t Token) Format(w fmt.State, verb rune) {
+	fmt.Fprint(w, "[")
+	t.Source.Format(w, verb)
+	fmt.Fprint(w, " ")
+	fmt.Fprint(w, t.Kind)
+	fmt.Fprint(w, " ")
+	fmt.Fprint(w, string(t.Runes))
+	fmt.Fprint(w, "]")
+}

tools/src/cmd/check-spec-examples/main.go

@@ -0,0 +1,314 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// check-spec-examples tests that WGSL specification examples compile as
+// expected.
+//
+// The tool parses the WGSL HTML specification from the web or from a local file
+// and then runs the WGSL compiler for all examples annotated with the 'wgsl'
+// and 'global-scope' or 'function-scope' HTML class types.
+//
+// To run:
+//   go get golang.org/x/net/html # Only required once
+//   go run tools/check-spec-examples/main.go --compiler=<path-to-tint>
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"golang.org/x/net/html"
+)
+
+const (
+	toolName        = "check-spec-examples"
+	defaultSpecPath = "https://gpuweb.github.io/gpuweb/wgsl/"
+)
+
+var (
+	errInvalidArg = errors.New("Invalid arguments")
+)
+
+func main() {
+	flag.Usage = func() {
+		out := flag.CommandLine.Output()
+		fmt.Fprintf(out, "%v tests that WGSL specification examples compile as expected.\n", toolName)
+		fmt.Fprintf(out, "\n")
+		fmt.Fprintf(out, "Usage:\n")
+		fmt.Fprintf(out, "  %s [spec] [flags]\n", toolName)
+		fmt.Fprintf(out, "\n")
+		fmt.Fprintf(out, "spec is an optional local file path or URL to the WGSL specification.\n")
+		fmt.Fprintf(out, "If spec is omitted then the specification is fetched from %v\n", defaultSpecPath)
+		fmt.Fprintf(out, "\n")
+		fmt.Fprintf(out, "flags may be any combination of:\n")
+		flag.PrintDefaults()
+	}
+
+	err := run()
+	switch err {
+	case nil:
+		return
+	case errInvalidArg:
+		fmt.Fprintf(os.Stderr, "Error: %v\n\n", err)
+		flag.Usage()
+	default:
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+	}
+	os.Exit(1)
+}
+
+func run() error {
+	// Parse flags
+	compilerPath := flag.String("compiler", "tint", "path to compiler executable")
+	verbose := flag.Bool("verbose", false, "print examples that pass")
+	flag.Parse()
+
+	// Try to find the WGSL compiler
+	compiler, err := exec.LookPath(*compilerPath)
+	if err != nil {
+		return fmt.Errorf("Failed to find WGSL compiler: %w", err)
+	}
+	if compiler, err = filepath.Abs(compiler); err != nil {
+		return fmt.Errorf("Failed to find WGSL compiler: %w", err)
+	}
+
+	// Check for explicit WGSL spec path
+	args := flag.Args()
+	specURL, _ := url.Parse(defaultSpecPath)
+	switch len(args) {
+	case 0:
+	case 1:
+		var err error
+		specURL, err = url.Parse(args[0])
+		if err != nil {
+			return err
+		}
+	default:
+		if len(args) > 1 {
+			return errInvalidArg
+		}
+	}
+
+	// The specURL might just be a local file path, in which case automatically
+	// add the 'file' URL scheme
+	if specURL.Scheme == "" {
+		specURL.Scheme = "file"
+	}
+
+	// Open the spec from HTTP(S) or from a local file
+	var specContent io.ReadCloser
+	switch specURL.Scheme {
+	case "http", "https":
+		response, err := http.Get(specURL.String())
+		if err != nil {
+			return fmt.Errorf("Failed to load the WGSL spec from '%v': %w", specURL, err)
+		}
+		specContent = response.Body
+	case "file":
+		specURL.Path, err = filepath.Abs(specURL.Path)
+		if err != nil {
+			return fmt.Errorf("Failed to load the WGSL spec from '%v': %w", specURL, err)
+		}
+
+		file, err := os.Open(specURL.Path)
+		if err != nil {
+			return fmt.Errorf("Failed to load the WGSL spec from '%v': %w", specURL, err)
+		}
+		specContent = file
+	default:
+		return fmt.Errorf("Unsupported URL scheme: %v", specURL.Scheme)
+	}
+	defer specContent.Close()
+
+	// Create the HTML parser
+	doc, err := html.Parse(specContent)
+	if err != nil {
+		return err
+	}
+
+	// Parse all the WGSL examples
+	examples := []example{}
+	if err := gatherExamples(doc, &examples); err != nil {
+		return err
+	}
+
+	if len(examples) == 0 {
+		return fmt.Errorf("no examples found")
+	}
+
+	// Create a temporary directory to hold the examples as separate files
+	tmpDir, err := ioutil.TempDir("", "wgsl-spec-examples")
+	if err != nil {
+		return err
+	}
+	if err := os.MkdirAll(tmpDir, 0666); err != nil {
+		return fmt.Errorf("Failed to create temporary directory: %w", err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	// For each compilable WGSL example...
+	for _, e := range examples {
+		exampleURL := specURL.String() + "#" + e.name
+
+		if err := tryCompile(compiler, tmpDir, e); err != nil {
+			if !e.expectError {
+				fmt.Printf("✘ %v ✘\n%v\n", exampleURL, err)
+				continue
+			}
+		} else if e.expectError {
+			fmt.Printf("✘ %v ✘\nCompiled even though it was marked with 'expect-error'\n", exampleURL)
+		}
+		if *verbose {
+			fmt.Printf("✔ %v ✔\n", exampleURL)
+		}
+	}
+	return nil
+}
+
+// Holds all the information about a single, compilable WGSL example in the spec
+type example struct {
+	name          string // The name (typically hash generated by bikeshed)
+	code          string // The example source
+	globalScope   bool   // Annotated with 'global-scope' ?
+	functionScope bool   // Annotated with 'function-scope' ?
+	expectError   bool   // Annotated with 'expect-error' ?
+}
+
+// tryCompile attempts to compile the example e in the directory wd, using the
+// compiler at the given path. If the example is annotated with 'function-scope'
+// then the code is wrapped with a basic vertex-stage-entry function.
+// If the first compile fails then a placeholder vertex-state-entry
+// function is appended to the source, and another attempt to compile
+// the shader is made.
+func tryCompile(compiler, wd string, e example) error {
+	code := e.code
+	if e.functionScope {
+		code = "\n@stage(vertex) fn main() -> @builtin(position) vec4<f32> {\n" + code + " return vec4<f32>();}\n"
+	}
+
+	addedStubFunction := false
+	for {
+		err := compile(compiler, wd, e.name, code)
+		if err == nil {
+			return nil
+		}
+
+		if !addedStubFunction {
+			code += "\n@stage(vertex) fn main() {}\n"
+			addedStubFunction = true
+			continue
+		}
+
+		return err
+	}
+}
+
+// compile creates a file in wd and uses the compiler to attempt to compile it.
+func compile(compiler, wd, name, code string) error {
+	filename := name + ".wgsl"
+	path := filepath.Join(wd, filename)
+	if err := ioutil.WriteFile(path, []byte(code), 0666); err != nil {
+		return fmt.Errorf("Failed to write example file '%v'", path)
+	}
+	cmd := exec.Command(compiler, filename)
+	cmd.Dir = wd
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("%v\n%v", err, string(out))
+	}
+	return nil
+}
+
+// gatherExamples scans the HTML node and its children for blocks that contain
+// WGSL example code, populating the examples slice.
+func gatherExamples(node *html.Node, examples *[]example) error {
+	if hasClass(node, "example") && hasClass(node, "wgsl") {
+		e := example{
+			name:          nodeID(node),
+			code:          exampleCode(node),
+			globalScope:   hasClass(node, "global-scope"),
+			functionScope: hasClass(node, "function-scope"),
+			expectError:   hasClass(node, "expect-error"),
+		}
+		// If the example is annotated with a scope, then it can be compiled.
+		if e.globalScope || e.functionScope {
+			*examples = append(*examples, e)
+		}
+	}
+	for child := node.FirstChild; child != nil; child = child.NextSibling {
+		if err := gatherExamples(child, examples); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// exampleCode returns a string formed from all the TextNodes found in <pre>
+// blocks that are children of node.
+func exampleCode(node *html.Node) string {
+	sb := strings.Builder{}
+	for child := node.FirstChild; child != nil; child = child.NextSibling {
+		if child.Data == "pre" {
+			printNodeText(child, &sb)
+		}
+	}
+	return sb.String()
+}
+
+// printNodeText traverses node and its children, writing the Data of all
+// TextNodes to sb.
+func printNodeText(node *html.Node, sb *strings.Builder) {
+	if node.Type == html.TextNode {
+		sb.WriteString(node.Data)
+	}
+
+	for child := node.FirstChild; child != nil; child = child.NextSibling {
+		printNodeText(child, sb)
+	}
+}
+
+// hasClass returns true iff node is has the given "class" attribute.
+func hasClass(node *html.Node, class string) bool {
+	for _, attr := range node.Attr {
+		if attr.Key == "class" {
+			classes := strings.Split(attr.Val, " ")
+			for _, c := range classes {
+				if c == class {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}
+
+// nodeID returns the given "id" attribute of node, or an empty string if there
+// is no "id" attribute.
+func nodeID(node *html.Node) string {
+	for _, attr := range node.Attr {
+		if attr.Key == "id" {
+			return attr.Val
+		}
+	}
+	return ""
+}

tools/src/cmd/fix-tests/main.go

@@ -0,0 +1,340 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// fix-tests is a tool to update tests with new expected output.
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"dawn.googlesource.com/tint/tools/src/substr"
+)
+
+func main() {
+	if err := run(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func showUsage() {
+	fmt.Println(`
+fix-tests is a tool to update tests with new expected output.
+
+fix-tests performs string matching and heuristics to fix up expected results of
+tests that use EXPECT_EQ(a, b) and EXPECT_THAT(a, HasSubstr(b))
+
+WARNING: Always thoroughly check the generated output for mistakes.
+This may produce incorrect output
+
+Usage:
+  fix-tests <executable>
+
+  executable         - the path to the test executable to run.`)
+	os.Exit(1)
+}
+
+func run() error {
+	flag.Parse()
+	args := flag.Args()
+	if len(args) < 1 {
+		showUsage()
+	}
+
+	exe := args[0]          // The path to the test executable
+	wd := filepath.Dir(exe) // The directory holding the test exe
+
+	// Create a temporary directory to hold the 'test-results.json' file
+	tmpDir, err := ioutil.TempDir("", "fix-tests")
+	if err != nil {
+		return err
+	}
+	if err := os.MkdirAll(tmpDir, 0666); err != nil {
+		return fmt.Errorf("Failed to create temporary directory: %w", err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	// Full path to the 'test-results.json' in the temporary directory
+	testResultsPath := filepath.Join(tmpDir, "test-results.json")
+
+	// Run the tests
+	testArgs := []string{"--gtest_output=json:" + testResultsPath}
+	if len(args) > 1 {
+		testArgs = append(testArgs, args[1:]...)
+	}
+	switch err := exec.Command(exe, testArgs...).Run().(type) {
+	default:
+		return err
+	case nil:
+		fmt.Println("All tests passed")
+	case *exec.ExitError:
+	}
+
+	// Read the 'test-results.json' file
+	testResultsFile, err := os.Open(testResultsPath)
+	if err != nil {
+		return err
+	}
+
+	var testResults Results
+	if err := json.NewDecoder(testResultsFile).Decode(&testResults); err != nil {
+		return err
+	}
+
+	// For each failing test...
+	seen := map[string]bool{}
+	numFixed, numFailed := 0, 0
+	for _, group := range testResults.Groups {
+		for _, suite := range group.Testsuites {
+			for _, failure := range suite.Failures {
+				// .. attempt to fix the problem
+				test := testName(group, suite)
+				if seen[test] {
+					continue
+				}
+				seen[test] = true
+
+				if err := processFailure(test, wd, failure.Failure); err != nil {
+					fmt.Println(fmt.Errorf("%v: %w", test, err))
+					numFailed++
+				} else {
+					numFixed++
+				}
+			}
+		}
+	}
+
+	fmt.Println()
+
+	if numFailed > 0 {
+		fmt.Println(numFailed, "tests could not be fixed")
+	}
+	if numFixed > 0 {
+		fmt.Println(numFixed, "tests fixed")
+	}
+	return nil
+}
+
+func testName(group TestsuiteGroup, suite Testsuite) string {
+	groupParts := strings.Split(group.Name, "/")
+	suiteParts := strings.Split(suite.Name, "/")
+	return groupParts[len(groupParts)-1] + "." + suiteParts[0]
+}
+
+var (
+	// Regular expression to match a test declaration
+	reTests = regexp.MustCompile(`TEST(?:_[FP])?\([ \n]*(\w+),[ \n]*(\w+)\)`)
+	// Regular expression to match a `EXPECT_EQ(a, b)` failure for strings
+	reExpectEq = regexp.MustCompile(`([./\\\w_\-:]*):(\d+).*\nExpected equality of these values:\n(?:.|\n)*?(?:Which is: |  )"((?:.|\n)*?[^\\])"\n(?:.|\n)*?(?:Which is: |  )"((?:.|\n)*?[^\\])"`)
+	// Regular expression to match a `EXPECT_THAT(a, HasSubstr(b))` failure for strings
+	reExpectHasSubstr = regexp.MustCompile(`([./\\\w_\-:]*):(\d+).*\nValue of: .*\nExpected: has substring "((?:.|\n)*?[^\\])"\n  Actual: "((?:.|\n)*?[^\\])"`)
+)
+
+func processFailure(test, wd, failure string) error {
+	// Start by un-escaping newlines in the failure message
+	failure = strings.ReplaceAll(failure, "\\n", "\n")
+	// Matched regex strings will also need to be un-escaped, but do this after
+	// the match, as unescaped quotes may upset the regex patterns
+	unescape := func(s string) string {
+		return strings.ReplaceAll(s, `\"`, `"`)
+	}
+	escape := func(s string) string {
+		s = strings.ReplaceAll(s, "\n", `\n`)
+		s = strings.ReplaceAll(s, "\"", `\"`)
+		return s
+	}
+
+	// Look for a EXPECT_EQ failure pattern
+	var file string
+	var fix func(testSource string) (string, error)
+	if parts := reExpectEq.FindStringSubmatch(failure); len(parts) == 5 {
+		// EXPECT_EQ(a, b)
+		a, b := unescape(parts[3]), unescape(parts[4])
+		file = parts[1]
+		fix = func(testSource string) (string, error) {
+			// We don't know if a or b is the expected, so just try flipping the string
+			// to the other form.
+
+			if len(b) > len(a) { // Go with the longer match, in case both are found
+				a, b = b, a
+			}
+			switch {
+			case strings.Contains(testSource, a):
+				testSource = strings.ReplaceAll(testSource, a, b)
+			case strings.Contains(testSource, b):
+				testSource = strings.ReplaceAll(testSource, b, a)
+			default:
+				// Try escaping for R"(...)" strings
+				a, b = escape(a), escape(b)
+				switch {
+				case strings.Contains(testSource, a):
+					testSource = strings.ReplaceAll(testSource, a, b)
+				case strings.Contains(testSource, b):
+					testSource = strings.ReplaceAll(testSource, b, a)
+				default:
+					return "", fmt.Errorf("Could not fix 'EXPECT_EQ' pattern in '%v'", file)
+				}
+			}
+			return testSource, nil
+		}
+	} else if parts := reExpectHasSubstr.FindStringSubmatch(failure); len(parts) == 5 {
+		// EXPECT_THAT(a, HasSubstr(b))
+		a, b := unescape(parts[4]), unescape(parts[3])
+		file = parts[1]
+		fix = func(testSource string) (string, error) {
+			if fix := substr.Fix(a, b); fix != "" {
+				if !strings.Contains(testSource, b) {
+					// Try escaping for R"(...)" strings
+					b, fix = escape(b), escape(fix)
+				}
+				if strings.Contains(testSource, b) {
+					testSource = strings.Replace(testSource, b, fix, -1)
+					return testSource, nil
+				}
+				return "", fmt.Errorf("Could apply fix for 'HasSubstr' pattern in '%v'", file)
+			}
+
+			return "", fmt.Errorf("Could find fix for 'HasSubstr' pattern in '%v'", file)
+		}
+	} else {
+		return fmt.Errorf("Cannot fix this type of failure")
+	}
+
+	// Get the absolute source path
+	sourcePath := file
+	if !filepath.IsAbs(sourcePath) {
+		sourcePath = filepath.Join(wd, file)
+	}
+
+	// Parse the source file, split into tests
+	sourceFile, err := parseSourceFile(sourcePath)
+	if err != nil {
+		return fmt.Errorf("Couldn't parse tests from file '%v': %w", file, err)
+	}
+
+	// Find the test
+	testIdx, ok := sourceFile.tests[test]
+	if !ok {
+		return fmt.Errorf("Test not found in '%v'", file)
+	}
+
+	// Grab the source for the particular test
+	testSource := sourceFile.parts[testIdx]
+
+	if testSource, err = fix(testSource); err != nil {
+		return err
+	}
+
+	// Replace the part of the source file
+	sourceFile.parts[testIdx] = testSource
+
+	// Write out the source file
+	return writeSourceFile(sourcePath, sourceFile)
+}
+
+// parseSourceFile() reads the file at path, splitting the content into chunks
+// for each TEST.
+func parseSourceFile(path string) (sourceFile, error) {
+	fileBytes, err := ioutil.ReadFile(path)
+	if err != nil {
+		return sourceFile{}, err
+	}
+	fileContent := string(fileBytes)
+
+	out := sourceFile{
+		tests: map[string]int{},
+	}
+
+	pos := 0
+	for _, span := range reTests.FindAllStringIndex(fileContent, -1) {
+		out.parts = append(out.parts, fileContent[pos:span[0]])
+		pos = span[0]
+
+		match := reTests.FindStringSubmatch(fileContent[span[0]:span[1]])
+		group := match[1]
+		suite := match[2]
+		out.tests[group+"."+suite] = len(out.parts)
+	}
+	out.parts = append(out.parts, fileContent[pos:])
+
+	return out, nil
+}
+
+// writeSourceFile() joins the chunks of the file, and writes the content out to
+// path.
+func writeSourceFile(path string, file sourceFile) error {
+	body := strings.Join(file.parts, "")
+	return ioutil.WriteFile(path, []byte(body), 0666)
+}
+
+type sourceFile struct {
+	parts []string
+	tests map[string]int // "X.Y" -> part index
+}
+
+// Results is the root JSON structure of the JSON --gtest_output file .
+type Results struct {
+	Tests     int              `json:"tests"`
+	Failures  int              `json:"failures"`
+	Disabled  int              `json:"disabled"`
+	Errors    int              `json:"errors"`
+	Timestamp string           `json:"timestamp"`
+	Time      string           `json:"time"`
+	Name      string           `json:"name"`
+	Groups    []TestsuiteGroup `json:"testsuites"`
+}
+
+// TestsuiteGroup is a group of test suites in the JSON --gtest_output file .
+type TestsuiteGroup struct {
+	Name       string      `json:"name"`
+	Tests      int         `json:"tests"`
+	Failures   int         `json:"failures"`
+	Disabled   int         `json:"disabled"`
+	Errors     int         `json:"errors"`
+	Timestamp  string      `json:"timestamp"`
+	Time       string      `json:"time"`
+	Testsuites []Testsuite `json:"testsuite"`
+}
+
+// Testsuite is a suite of tests in the JSON --gtest_output file.
+type Testsuite struct {
+	Name       string    `json:"name"`
+	ValueParam string    `json:"value_param,omitempty"`
+	Status     Status    `json:"status"`
+	Result     Result    `json:"result"`
+	Timestamp  string    `json:"timestamp"`
+	Time       string    `json:"time"`
+	Classname  string    `json:"classname"`
+	Failures   []Failure `json:"failures,omitempty"`
+}
+
+// Failure is a reported test failure in the JSON --gtest_output file.
+type Failure struct {
+	Failure string `json:"failure"`
+	Type    string `json:"type"`
+}
+
+// Status is a status code in the JSON --gtest_output file.
+type Status string
+
+// Result is a result code in the JSON --gtest_output file.
+type Result string

tools/src/cmd/gerrit-stats/main.go

@@ -0,0 +1,152 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// gerrit-stats gathers statistics about changes made to Tint.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"net/url"
+	"os"
+	"regexp"
+	"time"
+
+	"dawn.googlesource.com/tint/tools/src/gerrit"
+)
+
+const yyyymmdd = "2006-01-02"
+
+var (
+	// See https://dawn-review.googlesource.com/new-password for obtaining
+	// username and password for gerrit.
+	gerritUser  = flag.String("gerrit-user", "", "gerrit authentication username")
+	gerritPass  = flag.String("gerrit-pass", "", "gerrit authentication password")
+	repoFlag    = flag.String("repo", "tint", "the project (tint or dawn)")
+	userFlag    = flag.String("user", "", "user name / email")
+	afterFlag   = flag.String("after", "", "start date")
+	beforeFlag  = flag.String("before", "", "end date")
+	daysFlag    = flag.Int("days", 182, "interval in days (used if --after is not specified)")
+	verboseFlag = flag.Bool("v", false, "verbose mode - lists all the changes")
+)
+
+func main() {
+	flag.Parse()
+	if err := run(); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+func run() error {
+	var after, before time.Time
+	var err error
+	user := *userFlag
+	if user == "" {
+		return fmt.Errorf("Missing required 'user' flag")
+	}
+	if *beforeFlag != "" {
+		before, err = time.Parse(yyyymmdd, *beforeFlag)
+		if err != nil {
+			return fmt.Errorf("Couldn't parse before date: %w", err)
+		}
+	} else {
+		before = time.Now()
+	}
+	if *afterFlag != "" {
+		after, err = time.Parse(yyyymmdd, *afterFlag)
+		if err != nil {
+			return fmt.Errorf("Couldn't parse after date: %w", err)
+		}
+	} else {
+		after = before.Add(-time.Hour * time.Duration(24**daysFlag))
+	}
+
+	g, err := gerrit.New(gerrit.Config{Username: *gerritUser, Password: *gerritPass})
+	if err != nil {
+		return err
+	}
+
+	submitted, submittedQuery, err := g.QueryChanges(
+		"status:merged",
+		"owner:"+user,
+		"after:"+date(after),
+		"before:"+date(before),
+		"repo:"+*repoFlag)
+	if err != nil {
+		return fmt.Errorf("Query failed: %w", err)
+	}
+
+	reviewed, reviewQuery, err := g.QueryChanges(
+		"commentby:"+user,
+		"-owner:"+user,
+		"after:"+date(after),
+		"before:"+date(before),
+		"repo:"+*repoFlag)
+	if err != nil {
+		return fmt.Errorf("Query failed: %w", err)
+	}
+
+	ignorelist := []*regexp.Regexp{
+		regexp.MustCompile("Revert .*"),
+	}
+	ignore := func(s string) bool {
+		for _, re := range ignorelist {
+			if re.MatchString(s) {
+				return true
+			}
+		}
+		return false
+	}
+
+	insertions, deletions := 0, 0
+	for _, change := range submitted {
+		if ignore(change.Subject) {
+			continue
+		}
+		insertions += change.Insertions
+		deletions += change.Deletions
+	}
+
+	fmt.Printf("Between %v and %v, %v:\n", date(after), date(before), user)
+	fmt.Printf("  Submitted %v changes (LOC: %v+, %v-) \n", len(submitted), insertions, deletions)
+	fmt.Printf("  Reviewed %v changes\n", len(reviewed))
+	fmt.Printf("\n")
+
+	if *verboseFlag {
+		fmt.Printf("Submitted changes:\n")
+		for i, change := range submitted {
+			fmt.Printf("%3.1v: %6.v %v (LOC: %v+, %v-)\n", i, change.Number, change.Subject, change.Insertions, change.Deletions)
+		}
+		fmt.Printf("\n")
+		fmt.Printf("Reviewed changes:\n")
+		for i, change := range reviewed {
+			fmt.Printf("%3.1v: %6.v %v (LOC: %v+, %v-)\n", i, change.Number, change.Subject, change.Insertions, change.Deletions)
+		}
+	}
+
+	fmt.Printf("\n")
+	fmt.Printf("Submitted query: %vq/%v\n", gerrit.URL, url.QueryEscape(submittedQuery))
+	fmt.Printf("Review query: %vq/%v\n", gerrit.URL, url.QueryEscape(reviewQuery))
+
+	return nil
+}
+
+func today() time.Time {
+	return time.Now()
+}
+
+func date(t time.Time) string {
+	return t.Format(yyyymmdd)
+}

tools/src/cmd/perfmon/main.go

@@ -0,0 +1,937 @@
+// Copyright 2022 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"reflect"
+	"sort"
+	"strings"
+	"time"
+
+	"dawn.googlesource.com/tint/tools/src/bench"
+	"dawn.googlesource.com/tint/tools/src/git"
+	"github.com/andygrunwald/go-gerrit"
+	"github.com/go-git/go-git/v5/plumbing/transport"
+	"github.com/go-git/go-git/v5/plumbing/transport/http"
+	"github.com/shirou/gopsutil/cpu"
+)
+
+// main entry point
+func main() {
+	var cfgPath string
+	flag.StringVar(&cfgPath, "c", "~/.config/perfmon/config.json", "the config file")
+	flag.Parse()
+
+	if err := run(cfgPath); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+// run starts the perfmon tool with the given config path
+func run(cfgPath string) error {
+	cfgPath, err := expandHomeDir(cfgPath)
+	if err != nil {
+		return err
+	}
+
+	if err := findTools(); err != nil {
+		return err
+	}
+
+	g, err := git.New(tools.git)
+	if err != nil {
+		return err
+	}
+
+	cfg, err := loadConfig(cfgPath)
+	if err != nil {
+		return err
+	}
+
+	tintDir, resultsDir, err := makeWorkingDirs(cfg)
+	if err != nil {
+		return err
+	}
+	tintRepo, err := createOrOpenGitRepo(g, tintDir, cfg.Tint)
+	if err != nil {
+		return err
+	}
+	resultsRepo, err := createOrOpenGitRepo(g, resultsDir, cfg.Results)
+	if err != nil {
+		return err
+	}
+	gerritClient, err := gerrit.NewClient(cfg.Gerrit.URL, nil)
+	if err != nil {
+		return err
+	}
+	gerritClient.Authentication.SetBasicAuth(cfg.Gerrit.Username, cfg.Gerrit.Password)
+
+	sysInfo, err := cpu.Info()
+	if err != nil {
+		return fmt.Errorf("failed to obtain system info:\n  %v", err)
+	}
+
+	e := env{
+		cfg:         cfg,
+		git:         g,
+		system:      sysInfo,
+		systemID:    hash(sysInfo)[:8],
+		tintDir:     tintDir,
+		buildDir:    filepath.Join(tintDir, "out"),
+		resultsDir:  resultsDir,
+		tintRepo:    tintRepo,
+		resultsRepo: resultsRepo,
+		gerrit:      gerritClient,
+
+		benchmarkCache: map[git.Hash]*bench.Run{},
+	}
+
+	for true {
+		didSomething, err := e.doSomeWork()
+		if err != nil {
+			log.Printf("ERROR: %v", err)
+			time.Sleep(time.Minute * 10)
+			continue
+		}
+		if !didSomething {
+			log.Println("nothing to do. Sleeping...")
+			time.Sleep(time.Minute * 5)
+		}
+	}
+
+	return nil
+}
+
+// Config holds the root configuration options for the perfmon tool
+type Config struct {
+	WorkingDir           string
+	RootChange           git.Hash
+	Tint                 GitConfig
+	Results              GitConfig
+	Gerrit               GerritConfig
+	Timeouts             TimeoutsConfig
+	ExternalAccounts     []string
+	BenchmarkRepetitions int
+}
+
+// GitConfig holds the configuration options for accessing a git repo
+type GitConfig struct {
+	URL    string
+	Branch string
+	Auth   git.Auth
+}
+
+// GerritConfig holds the configuration options for accessing gerrit
+type GerritConfig struct {
+	URL      string
+	Username string
+	Email    string
+	Password string
+}
+
+// TimeoutsConfig holds the configuration options for timeouts
+type TimeoutsConfig struct {
+	Sync      time.Duration
+	Build     time.Duration
+	Benchmark time.Duration
+}
+
+// HistoricResults contains the full set of historic benchmark results for a
+// given system
+type HistoricResults struct {
+	System  []cpu.InfoStat
+	Commits []CommitResults
+}
+
+// CommitResults holds the results of a single tint commit
+type CommitResults struct {
+	Commit            string
+	CommitTime        time.Time
+	CommitDescription string
+	Benchmarks        []Benchmark
+}
+
+// Benchmark holds the benchmark results for a single test
+type Benchmark struct {
+	Name   string
+	Mean   float64
+	Median float64
+	Stddev float64
+}
+
+// setDefaults assigns default values to unassigned fields of cfg
+func (cfg *Config) setDefaults() {
+	if cfg.RootChange.IsZero() {
+		cfg.RootChange, _ = git.ParseHash("be2362b18c792364c6bf5744db6d3837fbc655a0")
+	}
+	cfg.Tint.setDefaults()
+	cfg.Results.setDefaults()
+	cfg.Timeouts.setDefaults()
+	if cfg.BenchmarkRepetitions < 2 {
+		cfg.BenchmarkRepetitions = 2
+	}
+}
+
+// setDefaults assigns default values to unassigned fields of cfg
+func (cfg *GitConfig) setDefaults() {
+	if cfg.Branch == "" {
+		cfg.Branch = "main"
+	}
+}
+
+// setDefaults assigns default values to unassigned fields of cfg
+func (cfg *TimeoutsConfig) setDefaults() {
+	if cfg.Sync == 0 {
+		cfg.Sync = time.Minute * 10
+	}
+	if cfg.Build == 0 {
+		cfg.Build = time.Minute * 10
+	}
+	if cfg.Benchmark == 0 {
+		cfg.Benchmark = time.Minute * 30
+	}
+}
+
+// AuthConfig holds the authentication options for accessing a git repo
+type AuthConfig struct {
+	Username string
+	Password string
+}
+
+// authMethod returns a http.BasicAuth constructed from the AuthConfig
+func (cfg AuthConfig) authMethod() transport.AuthMethod {
+	if cfg.Username != "" || cfg.Password != "" {
+		return &http.BasicAuth{Username: cfg.Username, Password: cfg.Password}
+	}
+	return nil
+}
+
+// env holds the perfmon main environment state
+type env struct {
+	cfg         Config
+	git         *git.Git
+	system      []cpu.InfoStat
+	systemID    string
+	tintDir     string
+	buildDir    string
+	resultsDir  string
+	tintRepo    *git.Repository
+	resultsRepo *git.Repository
+	gerrit      *gerrit.Client
+
+	benchmarkCache map[git.Hash]*bench.Run
+}
+
+// doSomeWork scans gerrit for changes up for review and submitted changes to
+// benchmark. If something was found to do, then returns true.
+func (e env) doSomeWork() (bool, error) {
+	{
+		log.Println("scanning for review changes to benchmark...")
+		change, err := e.findGerritChangeToBenchmark()
+		if err != nil {
+			return true, err
+		}
+		if change != nil {
+			if err := e.benchmarkGerritChange(*change); err != nil {
+				return true, err
+			}
+			return true, nil
+		}
+	}
+
+	{
+		log.Println("scanning for submitted changes to benchmark...")
+		changesToBenchmark, err := e.changesToBenchmark()
+		if err != nil {
+			return true, err
+		}
+
+		if len(changesToBenchmark) > 0 {
+			log.Printf("benchmarking %v changes...", len(changesToBenchmark))
+			for i, c := range changesToBenchmark {
+				log.Printf("benchmarking %v/%v....", i+1, len(changesToBenchmark))
+				benchRes, err := e.benchmarkTintChange(c)
+				if err != nil {
+					return true, err
+				}
+				commitRes, err := e.benchmarksToCommitResults(c, *benchRes)
+				if err != nil {
+					return true, err
+				}
+				log.Printf("pushing results...")
+				if err := e.pushUpdatedResults(*commitRes); err != nil {
+					return true, err
+				}
+			}
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// changesToBenchmark fetches the list of changes that do not currently have
+// benchmark results, which should be benchmarked.
+func (e env) changesToBenchmark() ([]git.Hash, error) {
+	log.Println("syncing tint repo...")
+	latest, err := e.tintRepo.Fetch(e.cfg.Tint.Branch, &git.FetchOptions{Auth: e.cfg.Tint.Auth})
+	if err != nil {
+		return nil, err
+	}
+	allChanges, err := e.tintRepo.Log(&git.LogOptions{
+		From: e.cfg.RootChange.String(),
+		To:   latest.String(),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to obtain tint log:\n  %w", err)
+	}
+	changesWithBenchmarks, err := e.changesWithBenchmarks()
+	if err != nil {
+		return nil, fmt.Errorf("failed to gather changes with existing benchmarks:\n  %w", err)
+	}
+	changesToBenchmark := make([]git.Hash, 0, len(allChanges))
+	for _, c := range allChanges {
+		if _, exists := changesWithBenchmarks[c.Hash]; !exists {
+			changesToBenchmark = append(changesToBenchmark, c.Hash)
+		}
+	}
+	// Reverse the order of changesToBenchmark, so that the oldest comes first.
+	for i := len(changesToBenchmark)/2 - 1; i >= 0; i-- {
+		j := len(changesToBenchmark) - 1 - i
+		changesToBenchmark[i], changesToBenchmark[j] = changesToBenchmark[j], changesToBenchmark[i]
+	}
+
+	return changesToBenchmark, nil
+}
+
+// benchmarkTintChange checks out the given commit, fetches the tint third party
+// dependencies, builds tint, then runs the benchmarks, returning the results.
+func (e env) benchmarkTintChange(hash git.Hash) (*bench.Run, error) {
+	if cached, ok := e.benchmarkCache[hash]; ok {
+		log.Printf("reusing cached benchmark results of '%v'...", hash)
+		return cached, nil
+	}
+
+	log.Printf("checking out tint at '%v'...", hash)
+	if err := checkout(hash, e.tintRepo); err != nil {
+		return nil, err
+	}
+	log.Println("fetching tint dependencies...")
+	if err := e.fetchTintDeps(); err != nil {
+		return nil, err
+	}
+	log.Println("building tint...")
+	if err := e.buildTint(); err != nil {
+		return nil, err
+	}
+	log.Println("benchmarking tint...")
+	run, err := e.benchmarkTint()
+	if err != nil {
+		return nil, err
+	}
+
+	e.benchmarkCache[hash] = run
+	return run, nil
+}
+
+// benchmarksToCommitResults converts the benchmarks in the provided bench.Run
+// to a CommitResults.
+func (e env) benchmarksToCommitResults(hash git.Hash, results bench.Run) (*CommitResults, error) {
+	commits, err := e.tintRepo.Log(&git.LogOptions{
+		From:  hash.String(),
+		Count: 1,
+	})
+	if err != nil || len(commits) != 1 {
+		return nil, fmt.Errorf("failed to get commit object '%v' of tint repo:\n  %w", hash, err)
+	}
+	commit := commits[0]
+
+	m := map[string]Benchmark{}
+	for _, b := range results.Benchmarks {
+		benchmark := m[b.Name]
+		benchmark.Name = b.Name
+		switch b.AggregateType {
+		case bench.Mean:
+			benchmark.Mean = float64(b.Duration) / float64(time.Second)
+		case bench.Median:
+			benchmark.Median = float64(b.Duration) / float64(time.Second)
+		case bench.Stddev:
+			benchmark.Stddev = float64(b.Duration) / float64(time.Second)
+		}
+		m[b.Name] = benchmark
+	}
+
+	sorted := make([]Benchmark, 0, len(m))
+	for _, b := range m {
+		sorted = append(sorted, b)
+	}
+	sort.Slice(sorted, func(i, j int) bool { return sorted[i].Name < sorted[i].Name })
+
+	return &CommitResults{
+		Commit:            commit.Hash.String(),
+		CommitDescription: commit.Subject,
+		CommitTime:        commit.Date,
+		Benchmarks:        sorted,
+	}, nil
+}
+
+// changesWithBenchmarks returns a set of tint changes that we already have
+// benchmarks for.
+func (e env) changesWithBenchmarks() (map[git.Hash]struct{}, error) {
+	log.Println("syncing results repo...")
+	if err := fetchAndCheckoutLatest(e.resultsRepo, e.cfg.Results); err != nil {
+		return nil, err
+	}
+
+	_, absPath, err := e.resultsFilePaths()
+	if err != nil {
+		return nil, err
+	}
+
+	results, err := e.loadHistoricResults(absPath)
+	if err != nil {
+		log.Println(fmt.Errorf("WARNING: failed to open result file '%v':\n  %w", absPath, err))
+		return nil, nil
+	}
+
+	m := make(map[git.Hash]struct{}, len(results.Commits))
+	for _, c := range results.Commits {
+		hash, err := git.ParseHash(c.Commit)
+		if err != nil {
+			return nil, err
+		}
+		m[hash] = struct{}{}
+	}
+	return m, nil
+}
+
+func (e env) pushUpdatedResults(res CommitResults) error {
+	log.Println("syncing results repo...")
+	if err := fetchAndCheckoutLatest(e.resultsRepo, e.cfg.Results); err != nil {
+		return err
+	}
+
+	relPath, absPath, err := e.resultsFilePaths()
+	if err != nil {
+		return err
+	}
+
+	h, err := e.loadHistoricResults(absPath)
+	if err != nil {
+		log.Println(fmt.Errorf("failed to open result file '%v'. Creating new file\n  %w", absPath, err))
+		h = &HistoricResults{System: e.system}
+	}
+
+	h.Commits = append(h.Commits, res)
+
+	// Sort the commits by timestamp
+	sort.Slice(h.Commits, func(i, j int) bool { return h.Commits[i].CommitTime.Before(h.Commits[j].CommitTime) })
+
+	// Write the new results to the file
+	f, err := os.Create(absPath)
+	if err != nil {
+		return fmt.Errorf("failed to create updated results file '%v':\n  %w", absPath, err)
+	}
+	defer f.Close()
+
+	enc := json.NewEncoder(f)
+	enc.SetIndent("", "  ")
+	if err := enc.Encode(h); err != nil {
+		return fmt.Errorf("failed to encode updated results file '%v':\n  %w", absPath, err)
+	}
+
+	// Stage the file
+	if err := e.resultsRepo.Add(relPath, nil); err != nil {
+		return fmt.Errorf("failed to stage updated results file '%v':\n  %w", relPath, err)
+	}
+
+	// Commit the change
+	msg := fmt.Sprintf("Add benchmark results for '%v'", res.Commit[:6])
+	hash, err := e.resultsRepo.Commit(msg, &git.CommitOptions{
+		AuthorName:  "tint perfmon bot",
+		AuthorEmail: "tint-perfmon-bot@gmail.com",
+	})
+	if err != nil {
+		return fmt.Errorf("failed to commit updated results file '%v':\n  %w", absPath, err)
+	}
+
+	// Push the change
+	log.Println("pushing updated results to results repo...")
+	if err := e.resultsRepo.Push(hash.String(), e.cfg.Results.Branch, &git.PushOptions{Auth: e.cfg.Results.Auth}); err != nil {
+		return fmt.Errorf("failed to push updated results file '%v':\n  %w", absPath, err)
+	}
+
+	return nil
+}
+
+// resultsFilePaths returns the paths to the results.json file, holding the
+// benchmarks for the given system.
+func (e env) resultsFilePaths() (relPath string, absPath string, err error) {
+	dir := filepath.Join(e.resultsDir, "results")
+	if err = os.MkdirAll(dir, 0777); err != nil {
+		err = fmt.Errorf("failed to create results directory '%v':\n  %w", dir, err)
+		return
+	}
+	relPath = filepath.Join("results", e.systemID+".json")
+	absPath = filepath.Join(dir, e.systemID+".json")
+	return
+}
+
+// loadHistoricResults loads and returns the results.json file for the given
+// system.
+func (e env) loadHistoricResults(path string) (*HistoricResults, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open result file '%v':\n  %w", path, err)
+	}
+	defer file.Close()
+	res := &HistoricResults{}
+	if err := json.NewDecoder(file).Decode(res); err != nil {
+		return nil, fmt.Errorf("failed to parse result file '%v':\n  %w", path, err)
+	}
+
+	if !reflect.DeepEqual(res.System, e.system) {
+		log.Printf(`WARNING: results file '%v' has different system information!
+File: %+v
+System: %+v`, path, res.System, e.system)
+	}
+
+	return res, nil
+}
+
+// fetchTintDeps fetches the third party tint dependencies using gclient.
+func (e env) fetchTintDeps() error {
+	gclientConfig := filepath.Join(e.tintDir, ".gclient")
+	if _, err := os.Stat(gclientConfig); errors.Is(err, os.ErrNotExist) {
+		standalone := filepath.Join(e.tintDir, "standalone.gclient")
+		if err := copyFile(gclientConfig, standalone); err != nil {
+			return fmt.Errorf("failed to copy '%v' to '%v':\n  %w", standalone, gclientConfig, err)
+		}
+	}
+	if _, err := call(tools.gclient, e.tintDir, e.cfg.Timeouts.Sync,
+		"sync",
+		"--force",
+	); err != nil {
+		return fmt.Errorf("failed to fetch tint dependencies:\n  %w", err)
+	}
+	return nil
+}
+
+// buildTint builds the tint benchmarks.
+func (e env) buildTint() error {
+	if err := os.MkdirAll(e.buildDir, 0777); err != nil {
+		return fmt.Errorf("failed to create build directory at '%v':\n  %w", e.buildDir, err)
+	}
+	if _, err := call(tools.cmake, e.buildDir, e.cfg.Timeouts.Build,
+		e.tintDir,
+		"-GNinja",
+		"-DCMAKE_CXX_COMPILER_LAUNCHER=ccache",
+		"-DCMAKE_BUILD_TYPE=Release",
+		"-DTINT_BUILD_SPV_READER=1",
+		"-DTINT_BUILD_WGSL_READER=1",
+		"-DTINT_BUILD_GLSL_WRITER=1",
+		"-DTINT_BUILD_HLSL_WRITER=1",
+		"-DTINT_BUILD_MSL_WRITER=1",
+		"-DTINT_BUILD_SPV_WRITER=1",
+		"-DTINT_BUILD_WGSL_WRITER=1",
+		"-DTINT_BUILD_BENCHMARKS=1",
+	); err != nil {
+		return errFailedToBuild{fmt.Errorf("failed to generate tint build config:\n  %w", err)}
+	}
+	if _, err := call(tools.ninja, e.buildDir, e.cfg.Timeouts.Build); err != nil {
+		return errFailedToBuild{err}
+	}
+	return nil
+}
+
+// errFailedToBuild is the error returned by buildTint() if the build failed
+type errFailedToBuild struct {
+	// The reason
+	reason error
+}
+
+func (e errFailedToBuild) Error() string {
+	return fmt.Sprintf("failed to build: %v", e.reason)
+}
+
+// benchmarkTint runs the tint benchmarks, returning the results.
+func (e env) benchmarkTint() (*bench.Run, error) {
+	exe := filepath.Join(e.buildDir, "tint-benchmark")
+	out, err := call(exe, e.buildDir, e.cfg.Timeouts.Benchmark,
+		"--benchmark_format=json",
+		fmt.Sprintf("--benchmark_repetitions=%v", e.cfg.BenchmarkRepetitions),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to benchmark tint:\n  %w", err)
+	}
+
+	results, err := bench.Parse(out)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse benchmark results:\n  %w", err)
+	}
+	return &results, nil
+}
+
+// findGerritChangeToBenchmark queries gerrit for a change to benchmark.
+func (e env) findGerritChangeToBenchmark() (*gerrit.ChangeInfo, error) {
+	log.Println("querying gerrit for changes...")
+	results, _, err := e.gerrit.Changes.QueryChanges(&gerrit.QueryChangeOptions{
+		QueryOptions: gerrit.QueryOptions{
+			Query: []string{"project:tint status:open+-age:3d"},
+			Limit: 100,
+		},
+		ChangeOptions: gerrit.ChangeOptions{
+			AdditionalFields: []string{"CURRENT_REVISION", "CURRENT_COMMIT", "MESSAGES", "LABELS", "DETAILED_ACCOUNTS"},
+		},
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to get list of changes:\n  %w", err)
+	}
+
+	type candidate struct {
+		change   gerrit.ChangeInfo
+		priority int
+	}
+
+	candidates := make([]candidate, 0, len(*results))
+
+	for _, change := range *results {
+		kokoroApproved := change.Labels["Kokoro"].Approved.AccountID != 0
+		codeReviewScore := change.Labels["Code-Review"].Value
+		codeReviewApproved := change.Labels["Code-Review"].Approved.AccountID != 0
+		presubmitReady := change.Labels["Presubmit-Ready"].Approved.AccountID != 0
+		verifiedScore := change.Labels["Verified"].Value
+
+		current, ok := change.Revisions[change.CurrentRevision]
+		if !ok {
+			log.Printf("WARNING: couldn't find current revision for change '%s'", change.ChangeID)
+		}
+
+		canBenchmark := func() bool {
+			// Is the change from a Googler, reviewed by a Googler or is from a allow-listed external developer?
+			if !(strings.HasSuffix(current.Commit.Committer.Email, "@google.com") ||
+				strings.HasSuffix(change.Labels["Code-Review"].Approved.Email, "@google.com") ||
+				strings.HasSuffix(change.Labels["Code-Review"].Recommended.Email, "@google.com") ||
+				strings.HasSuffix(change.Labels["Presubmit-Ready"].Approved.Email, "@google.com")) {
+				permitted := false
+				for _, email := range e.cfg.ExternalAccounts {
+					if strings.ToLower(current.Commit.Committer.Email) == strings.ToLower(email) {
+						permitted = true
+						break
+					}
+
+				}
+				if !permitted {
+					return false
+				}
+			}
+
+			// Don't benchmark if the change has negative scores.
+			if codeReviewScore < 0 || verifiedScore < 0 {
+				return false
+			}
+
+			// Has the latest patchset already been benchmarked?
+			for _, msg := range change.Messages {
+				if msg.RevisionNumber == current.Number &&
+					msg.Author.Email == e.cfg.Gerrit.Email {
+					return false
+				}
+			}
+
+			return true
+		}()
+		if !canBenchmark {
+			continue
+		}
+
+		priority := 0
+		if presubmitReady {
+			priority += 10
+		}
+		priority += codeReviewScore
+		if codeReviewApproved {
+			priority += 2
+		}
+		if kokoroApproved {
+			priority++
+		}
+
+		candidates = append(candidates, candidate{change, priority})
+	}
+
+	// Sort the candidates
+	sort.Slice(candidates, func(i, j int) bool {
+		return candidates[i].priority > candidates[j].priority
+	})
+
+	if len(candidates) > 0 {
+		log.Printf("%d gerrit changes to benchmark", len(candidates))
+		return &candidates[0].change, nil
+	}
+	return nil, nil
+}
+
+// benchmarks the gerrit change, posting the findings to the change
+func (e env) benchmarkGerritChange(change gerrit.ChangeInfo) error {
+	current := change.Revisions[change.CurrentRevision]
+	log.Printf("fetching '%v'...", current.Ref)
+	currentHash, err := e.tintRepo.Fetch(current.Ref, &git.FetchOptions{Auth: e.cfg.Tint.Auth})
+	if err != nil {
+		return err
+	}
+	parent := current.Commit.Parents[0].Commit
+	parentHash, err := git.ParseHash(parent)
+	if err != nil {
+		return fmt.Errorf("failed to parse parent hash '%v':\n  %v", parent, err)
+	}
+
+	postMsg := func(notify, msg string) error {
+		_, _, err = e.gerrit.Changes.SetReview(change.ChangeID, currentHash.String(), &gerrit.ReviewInput{
+			Message: msg,
+			Tag:     "autogenerated:perfmon",
+			Notify:  notify,
+		})
+		if err != nil {
+			return fmt.Errorf("failed to post message to gerrit change:\n  %v", err)
+		}
+		return nil
+	}
+
+	newRun, err := e.benchmarkTintChange(currentHash)
+	if err != nil {
+		var ftb errFailedToBuild
+		if errors.As(err, &ftb) {
+			return postMsg("OWNER", fmt.Sprintf("patchset %v failed to build", current.Number))
+		}
+		return err
+	}
+	if _, err := e.tintRepo.Fetch(parent, &git.FetchOptions{Auth: e.cfg.Tint.Auth}); err != nil {
+		return err
+	}
+	parentRun, err := e.benchmarkTintChange(parentHash)
+	if err != nil {
+		return err
+	}
+
+	// filters the benchmark results to only the mean aggregate values
+	meanBenchmarkResults := func(in []bench.Benchmark) []bench.Benchmark {
+		out := make([]bench.Benchmark, 0, len(in))
+		for _, b := range in {
+			if b.AggregateType == bench.Mean {
+				out = append(out, b)
+			}
+		}
+		return out
+	}
+
+	newResults := meanBenchmarkResults(newRun.Benchmarks)
+	parentResults := meanBenchmarkResults(parentRun.Benchmarks)
+
+	const minDiff = time.Microsecond * 50 // Ignore time diffs less than this duration
+	const minRelDiff = 0.01               // Ignore absolute relative diffs between [1, 1+x]
+	diff := bench.Compare(parentResults, newResults, minDiff, minRelDiff)
+	diffFmt := bench.DiffFormat{
+		TestName:        true,
+		Delta:           true,
+		PercentChangeAB: true,
+		TimeA:           true,
+		TimeB:           true,
+	}
+
+	msg := &strings.Builder{}
+	fmt.Fprintf(msg, "Tint perfmon analysis:\n")
+	fmt.Fprintf(msg, "  \n")
+	fmt.Fprintf(msg, "  A: parent change (%v) -> B: patchset %v\n", parent[:7], current.Number)
+	fmt.Fprintf(msg, "  \n")
+	for _, line := range strings.Split(diff.Format(diffFmt), "\n") {
+		fmt.Fprintf(msg, "  %v\n", line)
+	}
+
+	notify := "OWNER"
+	if len(diff) > 0 {
+		notify = "OWNER_REVIEWERS"
+	}
+	return postMsg(notify, msg.String())
+}
+
+// createOrOpenGitRepo creates a new local repo by cloning cfg.URL into
+// filepath, or opens the existing repo at filepath.
+func createOrOpenGitRepo(g *git.Git, filepath string, cfg GitConfig) (*git.Repository, error) {
+	repo, err := g.Open(filepath)
+	if errors.Is(err, git.ErrRepositoryDoesNotExist) {
+		log.Printf("cloning '%v' branch '%v' to '%v'...", cfg.URL, cfg.Branch, filepath)
+		repo, err = g.Clone(filepath, cfg.URL, &git.CloneOptions{
+			Branch: cfg.Branch,
+			Auth:   cfg.Auth,
+		})
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to open git repository '%v':\n  %w", filepath, err)
+	}
+	return repo, err
+}
+
+// loadConfig loads the perfmon config file.
+func loadConfig(path string) (Config, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return Config{}, fmt.Errorf("failed to open config file at '%v':\n  %w", path, err)
+	}
+	cfg := Config{}
+	if err := json.NewDecoder(f).Decode(&cfg); err != nil {
+		return Config{}, fmt.Errorf("failed to load config file at '%v':\n  %w", path, err)
+	}
+	cfg.setDefaults()
+	return cfg, nil
+}
+
+// makeWorkingDirs builds the tint repo and results repo directories.
+func makeWorkingDirs(cfg Config) (tintDir, resultsDir string, err error) {
+	wd, err := expandHomeDir(cfg.WorkingDir)
+	if err != nil {
+		return "", "", err
+	}
+	if err := os.MkdirAll(wd, 0777); err != nil {
+		return "", "", fmt.Errorf("failed to create working directory '%v':\n  %w", wd, err)
+	}
+	tintDir = filepath.Join(wd, "tint")
+	if err := os.MkdirAll(tintDir, 0777); err != nil {
+		return "", "", fmt.Errorf("failed to create working tint directory '%v':\n  %w", tintDir, err)
+	}
+	resultsDir = filepath.Join(wd, "results")
+	if err := os.MkdirAll(resultsDir, 0777); err != nil {
+		return "", "", fmt.Errorf("failed to create working results directory '%v':\n  %w", resultsDir, err)
+	}
+	return tintDir, resultsDir, nil
+}
+
+// fetchAndCheckoutLatest calls fetch(cfg.Branch) followed by checkoutLatest().
+func fetchAndCheckoutLatest(repo *git.Repository, cfg GitConfig) error {
+	hash, err := repo.Fetch(cfg.Branch, &git.FetchOptions{Auth: cfg.Auth})
+	if err != nil {
+		return err
+	}
+	if err := repo.Checkout(hash.String(), nil); err != nil {
+		return err
+	}
+	return checkout(hash, repo)
+}
+
+// checkout checks out the change with the given hash.
+// Note: call fetch() to ensure that this is the latest change on the
+// branch.
+func checkout(hash git.Hash, repo *git.Repository) error {
+	if err := repo.Checkout(hash.String(), nil); err != nil {
+		return fmt.Errorf("failed to checkout '%v':\n  %w", hash, err)
+	}
+	return nil
+}
+
+// expandHomeDir returns path with all occurrences of '~' replaced with the user
+// home directory.
+func expandHomeDir(path string) (string, error) {
+	if strings.ContainsRune(path, '~') {
+		home, err := os.UserHomeDir()
+		if err != nil {
+			return "", fmt.Errorf("failed to expand home dir:\n  %w", err)
+		}
+		path = strings.ReplaceAll(path, "~", home)
+	}
+	return path, nil
+}
+
+// tools holds the file paths to the executables used by this tool
+var tools struct {
+	ccache  string
+	cmake   string
+	gclient string
+	git     string
+	ninja   string
+}
+
+// findTools looks for the file paths for executables used by this tool,
+// returning an error if any could not be found.
+func findTools() error {
+	for _, tool := range []struct {
+		name string
+		path *string
+	}{
+		{"ccache", &tools.ccache},
+		{"cmake", &tools.cmake},
+		{"gclient", &tools.gclient},
+		{"git", &tools.git},
+		{"ninja", &tools.ninja},
+	} {
+		path, err := exec.LookPath(tool.name)
+		if err != nil {
+			return fmt.Errorf("failed to find path to '%v':\n  %w", tool.name, err)
+		}
+		*tool.path = path
+	}
+	return nil
+}
+
+// copyFile copies the file at srcPath to dstPath.
+func copyFile(dstPath, srcPath string) error {
+	src, err := os.Open(srcPath)
+	if err != nil {
+		return fmt.Errorf("failed to open file '%v':\n  %w", srcPath, err)
+	}
+	defer src.Close()
+	dst, err := os.Create(dstPath)
+	if err != nil {
+		return fmt.Errorf("failed to create file '%v':\n  %w", dstPath, err)
+	}
+	defer dst.Close()
+	_, err = io.Copy(dst, src)
+	return err
+}
+
+// call invokes the executable exe in the current working directory wd, with
+// the provided arguments.
+// If the executable does not complete within the timeout duration, then an
+// error is returned.
+func call(exe, wd string, timeout time.Duration, args ...string) (string, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+	cmd := exec.CommandContext(ctx, exe, args...)
+	cmd.Dir = wd
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return string(out), fmt.Errorf("'%v %v' failed:\n  %w\n%v", exe, args, err, string(out))
+	}
+	return string(out), nil
+}
+
+// hash returns a hash of the string representation of 'o'.
+func hash(o interface{}) string {
+	str := fmt.Sprintf("%+v", o)
+	hash := sha256.New()
+	hash.Write([]byte(str))
+	return hex.EncodeToString(hash.Sum(nil))[:8]
+}

tools/src/cmd/remote-compile/CMakeLists.txt

@@ -0,0 +1,40 @@
+# Copyright 2021 The Tint Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set(SRC
+    main.cc
+    rwmutex.h
+    socket.cc
+    socket.h
+)
+
+find_package (Threads REQUIRED)
+
+add_executable(tint-remote-compile ${SRC})
+
+target_link_libraries (tint-remote-compile Threads::Threads)
+target_include_directories(tint-remote-compile PRIVATE "${TINT_ROOT_SOURCE_DIR}")
+
+# If we're building on mac / ios and we have CoreGraphics, then we can use the
+# metal API to validate our shaders. This is roughly 4x faster than invoking
+# the metal shader compiler executable.
+if(APPLE)
+  find_library(LIB_CORE_GRAPHICS CoreGraphics)
+  if(LIB_CORE_GRAPHICS)
+    target_sources(tint-remote-compile PRIVATE "msl_metal.mm")
+    target_compile_definitions(tint-remote-compile PRIVATE "-DTINT_ENABLE_MSL_COMPILATION_USING_METAL_API=1")
+    target_compile_options(tint-remote-compile PRIVATE "-fmodules" "-fcxx-modules")
+    target_link_options(tint-remote-compile PRIVATE "-framework" "CoreGraphics")
+  endif()
+endif()

tools/src/cmd/remote-compile/compile.h

@@ -0,0 +1,30 @@
+// Copyright 2021 The Tint Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_SRC_CMD_REMOTE_COMPILE_COMPILE_H_
+#define TOOLS_SRC_CMD_REMOTE_COMPILE_COMPILE_H_
+
+#include <string>
+
+/// The return structure of a compile function
+struct CompileResult {
+  /// True if shader compiled
+  bool success = false;
+  /// Output of the compiler
+  std::string output;
+};
+
+CompileResult CompileMslUsingMetalAPI(const std::string& src);
+
+#endif  // TOOLS_SRC_CMD_REMOTE_COMPILE_COMPILE_H_

tools/src/cmd/remote-compile/main.cc

@@ -0,0 +1,445 @@
+// Copyright 2021 The Tint Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <fstream>
+#include <sstream>
+#include <string>
+#include <type_traits>
+#include <vector>
+
+#include <thread>  // NOLINT
+
+#include "tools/src/cmd/remote-compile/compile.h"
+#include "tools/src/cmd/remote-compile/socket.h"
+
+namespace {
+
+#if 0
+#define DEBUG(msg, ...) printf(msg "\n", ##__VA_ARGS__)
+#else
+#define DEBUG(...)
+#endif
+
+/// Print the tool usage, and exit with 1.
+void ShowUsage() {
+  const char* name = "tint-remote-compile";
+  printf(R"(%s is a tool for compiling a shader on a remote machine
+
+usage as server:
+  %s -s [-p port-number]
+
+usage as client:
+  %s [-p port-number] [server-address] shader-file-path
+
+  [server-address] can be omitted if the TINT_REMOTE_COMPILE_ADDRESS environment
+  variable is set.
+  Alternatively, you can pass xcrun arguments so %s can be used as a
+  drop-in replacement.
+)",
+         name, name, name, name);
+  exit(1);
+}
+
+/// The protocol version code. Bump each time the protocol changes
+constexpr uint32_t kProtocolVersion = 1;
+
+/// Supported shader source languages
+enum SourceLanguage {
+  MSL,
+};
+
+/// Stream is a serialization wrapper around a socket
+struct Stream {
+  /// The underlying socket
+  Socket* const socket;
+  /// Error state
+  std::string error;
+
+  /// Writes a uint32_t to the socket
+  Stream operator<<(uint32_t v) {
+    if (error.empty()) {
+      Write(&v, sizeof(v));
+    }
+    return *this;
+  }
+
+  /// Reads a uint32_t from the socket
+  Stream operator>>(uint32_t& v) {
+    if (error.empty()) {
+      Read(&v, sizeof(v));
+    }
+    return *this;
+  }
+
+  /// Writes a std::string to the socket
+  Stream operator<<(const std::string& v) {
+    if (error.empty()) {
+      uint32_t count = static_cast<uint32_t>(v.size());
+      *this << count;
+      if (count) {
+        Write(v.data(), count);
+      }
+    }
+    return *this;
+  }
+
+  /// Reads a std::string from the socket
+  Stream operator>>(std::string& v) {
+    uint32_t count = 0;
+    *this >> count;
+    if (count) {
+      std::vector<char> buf(count);
+      if (Read(buf.data(), count)) {
+        v = std::string(buf.data(), buf.size());
+      }
+    } else {
+      v.clear();
+    }
+    return *this;
+  }
+
+  /// Writes an enum value to the socket
+  template <typename T>
+  std::enable_if_t<std::is_enum<T>::value, Stream> operator<<(T e) {
+    return *this << static_cast<uint32_t>(e);
+  }
+
+  /// Reads an enum value from the socket
+  template <typename T>
+  std::enable_if_t<std::is_enum<T>::value, Stream> operator>>(T& e) {
+    uint32_t v;
+    *this >> v;
+    e = static_cast<T>(v);
+    return *this;
+  }
+
+ private:
+  bool Write(const void* data, size_t size) {
+    if (error.empty()) {
+      if (!socket->Write(data, size)) {
+        error = "Socket::Write() failed";
+      }
+    }
+    return error.empty();
+  }
+
+  bool Read(void* data, size_t size) {
+    auto buf = reinterpret_cast<uint8_t*>(data);
+    while (size > 0 && error.empty()) {
+      if (auto n = socket->Read(buf, size)) {
+        if (n > size) {
+          error = "Socket::Read() returned more bytes than requested";
+          return false;
+        }
+        size -= n;
+        buf += n;
+      }
+    }
+    return error.empty();
+  }
+};
+
+////////////////////////////////////////////////////////////////////////////////
+// Messages
+////////////////////////////////////////////////////////////////////////////////
+
+/// Base class for all messages
+struct Message {
+  /// The type of the message
+  enum class Type {
+    ConnectionRequest,
+    ConnectionResponse,
+    CompileRequest,
+    CompileResponse,
+  };
+
+  explicit Message(Type ty) : type(ty) {}
+
+  const Type type;
+};
+
+struct ConnectionResponse : Message {  // Server -> Client
+  ConnectionResponse() : Message(Type::ConnectionResponse) {}
+
+  template <typename T>
+  void Serialize(T&& f) {
+    f(error);
+  }
+
+  std::string error;
+};
+
+struct ConnectionRequest : Message {  // Client -> Server
+  using Response = ConnectionResponse;
+
+  explicit ConnectionRequest(uint32_t proto_ver = kProtocolVersion)
+      : Message(Type::ConnectionRequest), protocol_version(proto_ver) {}
+
+  template <typename T>
+  void Serialize(T&& f) {
+    f(protocol_version);
+  }
+
+  uint32_t protocol_version;
+};
+
+struct CompileResponse : Message {  //  Server -> Client
+  CompileResponse() : Message(Type::CompileResponse) {}
+
+  template <typename T>
+  void Serialize(T&& f) {
+    f(error);
+  }
+
+  std::string error;
+};
+
+struct CompileRequest : Message {  // Client -> Server
+  using Response = CompileResponse;
+
+  CompileRequest() : Message(Type::CompileRequest) {}
+  CompileRequest(SourceLanguage lang, std::string src)
+      : Message(Type::CompileRequest), language(lang), source(src) {}
+
+  template <typename T>
+  void Serialize(T&& f) {
+    f(language);
+    f(source);
+  }
+
+  SourceLanguage language;
+  std::string source;
+};
+
+/// Writes the message `m` to the stream `s`
+template <typename MESSAGE>
+std::enable_if_t<std::is_base_of<Message, MESSAGE>::value, Stream>& operator<<(
+    Stream& s,
+    const MESSAGE& m) {
+  s << m.type;
+  const_cast<MESSAGE&>(m).Serialize([&s](const auto& value) { s << value; });
+  return s;
+}
+
+/// Reads the message `m` from the stream `s`
+template <typename MESSAGE>
+std::enable_if_t<std::is_base_of<Message, MESSAGE>::value, Stream>& operator>>(
+    Stream& s,
+    MESSAGE& m) {
+  Message::Type ty;
+  s >> ty;
+  if (ty == m.type) {
+    m.Serialize([&s](auto& value) { s >> value; });
+  } else {
+    std::stringstream ss;
+    ss << "expected message type " << static_cast<int>(m.type) << ", got "
+       << static_cast<int>(ty);
+    s.error = ss.str();
+  }
+  return s;
+}
+
+/// Writes the request message `req` to the stream `s`, then reads and returns
+/// the response message from the same stream.
+template <typename REQUEST, typename RESPONSE = typename REQUEST::Response>
+RESPONSE Send(Stream& s, const REQUEST& req) {
+  s << req;
+  if (s.error.empty()) {
+    RESPONSE resp;
+    s >> resp;
+    if (s.error.empty()) {
+      return resp;
+    }
+  }
+  return {};
+}
+
+}  // namespace
+
+bool RunServer(std::string port);
+bool RunClient(std::string address, std::string port, std::string file);
+
+int main(int argc, char* argv[]) {
+  bool run_server = false;
+  std::string port = "19000";
+
+  std::vector<std::string> args;
+  for (int i = 1; i < argc; i++) {
+    std::string arg = argv[i];
+    if (arg == "-s" || arg == "--server") {
+      run_server = true;
+      continue;
+    }
+    if (arg == "-p" || arg == "--port") {
+      if (i < argc - 1) {
+        i++;
+        port = argv[i];
+      } else {
+        printf("expected port number");
+        exit(1);
+      }
+      continue;
+    }
+
+    // xcrun flags are ignored so this executable can be used as a replacement
+    // for xcrun.
+    if ((arg == "-x" || arg == "-sdk") && (i < argc - 1)) {
+      i++;
+      continue;
+    }
+    if (arg == "metal") {
+      for (; i < argc; i++) {
+        if (std::string(argv[i]) == "-c") {
+          break;
+        }
+      }
+      continue;
+    }
+
+    args.emplace_back(arg);
+  }
+
+  bool success = false;
+
+  if (run_server) {
+    success = RunServer(port);
+  } else {
+    std::string address;
+    std::string file;
+    switch (args.size()) {
+      case 1:
+        if (auto* addr = getenv("TINT_REMOTE_COMPILE_ADDRESS")) {
+          address = addr;
+        }
+        file = args[0];
+        break;
+      case 2:
+        address = args[0];
+        file = args[1];
+        break;
+    }
+    if (address.empty() || file.empty()) {
+      ShowUsage();
+    }
+    success = RunClient(address, port, file);
+  }
+
+  if (!success) {
+    exit(1);
+  }
+
+  return 0;
+}
+
+bool RunServer(std::string port) {
+  auto server_socket = Socket::Listen("", port.c_str());
+  if (!server_socket) {
+    printf("Failed to listen on port %s\n", port.c_str());
+    return false;
+  }
+  printf("Listening on port %s...\n", port.c_str());
+  while (auto conn = server_socket->Accept()) {
+    std::thread([=] {
+      DEBUG("Client connected...");
+      Stream stream{conn.get()};
+
+      {
+        ConnectionRequest req;
+        stream >> req;
+        if (!stream.error.empty()) {
+          printf("%s\n", stream.error.c_str());
+          return;
+        }
+        ConnectionResponse resp;
+        if (req.protocol_version != kProtocolVersion) {
+          DEBUG("Protocol version mismatch");
+          resp.error = "Protocol version mismatch";
+          stream << resp;
+          return;
+        }
+        stream << resp;
+      }
+      DEBUG("Connection established");
+      {
+        CompileRequest req;
+        stream >> req;
+        if (!stream.error.empty()) {
+          printf("%s\n", stream.error.c_str());
+          return;
+        }
+#ifdef TINT_ENABLE_MSL_COMPILATION_USING_METAL_API
+        if (req.language == SourceLanguage::MSL) {
+          auto result = CompileMslUsingMetalAPI(req.source);
+          CompileResponse resp;
+          if (!result.success) {
+            resp.error = result.output;
+          }
+          stream << resp;
+          return;
+        }
+#endif
+        CompileResponse resp;
+        resp.error = "server cannot compile this type of shader";
+        stream << resp;
+      }
+    }).detach();
+  }
+  return true;
+}
+
+bool RunClient(std::string address, std::string port, std::string file) {
+  // Read the file
+  std::ifstream input(file, std::ios::binary);
+  if (!input) {
+    printf("Couldn't open '%s'\n", file.c_str());
+    return false;
+  }
+  std::string source((std::istreambuf_iterator<char>(input)),
+                     std::istreambuf_iterator<char>());
+
+  constexpr const int timeout_ms = 10000;
+  DEBUG("Connecting to %s:%s...", address.c_str(), port.c_str());
+  auto conn = Socket::Connect(address.c_str(), port.c_str(), timeout_ms);
+  if (!conn) {
+    printf("Connection failed\n");
+    return false;
+  }
+
+  Stream stream{conn.get()};
+
+  DEBUG("Sending connection request...");
+  auto conn_resp = Send(stream, ConnectionRequest{kProtocolVersion});
+  if (!stream.error.empty()) {
+    printf("%s\n", stream.error.c_str());
+    return false;
+  }
+  if (!conn_resp.error.empty()) {
+    printf("%s\n", conn_resp.error.c_str());
+    return false;
+  }
+  DEBUG("Connection established. Requesting compile...");
+  auto comp_resp = Send(stream, CompileRequest{SourceLanguage::MSL, source});
+  if (!stream.error.empty()) {
+    printf("%s\n", stream.error.c_str());
+    return false;
+  }
+  if (!comp_resp.error.empty()) {
+    printf("%s\n", comp_resp.error.c_str());
+    return false;
+  }
+  DEBUG("Compilation successful");
+  return true;
+}

tools/src/cmd/remote-compile/msl_metal.mm

@@ -0,0 +1,56 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifdef TINT_ENABLE_MSL_COMPILATION_USING_METAL_API
+
+@import Metal;
+
+// Disable: error: treating #include as an import of module 'std.string'
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wauto-import"
+#include "compile.h"
+#pragma clang diagnostic pop
+
+CompileResult CompileMslUsingMetalAPI(const std::string& src) {
+  CompileResult result;
+  result.success = false;
+
+  NSError* error = nil;
+
+  id<MTLDevice> device = MTLCreateSystemDefaultDevice();
+  if (!device) {
+    result.output = "MTLCreateSystemDefaultDevice returned null";
+    result.success = false;
+    return result;
+  }
+
+  NSString* source = [NSString stringWithCString:src.c_str()
+                                        encoding:NSUTF8StringEncoding];
+
+  MTLCompileOptions* compileOptions = [MTLCompileOptions new];
+  compileOptions.languageVersion = MTLLanguageVersion1_2;
+
+  id<MTLLibrary> library = [device newLibraryWithSource:source
+                                                options:compileOptions
+                                                  error:&error];
+  if (!library) {
+    NSString* output = [error localizedDescription];
+    result.output = [output UTF8String];
+    result.success = false;
+  }
+
+  return result;
+}
+
+#endif

tools/src/cmd/remote-compile/rwmutex.h

@@ -0,0 +1,190 @@
+// Copyright 2020 The Tint Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_SRC_CMD_REMOTE_COMPILE_RWMUTEX_H_
+#define TOOLS_SRC_CMD_REMOTE_COMPILE_RWMUTEX_H_
+
+#include <condition_variable>  // NOLINT
+#include <mutex>               // NOLINT
+
+////////////////////////////////////////////////////////////////////////////////
+// RWMutex
+////////////////////////////////////////////////////////////////////////////////
+
+/// A RWMutex is a reader/writer mutual exclusion lock.
+/// The lock can be held by an arbitrary number of readers or a single writer.
+/// Also known as a shared mutex.
+class RWMutex {
+ public:
+  inline RWMutex() = default;
+
+  /// lockReader() locks the mutex for reading.
+  /// Multiple read locks can be held while there are no writer locks.
+  inline void lockReader();
+
+  /// unlockReader() unlocks the mutex for reading.
+  inline void unlockReader();
+
+  /// lockWriter() locks the mutex for writing.
+  /// If the lock is already locked for reading or writing, lockWriter blocks
+  /// until the lock is available.
+  inline void lockWriter();
+
+  /// unlockWriter() unlocks the mutex for writing.
+  inline void unlockWriter();
+
+ private:
+  RWMutex(const RWMutex&) = delete;
+  RWMutex& operator=(const RWMutex&) = delete;
+
+  int readLocks = 0;
+  int pendingWriteLocks = 0;
+  std::mutex mutex;
+  std::condition_variable cv;
+};
+
+void RWMutex::lockReader() {
+  std::unique_lock<std::mutex> lock(mutex);
+  readLocks++;
+}
+
+void RWMutex::unlockReader() {
+  std::unique_lock<std::mutex> lock(mutex);
+  readLocks--;
+  if (readLocks == 0 && pendingWriteLocks > 0) {
+    cv.notify_one();
+  }
+}
+
+void RWMutex::lockWriter() {
+  std::unique_lock<std::mutex> lock(mutex);
+  if (readLocks > 0) {
+    pendingWriteLocks++;
+    cv.wait(lock, [&] { return readLocks == 0; });
+    pendingWriteLocks--;
+  }
+  lock.release();  // Keep lock held
+}
+
+void RWMutex::unlockWriter() {
+  if (pendingWriteLocks > 0) {
+    cv.notify_one();
+  }
+  mutex.unlock();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// RLock
+////////////////////////////////////////////////////////////////////////////////
+
+/// RLock is a RAII read lock helper for a RWMutex.
+class RLock {
+ public:
+  /// Constructor.
+  /// Locks `mutex` with a read-lock for the lifetime of the WLock.
+  /// @param mutex the mutex
+  explicit inline RLock(RWMutex& mutex);
+  /// Destructor.
+  /// Unlocks the RWMutex.
+  inline ~RLock();
+
+  /// Move constructor
+  /// @param other the other RLock to move into this RLock.
+  inline RLock(RLock&& other);
+  /// Move assignment operator
+  /// @param other the other RLock to move into this RLock.
+  /// @returns this RLock so calls can be chained
+  inline RLock& operator=(RLock&& other);
+
+ private:
+  RLock(const RLock&) = delete;
+  RLock& operator=(const RLock&) = delete;
+
+  RWMutex* m;
+};
+
+RLock::RLock(RWMutex& mutex) : m(&mutex) {
+  m->lockReader();
+}
+
+RLock::~RLock() {
+  if (m != nullptr) {
+    m->unlockReader();
+  }
+}
+
+RLock::RLock(RLock&& other) {
+  m = other.m;
+  other.m = nullptr;
+}
+
+RLock& RLock::operator=(RLock&& other) {
+  m = other.m;
+  other.m = nullptr;
+  return *this;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// WLock
+////////////////////////////////////////////////////////////////////////////////
+
+/// WLock is a RAII write lock helper for a RWMutex.
+class WLock {
+ public:
+  /// Constructor.
+  /// Locks `mutex` with a write-lock for the lifetime of the WLock.
+  /// @param mutex the mutex
+  explicit inline WLock(RWMutex& mutex);
+
+  /// Destructor.
+  /// Unlocks the RWMutex.
+  inline ~WLock();
+
+  /// Move constructor
+  /// @param other the other WLock to move into this WLock.
+  inline WLock(WLock&& other);
+  /// Move assignment operator
+  /// @param other the other WLock to move into this WLock.
+  /// @returns this WLock so calls can be chained
+  inline WLock& operator=(WLock&& other);
+
+ private:
+  WLock(const WLock&) = delete;
+  WLock& operator=(const WLock&) = delete;
+
+  RWMutex* m;
+};
+
+WLock::WLock(RWMutex& mutex) : m(&mutex) {
+  m->lockWriter();
+}
+
+WLock::~WLock() {
+  if (m != nullptr) {
+    m->unlockWriter();
+  }
+}
+
+WLock::WLock(WLock&& other) {
+  m = other.m;
+  other.m = nullptr;
+}
+
+WLock& WLock::operator=(WLock&& other) {
+  m = other.m;
+  other.m = nullptr;
+  return *this;
+}
+
+#endif  // TOOLS_SRC_CMD_REMOTE_COMPILE_RWMUTEX_H_

tools/src/cmd/remote-compile/socket.cc

@@ -0,0 +1,310 @@
+// Copyright 2021 The Tint Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tools/src/cmd/remote-compile/socket.h"
+
+#include "tools/src/cmd/remote-compile/rwmutex.h"
+
+#if defined(_WIN32)
+#include <winsock2.h>
+#include <ws2tcpip.h>
+#else
+#include <netdb.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+#include <sys/select.h>
+#include <sys/socket.h>
+#include <sys/time.h>
+#include <unistd.h>
+#endif
+
+#if defined(_WIN32)
+#include <atomic>
+namespace {
+std::atomic<int> wsaInitCount = {0};
+}  // anonymous namespace
+#else
+#include <fcntl.h>
+namespace {
+using SOCKET = int;
+}  // anonymous namespace
+#endif
+
+namespace {
+constexpr SOCKET InvalidSocket = static_cast<SOCKET>(-1);
+void init() {
+#if defined(_WIN32)
+  if (wsaInitCount++ == 0) {
+    WSADATA winsockData;
+    (void)WSAStartup(MAKEWORD(2, 2), &winsockData);
+  }
+#endif
+}
+
+void term() {
+#if defined(_WIN32)
+  if (--wsaInitCount == 0) {
+    WSACleanup();
+  }
+#endif
+}
+
+bool setBlocking(SOCKET s, bool blocking) {
+#if defined(_WIN32)
+  u_long mode = blocking ? 0 : 1;
+  return ioctlsocket(s, FIONBIO, &mode) == NO_ERROR;
+#else
+  auto arg = fcntl(s, F_GETFL, nullptr);
+  if (arg < 0) {
+    return false;
+  }
+  arg = blocking ? (arg & ~O_NONBLOCK) : (arg | O_NONBLOCK);
+  return fcntl(s, F_SETFL, arg) >= 0;
+#endif
+}
+
+bool errored(SOCKET s) {
+  if (s == InvalidSocket) {
+    return true;
+  }
+  char error = 0;
+  socklen_t len = sizeof(error);
+  getsockopt(s, SOL_SOCKET, SO_ERROR, &error, &len);
+  return error != 0;
+}
+
+class Impl : public Socket {
+ public:
+  static std::shared_ptr<Impl> create(const char* address, const char* port) {
+    init();
+
+    addrinfo hints = {};
+    hints.ai_family = AF_INET;
+    hints.ai_socktype = SOCK_STREAM;
+    hints.ai_protocol = IPPROTO_TCP;
+    hints.ai_flags = AI_PASSIVE;
+
+    addrinfo* info = nullptr;
+    auto err = getaddrinfo(address, port, &hints, &info);
+#if !defined(_WIN32)
+    if (err) {
+      printf("getaddrinfo(%s, %s) error: %s\n", address, port,
+             gai_strerror(err));
+    }
+#endif
+
+    if (info) {
+      auto socket =
+          ::socket(info->ai_family, info->ai_socktype, info->ai_protocol);
+      auto out = std::make_shared<Impl>(info, socket);
+      out->setOptions();
+      return out;
+    }
+
+    freeaddrinfo(info);
+    term();
+    return nullptr;
+  }
+
+  explicit Impl(SOCKET socket) : info(nullptr), s(socket) {}
+  Impl(addrinfo* info, SOCKET socket) : info(info), s(socket) {}
+
+  ~Impl() {
+    freeaddrinfo(info);
+    Close();
+    term();
+  }
+
+  template <typename FUNCTION>
+  void lock(FUNCTION&& f) {
+    RLock l(mutex);
+    f(s, info);
+  }
+
+  void setOptions() {
+    RLock l(mutex);
+    if (s == InvalidSocket) {
+      return;
+    }
+
+    int enable = 1;
+
+#if !defined(_WIN32)
+    // Prevent sockets lingering after process termination, causing
+    // reconnection issues on the same port.
+    setsockopt(s, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast<char*>(&enable),
+               sizeof(enable));
+
+    struct {
+      int l_onoff;  /* linger active */
+      int l_linger; /* how many seconds to linger for */
+    } linger = {false, 0};
+    setsockopt(s, SOL_SOCKET, SO_LINGER, reinterpret_cast<char*>(&linger),
+               sizeof(linger));
+#endif  // !defined(_WIN32)
+
+    // Enable TCP_NODELAY.
+    setsockopt(s, IPPROTO_TCP, TCP_NODELAY, reinterpret_cast<char*>(&enable),
+               sizeof(enable));
+  }
+
+  bool IsOpen() override {
+    {
+      RLock l(mutex);
+      if ((s != InvalidSocket) && !errored(s)) {
+        return true;
+      }
+    }
+    WLock lock(mutex);
+    s = InvalidSocket;
+    return false;
+  }
+
+  void Close() override {
+    {
+      RLock l(mutex);
+      if (s != InvalidSocket) {
+#if defined(_WIN32)
+        closesocket(s);
+#else
+        ::shutdown(s, SHUT_RDWR);
+#endif
+      }
+    }
+
+    WLock l(mutex);
+    if (s != InvalidSocket) {
+#if !defined(_WIN32)
+      ::close(s);
+#endif
+      s = InvalidSocket;
+    }
+  }
+
+  size_t Read(void* buffer, size_t bytes) override {
+    RLock lock(mutex);
+    if (s == InvalidSocket) {
+      return 0;
+    }
+    auto len =
+        recv(s, reinterpret_cast<char*>(buffer), static_cast<int>(bytes), 0);
+    return (len < 0) ? 0 : len;
+  }
+
+  bool Write(const void* buffer, size_t bytes) override {
+    RLock lock(mutex);
+    if (s == InvalidSocket) {
+      return false;
+    }
+    if (bytes == 0) {
+      return true;
+    }
+    return ::send(s, reinterpret_cast<const char*>(buffer),
+                  static_cast<int>(bytes), 0) > 0;
+  }
+
+  std::shared_ptr<Socket> Accept() override {
+    std::shared_ptr<Impl> out;
+    lock([&](SOCKET socket, const addrinfo*) {
+      if (socket != InvalidSocket) {
+        init();
+        out = std::make_shared<Impl>(::accept(socket, 0, 0));
+        out->setOptions();
+      }
+    });
+    return out;
+  }
+
+ private:
+  addrinfo* const info;
+  SOCKET s = InvalidSocket;
+  RWMutex mutex;
+};
+
+}  // anonymous namespace
+
+std::shared_ptr<Socket> Socket::Listen(const char* address, const char* port) {
+  auto impl = Impl::create(address, port);
+  if (!impl) {
+    return nullptr;
+  }
+  impl->lock([&](SOCKET socket, const addrinfo* info) {
+    if (bind(socket, info->ai_addr, static_cast<int>(info->ai_addrlen)) != 0) {
+      impl.reset();
+      return;
+    }
+
+    if (listen(socket, 0) != 0) {
+      impl.reset();
+      return;
+    }
+  });
+  return impl;
+}
+
+std::shared_ptr<Socket> Socket::Connect(const char* address,
+                                        const char* port,
+                                        uint32_t timeoutMillis) {
+  auto impl = Impl::create(address, port);
+  if (!impl) {
+    return nullptr;
+  }
+
+  std::shared_ptr<Socket> out;
+  impl->lock([&](SOCKET socket, const addrinfo* info) {
+    if (socket == InvalidSocket) {
+      return;
+    }
+
+    if (timeoutMillis == 0) {
+      if (::connect(socket, info->ai_addr,
+                    static_cast<int>(info->ai_addrlen)) == 0) {
+        out = impl;
+      }
+      return;
+    }
+
+    if (!setBlocking(socket, false)) {
+      return;
+    }
+
+    auto res =
+        ::connect(socket, info->ai_addr, static_cast<int>(info->ai_addrlen));
+    if (res == 0) {
+      if (setBlocking(socket, true)) {
+        out = impl;
+      }
+    } else {
+      const auto microseconds = timeoutMillis * 1000;
+
+      fd_set fdset;
+      FD_ZERO(&fdset);
+      FD_SET(socket, &fdset);
+
+      timeval tv;
+      tv.tv_sec = microseconds / 1000000;
+      tv.tv_usec = microseconds - static_cast<uint32_t>(tv.tv_sec * 1000000);
+      res = select(static_cast<int>(socket + 1), nullptr, &fdset, nullptr, &tv);
+      if (res > 0 && !errored(socket) && setBlocking(socket, true)) {
+        out = impl;
+      }
+    }
+  });
+
+  if (!out) {
+    return nullptr;
+  }
+
+  return out->IsOpen() ? out : nullptr;
+}

tools/src/cmd/remote-compile/socket.h

@@ -0,0 +1,71 @@
+// Copyright 2021 The Tint Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_SRC_CMD_REMOTE_COMPILE_SOCKET_H_
+#define TOOLS_SRC_CMD_REMOTE_COMPILE_SOCKET_H_
+
+#include <atomic>
+#include <memory>
+
+/// Socket provides an OS abstraction to a TCP socket.
+class Socket {
+ public:
+  /// Connects to the given TCP address and port.
+  /// @param address the target socket address
+  /// @param port the target socket port
+  /// @param timeoutMillis the timeout for the connection attempt.
+  ///        If timeoutMillis is non-zero and no connection was made before
+  ///        timeoutMillis milliseconds, then nullptr is returned.
+  /// @returns the connected Socket, or nullptr on failure
+  static std::shared_ptr<Socket> Connect(const char* address,
+                                         const char* port,
+                                         uint32_t timeoutMillis);
+
+  /// Begins listening for connections on the given TCP address and port.
+  /// Call Accept() on the returned Socket to block and wait for a connection.
+  /// @param address the socket address to listen on. Use "localhost" for
+  ///        connections from only this machine, or an empty string to allow
+  ///        connections from any incoming address.
+  /// @param port the socket port to listen on
+  /// @returns the Socket that listens for connections
+  static std::shared_ptr<Socket> Listen(const char* address, const char* port);
+
+  /// Attempts to read at most `n` bytes into buffer, returning the actual
+  /// number of bytes read.
+  /// read() will block until the socket is closed or at least one byte is read.
+  /// @param buffer the output buffer. Must be at least `n` bytes in size.
+  /// @param n the maximum number of bytes to read
+  /// @return the number of bytes read, or 0 if the socket was closed
+  virtual size_t Read(void* buffer, size_t n) = 0;
+
+  /// Writes `n` bytes from buffer into the socket.
+  /// @param buffer the source data buffer. Must be at least `n` bytes in size.
+  /// @param n the number of bytes to read from `buffer`
+  /// @returns true on success, or false if there was an error or the socket was
+  /// closed.
+  virtual bool Write(const void* buffer, size_t n) = 0;
+
+  /// @returns true if the socket has not been closed.
+  virtual bool IsOpen() = 0;
+
+  /// Closes the socket.
+  virtual void Close() = 0;
+
+  /// Blocks for a connection to be made to the listening port, or for the
+  /// Socket to be closed.
+  /// @returns a pointer to the next established incoming connection
+  virtual std::shared_ptr<Socket> Accept() = 0;
+};
+
+#endif  // TOOLS_SRC_CMD_REMOTE_COMPILE_SOCKET_H_

tools/src/cmd/run-parallel/main.go

@@ -0,0 +1,133 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// run-parallel is a tool to run an executable with the provided templated
+// arguments across all the hardware threads.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"runtime"
+	"strings"
+	"sync"
+)
+
+func main() {
+	if err := run(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func showUsage() {
+	fmt.Println(`
+run-parallel is a tool to run an executable with the provided templated
+arguments across all the hardware threads.
+
+Usage:
+  run-parallel <executable> [arguments...] -- [per-instance-value...]
+
+  executable         - the path to the executable to run.
+  arguments          - a list of arguments to pass to the executable.
+                       Any occurrance of $ will be substituted with the
+                       per-instance-value for the given invocation.
+  per-instance-value - a list of values. The executable will be invoked for each
+                       value in this list.`)
+	os.Exit(1)
+}
+
+func run() error {
+	onlyPrintFailures := flag.Bool("only-print-failures", false, "Omit output for processes that did not fail")
+	flag.Parse()
+
+	args := flag.Args()
+	if len(args) < 2 {
+		showUsage()
+	}
+	exe := args[0]
+	args = args[1:]
+
+	var perInstanceValues []string
+	for i, arg := range args {
+		if arg == "--" {
+			perInstanceValues = args[i+1:]
+			args = args[:i]
+			break
+		}
+	}
+	if perInstanceValues == nil {
+		showUsage()
+	}
+
+	taskIndices := make(chan int, 64)
+	type result struct {
+		msg    string
+		failed bool
+	}
+	results := make([]result, len(perInstanceValues))
+
+	numCPU := runtime.NumCPU()
+	wg := sync.WaitGroup{}
+	wg.Add(numCPU)
+	for i := 0; i < numCPU; i++ {
+		go func() {
+			defer wg.Done()
+			for idx := range taskIndices {
+				taskArgs := make([]string, len(args))
+				for i, arg := range args {
+					taskArgs[i] = strings.ReplaceAll(arg, "$", perInstanceValues[idx])
+				}
+				success, out := invoke(exe, taskArgs)
+				if !success || !*onlyPrintFailures {
+					results[idx] = result{out, !success}
+				}
+			}
+		}()
+	}
+
+	for i := range perInstanceValues {
+		taskIndices <- i
+	}
+	close(taskIndices)
+
+	wg.Wait()
+
+	failed := false
+	for _, result := range results {
+		if result.msg != "" {
+			fmt.Println(result.msg)
+		}
+		failed = failed || result.failed
+	}
+	if failed {
+		os.Exit(1)
+	}
+	return nil
+}
+
+func invoke(exe string, args []string) (ok bool, output string) {
+	cmd := exec.Command(exe, args...)
+	out, err := cmd.CombinedOutput()
+	str := string(out)
+	if err != nil {
+		if str != "" {
+			return false, str
+		}
+		return false, err.Error()
+	}
+	return true, str
+}

tools/src/cmd/snippets/main.go

@@ -0,0 +1,112 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// snippets gathers information about changes merged for weekly reports (snippets).
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"dawn.googlesource.com/tint/tools/src/gerrit"
+)
+
+const yyyymmdd = "2006-01-02"
+
+var (
+	// See https://dawn-review.googlesource.com/new-password for obtaining
+	// username and password for gerrit.
+	gerritUser = flag.String("gerrit-user", "", "gerrit authentication username")
+	gerritPass = flag.String("gerrit-pass", "", "gerrit authentication password")
+	userFlag   = flag.String("user", "", "user name / email")
+	afterFlag  = flag.String("after", "", "start date")
+	beforeFlag = flag.String("before", "", "end date")
+	daysFlag   = flag.Int("days", 7, "interval in days (used if --after is not specified)")
+)
+
+func main() {
+	flag.Parse()
+	if err := run(); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+func run() error {
+	var after, before time.Time
+	var err error
+	user := *userFlag
+	if user == "" {
+		return fmt.Errorf("Missing required 'user' flag")
+	}
+	if *beforeFlag != "" {
+		before, err = time.Parse(yyyymmdd, *beforeFlag)
+		if err != nil {
+			return fmt.Errorf("Couldn't parse before date: %w", err)
+		}
+	} else {
+		before = time.Now()
+	}
+	if *afterFlag != "" {
+		after, err = time.Parse(yyyymmdd, *afterFlag)
+		if err != nil {
+			return fmt.Errorf("Couldn't parse after date: %w", err)
+		}
+	} else {
+		after = before.Add(-time.Hour * time.Duration(24**daysFlag))
+	}
+
+	g, err := gerrit.New(gerrit.Config{Username: *gerritUser, Password: *gerritPass})
+	if err != nil {
+		return err
+	}
+
+	submitted, _, err := g.QueryChanges(
+		"status:merged",
+		"owner:"+user,
+		"after:"+date(after),
+		"before:"+date(before))
+	if err != nil {
+		return fmt.Errorf("Query failed: %w", err)
+	}
+
+	changesByProject := map[string][]string{}
+	for _, change := range submitted {
+		str := fmt.Sprintf(`* [%s](%sc/%s/+/%d)`, change.Subject, gerrit.URL, change.Project, change.Number)
+		changesByProject[change.Project] = append(changesByProject[change.Project], str)
+	}
+
+	for _, project := range []string{"tint", "dawn"} {
+		if changes := changesByProject[project]; len(changes) > 0 {
+			fmt.Println("##", strings.Title(project))
+			for _, change := range changes {
+				fmt.Println(change)
+			}
+			fmt.Println()
+		}
+	}
+
+	return nil
+}
+
+func today() time.Time {
+	return time.Now()
+}
+
+func date(t time.Time) string {
+	return t.Format(yyyymmdd)
+}

tools/src/cmd/test-runner/main.go

@@ -0,0 +1,796 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// test-runner runs tint against a number of test shaders checking for expected behavior
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"sort"
+	"strings"
+	"time"
+	"unicode/utf8"
+
+	"dawn.googlesource.com/tint/tools/src/fileutils"
+	"dawn.googlesource.com/tint/tools/src/glob"
+	"github.com/fatih/color"
+	"github.com/sergi/go-diff/diffmatchpatch"
+)
+
+type outputFormat string
+
+const (
+	testTimeout = 30 * time.Second
+
+	glsl   = outputFormat("glsl")
+	hlsl   = outputFormat("hlsl")
+	msl    = outputFormat("msl")
+	spvasm = outputFormat("spvasm")
+	wgsl   = outputFormat("wgsl")
+)
+
+// Directories we don't generate expected PASS result files for.
+// These directories contain large corpora of tests for which the generated code
+// is uninteresting.
+var dirsWithNoPassExpectations = []string{
+	"/test/tint/benchmark/",
+	"/test/tint/unittest/",
+	"/test/tint/vk-gl-cts/",
+}
+
+func main() {
+	if err := run(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func showUsage() {
+	fmt.Println(`
+test-runner runs tint against a number of test shaders checking for expected behavior
+
+usage:
+  test-runner [flags...] <executable> [<directory>]
+
+  <executable> the path to the tint executable
+  <directory>  the root directory of the test files
+
+optional flags:`)
+	flag.PrintDefaults()
+	fmt.Println(``)
+	os.Exit(1)
+}
+
+func run() error {
+	var formatList, filter, dxcPath, xcrunPath string
+	var maxFilenameColumnWidth int
+	numCPU := runtime.NumCPU()
+	fxc, fxcAndDxc, verbose, generateExpected, generateSkip := false, false, false, false, false
+	flag.StringVar(&formatList, "format", "all", "comma separated list of formats to emit. Possible values are: all, wgsl, spvasm, msl, hlsl, glsl")
+	flag.StringVar(&filter, "filter", "**.wgsl, **.spvasm, **.spv", "comma separated list of glob patterns for test files")
+	flag.StringVar(&dxcPath, "dxc", "", "path to DXC executable for validating HLSL output")
+	flag.StringVar(&xcrunPath, "xcrun", "", "path to xcrun executable for validating MSL output")
+	flag.BoolVar(&fxc, "fxc", false, "validate with FXC instead of DXC")
+	flag.BoolVar(&fxcAndDxc, "fxc-and-dxc", false, "validate with both FXC and DXC")
+	flag.BoolVar(&verbose, "verbose", false, "print all run tests, including rows that all pass")
+	flag.BoolVar(&generateExpected, "generate-expected", false, "create or update all expected outputs")
+	flag.BoolVar(&generateSkip, "generate-skip", false, "create or update all expected outputs that fail with SKIP")
+	flag.IntVar(&numCPU, "j", numCPU, "maximum number of concurrent threads to run tests")
+	flag.IntVar(&maxFilenameColumnWidth, "filename-column-width", 0, "maximum width of the filename column")
+	flag.Usage = showUsage
+	flag.Parse()
+
+	args := flag.Args()
+	if len(args) == 0 {
+		showUsage()
+	}
+
+	if fxcAndDxc {
+		fxc = true
+	}
+
+	// executable path is the first argument
+	exe, args := args[0], args[1:]
+
+	// (optional) target directory is the second argument
+	dir := "."
+	if len(args) > 0 {
+		dir, args = args[0], args[1:]
+	}
+
+	// Check the executable can be found and actually is executable
+	if !fileutils.IsExe(exe) {
+		return fmt.Errorf("'%s' not found or is not executable", exe)
+	}
+	exe, err := filepath.Abs(exe)
+	if err != nil {
+		return err
+	}
+
+	// Allow using '/' in the filter on Windows
+	filter = strings.ReplaceAll(filter, "/", string(filepath.Separator))
+
+	// Split the --filter flag up by ',', trimming any whitespace at the start and end
+	globIncludes := strings.Split(filter, ",")
+	for i, s := range globIncludes {
+		s = filepath.ToSlash(s) // Replace '\' with '/'
+		globIncludes[i] = `"` + strings.TrimSpace(s) + `"`
+	}
+
+	// Glob the files to test
+	files, err := glob.Scan(dir, glob.MustParseConfig(`{
+		"paths": [
+			{
+				"include": [ `+strings.Join(globIncludes, ",")+` ]
+			},
+			{
+				"exclude": [
+					"**.expected.wgsl",
+					"**.expected.spvasm",
+					"**.expected.msl",
+					"**.expected.hlsl",
+					"**.expected.glsl"
+				]
+			}
+		]
+	}`))
+	if err != nil {
+		return fmt.Errorf("Failed to glob files: %w", err)
+	}
+
+	// Ensure the files are sorted (globbing should do this, but why not)
+	sort.Strings(files)
+
+	// Parse --format into a list of outputFormat
+	formats := []outputFormat{}
+	if formatList == "all" {
+		formats = []outputFormat{wgsl, spvasm, msl, hlsl, glsl}
+	} else {
+		for _, f := range strings.Split(formatList, ",") {
+			switch strings.TrimSpace(f) {
+			case "wgsl":
+				formats = append(formats, wgsl)
+			case "spvasm":
+				formats = append(formats, spvasm)
+			case "msl":
+				formats = append(formats, msl)
+			case "hlsl":
+				formats = append(formats, hlsl)
+			case "glsl":
+				formats = append(formats, glsl)
+			default:
+				return fmt.Errorf("unknown format '%s'", f)
+			}
+		}
+	}
+
+	defaultMSLExe := "xcrun"
+	if runtime.GOOS == "windows" {
+		defaultMSLExe = "metal.exe"
+	}
+
+	// If explicit verification compilers have been specified, check they exist.
+	// Otherwise, look on PATH for them, but don't error if they cannot be found.
+	for _, tool := range []struct {
+		name string
+		lang string
+		path *string
+	}{
+		{"dxc", "hlsl", &dxcPath},
+		{defaultMSLExe, "msl", &xcrunPath},
+	} {
+		if *tool.path == "" {
+			p, err := exec.LookPath(tool.name)
+			if err == nil && fileutils.IsExe(p) {
+				*tool.path = p
+			}
+		} else if !fileutils.IsExe(*tool.path) {
+			return fmt.Errorf("%v not found at '%v'", tool.name, *tool.path)
+		}
+
+		color.Set(color.FgCyan)
+		fmt.Printf("%-4s", tool.lang)
+		color.Unset()
+		fmt.Printf(" validation ")
+		if *tool.path != "" || (fxc && tool.lang == "hlsl") {
+			color.Set(color.FgGreen)
+			tool_path := *tool.path
+			if fxc && tool.lang == "hlsl" {
+				if fxcAndDxc {
+					tool_path += " AND Tint will use FXC dll in PATH"
+				} else {
+					tool_path = "Tint will use FXC dll in PATH"
+				}
+			}
+			fmt.Printf("ENABLED (" + tool_path + ")")
+		} else {
+			color.Set(color.FgRed)
+			fmt.Printf("DISABLED")
+		}
+		color.Unset()
+		fmt.Println()
+	}
+	fmt.Println()
+
+	// Build the list of results.
+	// These hold the chans used to report the job results.
+	results := make([]map[outputFormat]chan status, len(files))
+	for i := range files {
+		fileResults := map[outputFormat]chan status{}
+		for _, format := range formats {
+			fileResults[format] = make(chan status, 1)
+		}
+		results[i] = fileResults
+	}
+
+	pendingJobs := make(chan job, 256)
+
+	// Spawn numCPU job runners...
+	for cpu := 0; cpu < numCPU; cpu++ {
+		go func() {
+			for job := range pendingJobs {
+				job.run(dir, exe, fxc, fxcAndDxc, dxcPath, xcrunPath, generateExpected, generateSkip)
+			}
+		}()
+	}
+
+	// Issue the jobs...
+	go func() {
+		for i, file := range files { // For each test file...
+			file := filepath.Join(dir, file)
+			flags := parseFlags(file)
+			for _, format := range formats { // For each output format...
+				pendingJobs <- job{
+					file:   file,
+					flags:  flags,
+					format: format,
+					result: results[i][format],
+				}
+			}
+		}
+		close(pendingJobs)
+	}()
+
+	type failure struct {
+		file   string
+		format outputFormat
+		err    error
+	}
+
+	type stats struct {
+		numTests, numPass, numSkip, numFail int
+		timeTaken                           time.Duration
+	}
+
+	// Statistics per output format
+	statsByFmt := map[outputFormat]*stats{}
+	for _, format := range formats {
+		statsByFmt[format] = &stats{}
+	}
+
+	// Print the table of file x format and gather per-format stats
+	failures := []failure{}
+	filenameColumnWidth := maxStringLen(files)
+	if maxFilenameColumnWidth > 0 {
+		filenameColumnWidth = maxFilenameColumnWidth
+	}
+
+	red := color.New(color.FgRed)
+	green := color.New(color.FgGreen)
+	yellow := color.New(color.FgYellow)
+	cyan := color.New(color.FgCyan)
+
+	printFormatsHeader := func() {
+		fmt.Printf(strings.Repeat(" ", filenameColumnWidth))
+		fmt.Printf(" ┃ ")
+		for _, format := range formats {
+			cyan.Printf(alignCenter(format, formatWidth(format)))
+			fmt.Printf(" │ ")
+		}
+		fmt.Println()
+	}
+	printHorizontalLine := func() {
+		fmt.Printf(strings.Repeat("━", filenameColumnWidth))
+		fmt.Printf("━╋━")
+		for _, format := range formats {
+			fmt.Printf(strings.Repeat("━", formatWidth(format)))
+			fmt.Printf("━┿━")
+		}
+		fmt.Println()
+	}
+
+	fmt.Println()
+
+	printFormatsHeader()
+	printHorizontalLine()
+
+	for i, file := range files {
+		results := results[i]
+
+		row := &strings.Builder{}
+		rowAllPassed := true
+
+		filenameLength := utf8.RuneCountInString(file)
+		shortFile := file
+		if filenameLength > filenameColumnWidth {
+			shortFile = "..." + file[filenameLength-filenameColumnWidth+3:]
+		}
+
+		fmt.Fprintf(row, alignRight(shortFile, filenameColumnWidth))
+		fmt.Fprintf(row, " ┃ ")
+		for _, format := range formats {
+			columnWidth := formatWidth(format)
+			result := <-results[format]
+			stats := statsByFmt[format]
+			stats.numTests++
+			stats.timeTaken += result.timeTaken
+			if err := result.err; err != nil {
+				failures = append(failures, failure{
+					file: file, format: format, err: err,
+				})
+			}
+			switch result.code {
+			case pass:
+				green.Fprintf(row, alignCenter("PASS", columnWidth))
+				stats.numPass++
+			case fail:
+				red.Fprintf(row, alignCenter("FAIL", columnWidth))
+				rowAllPassed = false
+				stats.numFail++
+			case skip:
+				yellow.Fprintf(row, alignCenter("SKIP", columnWidth))
+				rowAllPassed = false
+				stats.numSkip++
+			default:
+				fmt.Fprintf(row, alignCenter(result.code, columnWidth))
+				rowAllPassed = false
+			}
+			fmt.Fprintf(row, " │ ")
+		}
+
+		if verbose || !rowAllPassed {
+			fmt.Fprintln(color.Output, row)
+		}
+	}
+
+	printHorizontalLine()
+	printFormatsHeader()
+	printHorizontalLine()
+	printStat := func(col *color.Color, name string, num func(*stats) int) {
+		row := &strings.Builder{}
+		anyNonZero := false
+		for _, format := range formats {
+			columnWidth := formatWidth(format)
+			count := num(statsByFmt[format])
+			if count > 0 {
+				col.Fprintf(row, alignLeft(count, columnWidth))
+				anyNonZero = true
+			} else {
+				fmt.Fprintf(row, alignLeft(count, columnWidth))
+			}
+			fmt.Fprintf(row, " │ ")
+		}
+
+		if !anyNonZero {
+			return
+		}
+		col.Printf(alignRight(name, filenameColumnWidth))
+		fmt.Printf(" ┃ ")
+		fmt.Fprintln(color.Output, row)
+
+		col.Printf(strings.Repeat(" ", filenameColumnWidth))
+		fmt.Printf(" ┃ ")
+		for _, format := range formats {
+			columnWidth := formatWidth(format)
+			stats := statsByFmt[format]
+			count := num(stats)
+			percent := percentage(count, stats.numTests)
+			if count > 0 {
+				col.Print(alignRight(percent, columnWidth))
+			} else {
+				fmt.Print(alignRight(percent, columnWidth))
+			}
+			fmt.Printf(" │ ")
+		}
+		fmt.Println()
+	}
+	printStat(green, "PASS", func(s *stats) int { return s.numPass })
+	printStat(yellow, "SKIP", func(s *stats) int { return s.numSkip })
+	printStat(red, "FAIL", func(s *stats) int { return s.numFail })
+
+	cyan.Printf(alignRight("TIME", filenameColumnWidth))
+	fmt.Printf(" ┃ ")
+	for _, format := range formats {
+		timeTaken := printDuration(statsByFmt[format].timeTaken)
+		cyan.Printf(alignLeft(timeTaken, formatWidth(format)))
+		fmt.Printf(" │ ")
+	}
+	fmt.Println()
+
+	for _, f := range failures {
+		color.Set(color.FgBlue)
+		fmt.Printf("%s ", f.file)
+		color.Set(color.FgCyan)
+		fmt.Printf("%s ", f.format)
+		color.Set(color.FgRed)
+		fmt.Println("FAIL")
+		color.Unset()
+		fmt.Println(indent(f.err.Error(), 4))
+	}
+	if len(failures) > 0 {
+		fmt.Println()
+	}
+
+	allStats := stats{}
+	for _, format := range formats {
+		stats := statsByFmt[format]
+		allStats.numTests += stats.numTests
+		allStats.numPass += stats.numPass
+		allStats.numSkip += stats.numSkip
+		allStats.numFail += stats.numFail
+	}
+
+	fmt.Printf("%d tests run", allStats.numTests)
+	if allStats.numPass > 0 {
+		fmt.Printf(", ")
+		color.Set(color.FgGreen)
+		fmt.Printf("%d tests pass", allStats.numPass)
+		color.Unset()
+	} else {
+		fmt.Printf(", %d tests pass", allStats.numPass)
+	}
+	if allStats.numSkip > 0 {
+		fmt.Printf(", ")
+		color.Set(color.FgYellow)
+		fmt.Printf("%d tests skipped", allStats.numSkip)
+		color.Unset()
+	} else {
+		fmt.Printf(", %d tests skipped", allStats.numSkip)
+	}
+	if allStats.numFail > 0 {
+		fmt.Printf(", ")
+		color.Set(color.FgRed)
+		fmt.Printf("%d tests failed", allStats.numFail)
+		color.Unset()
+	} else {
+		fmt.Printf(", %d tests failed", allStats.numFail)
+	}
+	fmt.Println()
+	fmt.Println()
+
+	if allStats.numFail > 0 {
+		os.Exit(1)
+	}
+
+	return nil
+}
+
+// Structures to hold the results of the tests
+type statusCode string
+
+const (
+	fail statusCode = "FAIL"
+	pass statusCode = "PASS"
+	skip statusCode = "SKIP"
+)
+
+type status struct {
+	code      statusCode
+	err       error
+	timeTaken time.Duration
+}
+
+type job struct {
+	file   string
+	flags  []string
+	format outputFormat
+	result chan status
+}
+
+func (j job) run(wd, exe string, fxc, fxcAndDxc bool, dxcPath, xcrunPath string, generateExpected, generateSkip bool) {
+	j.result <- func() status {
+		// expectedFilePath is the path to the expected output file for the given test
+		expectedFilePath := j.file + ".expected." + string(j.format)
+
+		// Is there an expected output file? If so, load it.
+		expected, expectedFileExists := "", false
+		if content, err := ioutil.ReadFile(expectedFilePath); err == nil {
+			expected = string(content)
+			expectedFileExists = true
+		}
+
+		skipped := false
+		if strings.HasPrefix(expected, "SKIP") { // Special SKIP token
+			skipped = true
+		}
+
+		expected = strings.ReplaceAll(expected, "\r\n", "\n")
+
+		file, err := filepath.Rel(wd, j.file)
+		if err != nil {
+			file = j.file
+		}
+
+		// Make relative paths use forward slash separators (on Windows) so that paths in tint
+		// output match expected output that contain errors
+		file = strings.ReplaceAll(file, `\`, `/`)
+
+		args := []string{
+			file,
+			"--format", string(j.format),
+		}
+
+		// Can we validate?
+		validate := false
+		switch j.format {
+		case wgsl:
+			validate = true
+		case spvasm, glsl:
+			args = append(args, "--validate") // spirv-val and glslang are statically linked, always available
+			validate = true
+		case hlsl:
+			// Handled below
+		case msl:
+			if xcrunPath != "" {
+				args = append(args, "--xcrun", xcrunPath)
+				validate = true
+			}
+		}
+
+		// Invoke the compiler...
+		ok := false
+		var out string
+		start := time.Now()
+		if j.format == hlsl {
+			// If fxcAndDxc is set, run FXC first as it's more likely to fail, then DXC iff FXC succeeded.
+			if fxc || fxcAndDxc {
+				validate = true
+				args_fxc := append(args, "--fxc")
+				args_fxc = append(args_fxc, j.flags...)
+				ok, out = invoke(wd, exe, args_fxc...)
+			}
+
+			if dxcPath != "" && (!fxc || (fxcAndDxc && ok)) {
+				validate = true
+				args_dxc := append(args, "--dxc", dxcPath)
+				args_dxc = append(args_dxc, j.flags...)
+				ok, out = invoke(wd, exe, args_dxc...)
+			}
+
+			// If we didn't run either fxc or dxc validation, run as usual
+			if !validate {
+				args = append(args, j.flags...)
+				ok, out = invoke(wd, exe, args...)
+			}
+
+		} else {
+			args = append(args, j.flags...)
+			ok, out = invoke(wd, exe, args...)
+		}
+		timeTaken := time.Since(start)
+		out = strings.ReplaceAll(out, "\r\n", "\n")
+		matched := expected == "" || expected == out
+
+		canEmitPassExpectationFile := true
+		for _, noPass := range dirsWithNoPassExpectations {
+			if strings.Contains(j.file, filepath.FromSlash(noPass)) {
+				canEmitPassExpectationFile = false
+				break
+			}
+		}
+
+		saveExpectedFile := func(path string, content string) error {
+			return ioutil.WriteFile(path, []byte(content), 0666)
+		}
+
+		if ok && generateExpected && (validate || !skipped) {
+			// User requested to update PASS expectations, and test passed.
+			if canEmitPassExpectationFile {
+				saveExpectedFile(expectedFilePath, out)
+			} else if expectedFileExists {
+				// Test lives in a directory where we do not want to save PASS
+				// files, and there already exists an expectation file. Test has
+				// likely started passing. Delete the old expectation.
+				os.Remove(expectedFilePath)
+			}
+			matched = true // test passed and matched expectations
+		}
+
+		switch {
+		case ok && matched:
+			// Test passed
+			return status{code: pass, timeTaken: timeTaken}
+
+			//       --- Below this point the test has failed ---
+
+		case skipped:
+			if generateSkip {
+				saveExpectedFile(expectedFilePath, "SKIP: FAILED\n\n"+out)
+			}
+			return status{code: skip, timeTaken: timeTaken}
+
+		case !ok:
+			// Compiler returned non-zero exit code
+			if generateSkip {
+				saveExpectedFile(expectedFilePath, "SKIP: FAILED\n\n"+out)
+			}
+			err := fmt.Errorf("%s", out)
+			return status{code: fail, err: err, timeTaken: timeTaken}
+
+		default:
+			// Compiler returned zero exit code, or output was not as expected
+			if generateSkip {
+				saveExpectedFile(expectedFilePath, "SKIP: FAILED\n\n"+out)
+			}
+
+			// Expected output did not match
+			dmp := diffmatchpatch.New()
+			diff := dmp.DiffPrettyText(dmp.DiffMain(expected, out, true))
+			err := fmt.Errorf(`Output was not as expected
+
+--------------------------------------------------------------------------------
+-- Expected:                                                                  --
+--------------------------------------------------------------------------------
+%s
+
+--------------------------------------------------------------------------------
+-- Got:                                                                       --
+--------------------------------------------------------------------------------
+%s
+
+--------------------------------------------------------------------------------
+-- Diff:                                                                      --
+--------------------------------------------------------------------------------
+%s`,
+				expected, out, diff)
+			return status{code: fail, err: err, timeTaken: timeTaken}
+		}
+	}()
+}
+
+// indent returns the string 's' indented with 'n' whitespace characters
+func indent(s string, n int) string {
+	tab := strings.Repeat(" ", n)
+	return tab + strings.ReplaceAll(s, "\n", "\n"+tab)
+}
+
+// alignLeft returns the string of 'val' padded so that it is aligned left in
+// a column of the given width
+func alignLeft(val interface{}, width int) string {
+	s := fmt.Sprint(val)
+	padding := width - utf8.RuneCountInString(s)
+	if padding < 0 {
+		return s
+	}
+	return s + strings.Repeat(" ", padding)
+}
+
+// alignCenter returns the string of 'val' padded so that it is centered in a
+// column of the given width.
+func alignCenter(val interface{}, width int) string {
+	s := fmt.Sprint(val)
+	padding := width - utf8.RuneCountInString(s)
+	if padding < 0 {
+		return s
+	}
+	return strings.Repeat(" ", padding/2) + s + strings.Repeat(" ", (padding+1)/2)
+}
+
+// alignRight returns the string of 'val' padded so that it is aligned right in
+// a column of the given width
+func alignRight(val interface{}, width int) string {
+	s := fmt.Sprint(val)
+	padding := width - utf8.RuneCountInString(s)
+	if padding < 0 {
+		return s
+	}
+	return strings.Repeat(" ", padding) + s
+}
+
+// maxStringLen returns the maximum number of runes found in all the strings in
+// 'l'
+func maxStringLen(l []string) int {
+	max := 0
+	for _, s := range l {
+		if c := utf8.RuneCountInString(s); c > max {
+			max = c
+		}
+	}
+	return max
+}
+
+// formatWidth returns the width in runes for the outputFormat column 'b'
+func formatWidth(b outputFormat) int {
+	const min = 6
+	c := utf8.RuneCountInString(string(b))
+	if c < min {
+		return min
+	}
+	return c
+}
+
+// percentage returns the percentage of n out of total as a string
+func percentage(n, total int) string {
+	if total == 0 {
+		return "-"
+	}
+	f := float64(n) / float64(total)
+	return fmt.Sprintf("%.1f%c", f*100.0, '%')
+}
+
+// invoke runs the executable 'exe' with the provided arguments.
+func invoke(wd, exe string, args ...string) (ok bool, output string) {
+	ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
+	defer cancel()
+
+	cmd := exec.CommandContext(ctx, exe, args...)
+	cmd.Dir = wd
+	out, err := cmd.CombinedOutput()
+	str := string(out)
+	if err != nil {
+		if ctx.Err() == context.DeadlineExceeded {
+			return false, fmt.Sprintf("test timed out after %v", testTimeout)
+		}
+		if str != "" {
+			return false, str
+		}
+		return false, err.Error()
+	}
+	return true, str
+}
+
+var reFlags = regexp.MustCompile(` *\/\/ *flags:(.*)\n`)
+
+// parseFlags looks for a `// flags:` header at the start of the file with the
+// given path, returning each of the space delimited tokens that follow for the
+// line
+func parseFlags(path string) []string {
+	content, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil
+	}
+	header := strings.SplitN(string(content), "\n", 1)[0]
+	m := reFlags.FindStringSubmatch(header)
+	if len(m) != 2 {
+		return nil
+	}
+	return strings.Split(m[1], " ")
+}
+
+func printDuration(d time.Duration) string {
+	sec := int(d.Seconds())
+	min := int(sec) / 60
+	hour := min / 60
+	min -= hour * 60
+	sec -= min * 60
+	sb := &strings.Builder{}
+	if hour > 0 {
+		fmt.Fprintf(sb, "%dh", hour)
+	}
+	if min > 0 {
+		fmt.Fprintf(sb, "%dm", min)
+	}
+	if sec > 0 {
+		fmt.Fprintf(sb, "%ds", sec)
+	}
+	return sb.String()
+}

tools/src/cmd/trim-includes/config.cfg

@@ -0,0 +1,6 @@
+{
+  "paths": [
+    { "include": [ "src/**.h", "src/**.cc" ] },
+    { "exclude": [ "src/**_windows.*", "src/**_other.*" ] }
+  ]
+}

tools/src/cmd/trim-includes/main.go

@@ -0,0 +1,268 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// trim-includes is a tool to try removing unnecessary include statements from
+// all .cc and .h files in the tint project.
+//
+// trim-includes removes each #include from each file, then runs the provided
+// build script to determine whether the line was necessary. If the include is
+// required, it is restored, otherwise it is left deleted.
+// After all the #include statements have been tested, the file is
+// clang-formatted and git staged.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	"dawn.googlesource.com/tint/tools/src/fileutils"
+	"dawn.googlesource.com/tint/tools/src/glob"
+)
+
+var (
+	// Path to the build script to run after each attempting to remove each
+	// #include
+	buildScript = ""
+)
+
+func main() {
+	if err := run(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func showUsage() {
+	fmt.Println(`
+trim-includes is a tool to try removing unnecessary include statements from all
+.cc and .h files in the tint project.
+
+trim-includes removes each #include from each file, then runs the provided build
+script to determine whether the line was necessary. If the include is required,
+it is restored, otherwise it is left deleted.
+After all the #include statements have been tested, the file is clang-formatted
+and git staged.
+
+Usage:
+  trim-includes <path-to-build-script>`)
+	os.Exit(1)
+}
+
+func run() error {
+	flag.Parse()
+	args := flag.Args()
+	if len(args) < 1 {
+		showUsage()
+	}
+
+	var err error
+	buildScript, err = exec.LookPath(args[0])
+	if err != nil {
+		return err
+	}
+	buildScript, err = filepath.Abs(buildScript)
+	if err != nil {
+		return err
+	}
+
+	cfg, err := glob.LoadConfig("config.cfg")
+	if err != nil {
+		return err
+	}
+
+	fmt.Println("Checking the project builds with no changes...")
+	ok, err := tryBuild()
+	if err != nil {
+		return err
+	}
+	if !ok {
+		return fmt.Errorf("Project does not build without edits")
+	}
+
+	fmt.Println("Scanning for files...")
+	paths, err := glob.Scan(fileutils.ProjectRoot(), cfg)
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("Loading %v source files...\n", len(paths))
+	files, err := loadFiles(paths)
+	if err != nil {
+		return err
+	}
+
+	for fileIdx, file := range files {
+		fmt.Printf("[%d/%d]: %v\n", fileIdx+1, len(files), file.path)
+		includeLines := file.includesLineNumbers()
+		enabled := make(map[int]bool, len(file.lines))
+		for i := range file.lines {
+			enabled[i] = true
+		}
+		for includeIdx, line := range includeLines {
+			fmt.Printf("    [%d/%d]: %v", includeIdx+1, len(includeLines), file.lines[line])
+			enabled[line] = false
+			if err := file.save(enabled); err != nil {
+				return err
+			}
+			ok, err := tryBuild()
+			if err != nil {
+				return err
+			}
+			if ok {
+				fmt.Printf(" removed\n")
+				// Wait a bit so file timestamps get an opportunity to change.
+				// Attempting to save too soon after a successful build may
+				// result in a false-positive build.
+				time.Sleep(time.Second)
+			} else {
+				fmt.Printf(" required\n")
+				enabled[line] = true
+			}
+		}
+		if err := file.save(enabled); err != nil {
+			return err
+		}
+		if err := file.format(); err != nil {
+			return err
+		}
+		if err := file.stage(); err != nil {
+			return err
+		}
+	}
+	fmt.Println("Done")
+	return nil
+}
+
+// Attempt to build the project. Returns true on successful build, false if
+// there was a build failure.
+func tryBuild() (bool, error) {
+	cmd := exec.Command("sh", "-c", buildScript)
+	out, err := cmd.CombinedOutput()
+	switch err := err.(type) {
+	case nil:
+		return cmd.ProcessState.Success(), nil
+	case *exec.ExitError:
+		return false, nil
+	default:
+		return false, fmt.Errorf("Test failed with error: %v\n%v", err, string(out))
+	}
+}
+
+type file struct {
+	path  string
+	lines []string
+}
+
+var includeRE = regexp.MustCompile(`^\s*#include (?:\"([^"]*)\"|:?\<([^"]*)\>)`)
+
+// Returns the file path with the extension stripped
+func stripExtension(path string) string {
+	if dot := strings.IndexRune(path, '.'); dot > 0 {
+		return path[:dot]
+	}
+	return path
+}
+
+// Returns the zero-based line numbers of all #include statements in the file
+func (f *file) includesLineNumbers() []int {
+	out := []int{}
+	for i, l := range f.lines {
+		matches := includeRE.FindStringSubmatch(l)
+		if len(matches) == 0 {
+			continue
+		}
+
+		include := matches[1]
+		if include == "" {
+			include = matches[2]
+		}
+
+		if strings.HasSuffix(stripExtension(f.path), stripExtension(include)) {
+			// Don't remove #include for header of cc
+			continue
+		}
+
+		out = append(out, i)
+	}
+	return out
+}
+
+// Saves the file, omitting the lines with the zero-based line number that are
+// either not in `lines` or have a `false` value.
+func (f *file) save(lines map[int]bool) error {
+	content := []string{}
+	for i, l := range f.lines {
+		if lines[i] {
+			content = append(content, l)
+		}
+	}
+	data := []byte(strings.Join(content, "\n"))
+	return ioutil.WriteFile(f.path, data, 0666)
+}
+
+// Runs clang-format on the file
+func (f *file) format() error {
+	err := exec.Command("clang-format", "-i", f.path).Run()
+	if err != nil {
+		return fmt.Errorf("Couldn't format file '%v': %w", f.path, err)
+	}
+	return nil
+}
+
+// Runs git add on the file
+func (f *file) stage() error {
+	err := exec.Command("git", "-C", fileutils.ProjectRoot(), "add", f.path).Run()
+	if err != nil {
+		return fmt.Errorf("Couldn't stage file '%v': %w", f.path, err)
+	}
+	return nil
+}
+
+// Loads all the files with the given file paths, splitting their content into
+// into lines.
+func loadFiles(paths []string) ([]file, error) {
+	wg := sync.WaitGroup{}
+	wg.Add(len(paths))
+	files := make([]file, len(paths))
+	errs := make([]error, len(paths))
+	for i, path := range paths {
+		i, path := i, filepath.Join(fileutils.ProjectRoot(), path)
+		go func() {
+			defer wg.Done()
+			body, err := ioutil.ReadFile(path)
+			if err != nil {
+				errs[i] = fmt.Errorf("Failed to open %v: %w", path, err)
+			} else {
+				content := string(body)
+				lines := strings.Split(content, "\n")
+				files[i] = file{path: path, lines: lines}
+			}
+		}()
+	}
+	wg.Wait()
+	for _, err := range errs {
+		if err != nil {
+			return nil, err
+		}
+	}
+	return files, nil
+}

tools/src/fileutils/fileutils_common.go

@@ -0,0 +1,45 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package fileutils contains utility functions for files
+package fileutils
+
+import (
+	"path/filepath"
+	"runtime"
+)
+
+// GoSourcePath returns the absolute path to the .go file that calls the
+// function
+func GoSourcePath() string {
+	_, filename, _, ok := runtime.Caller(1)
+	if !ok {
+		panic("No caller information")
+	}
+	path, err := filepath.Abs(filename)
+	if err != nil {
+		panic(err)
+	}
+	return path
+}
+
+// ProjectRoot returns the path to the tint project root
+func ProjectRoot() string {
+	toolRoot := filepath.Dir(GoSourcePath())
+	root, err := filepath.Abs(filepath.Join(toolRoot, "../../.."))
+	if err != nil {
+		panic(err)
+	}
+	return root
+}

tools/src/fileutils/fileutils_other.go

@@ -0,0 +1,31 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !windows
+
+// Package fileutils contains utility functions for files
+package fileutils
+
+import (
+	"os"
+)
+
+// IsExe returns true if the file at path is an executable
+func IsExe(path string) bool {
+	s, err := os.Stat(path)
+	if err != nil {
+		return false
+	}
+	return s.Mode()&0100 != 0
+}

tools/src/fileutils/fileutils_test.go

@@ -0,0 +1,38 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fileutils_test
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"dawn.googlesource.com/tint/tools/src/fileutils"
+)
+
+func TestGoSourcePath(t *testing.T) {
+	p := fileutils.GoSourcePath()
+	if !strings.HasSuffix(p, "fileutils/fileutils_test.go") {
+		t.Errorf("GoSourcePath() returned %v", p)
+	}
+}
+
+func TestProjectRoot(t *testing.T) {
+	p := filepath.Join(fileutils.ProjectRoot(), "tint_overrides_with_defaults.gni")
+	if _, err := os.Stat(p); os.IsNotExist(err) {
+		t.Errorf("ProjectRoot() returned %v", p)
+	}
+}

tools/src/fileutils/fileutils_windows.go

@@ -0,0 +1,21 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package fileutils contains utility functions for files
+package fileutils
+
+// IsExe returns true if the file at path is an executable
+func IsExe(path string) bool {
+	return true
+}

tools/src/gerrit/gerrit.go

@@ -0,0 +1,95 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// gerrit provides helpers for obtaining information from Tint's gerrit instance
+package gerrit
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"regexp"
+	"strings"
+
+	"github.com/andygrunwald/go-gerrit"
+)
+
+const URL = "https://dawn-review.googlesource.com/"
+
+// G is the interface to gerrit
+type G struct {
+	client        *gerrit.Client
+	authenticated bool
+}
+
+type Config struct {
+	Username string
+	Password string
+}
+
+func LoadCredentials() (user, pass string) {
+	cookiesFile := os.Getenv("HOME") + "/.gitcookies"
+	if cookies, err := ioutil.ReadFile(cookiesFile); err == nil {
+		re := regexp.MustCompile(`dawn-review.googlesource.com\s+(?:FALSE|TRUE)[\s/]+(?:FALSE|TRUE)\s+[0-9]+\s+.\s+(.*)=(.*)`)
+		match := re.FindStringSubmatch(string(cookies))
+		if len(match) == 3 {
+			return match[1], match[2]
+		}
+	}
+	return "", ""
+}
+
+func New(cfg Config) (*G, error) {
+	client, err := gerrit.NewClient(URL, nil)
+	if err != nil {
+		return nil, fmt.Errorf("couldn't create gerrit client: %w", err)
+	}
+
+	user, pass := cfg.Username, cfg.Password
+	if user == "" {
+		user, pass = LoadCredentials()
+	}
+
+	if user != "" {
+		client.Authentication.SetBasicAuth(user, pass)
+	}
+
+	return &G{client, user != ""}, nil
+}
+
+func (g *G) QueryChanges(queryParts ...string) (changes []gerrit.ChangeInfo, query string, err error) {
+	changes = []gerrit.ChangeInfo{}
+	query = strings.Join(queryParts, "+")
+	for {
+		batch, _, err := g.client.Changes.QueryChanges(&gerrit.QueryChangeOptions{
+			QueryOptions: gerrit.QueryOptions{Query: []string{query}},
+			Skip:         len(changes),
+		})
+		if err != nil {
+			if !g.authenticated {
+				err = fmt.Errorf(`query failed, possibly because of authentication.
+	See https://dawn-review.googlesource.com/new-password for obtaining a username
+	and password which can be provided with --gerrit-user and --gerrit-pass.
+	%w`, err)
+			}
+			return nil, "", err
+		}
+
+		changes = append(changes, *batch...)
+		if len(*batch) == 0 || !(*batch)[len(*batch)-1].MoreChanges {
+			break
+		}
+	}
+	return changes, query, nil
+}

tools/src/git/git.go

@@ -0,0 +1,381 @@
+// Copyright 2022 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package git
+
+import (
+	"context"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"net/url"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+// Hash is a 20 byte, git object hash.
+type Hash [20]byte
+
+func (h Hash) String() string { return hex.EncodeToString(h[:]) }
+
+// IsZero returns true if the hash h is all zeros
+func (h Hash) IsZero() bool {
+	zero := Hash{}
+	return h == zero
+}
+
+// ParseHash returns a Hash from a hexadecimal string.
+func ParseHash(s string) (Hash, error) {
+	b, err := hex.DecodeString(s)
+	if err != nil {
+		return Hash{}, fmt.Errorf("failed to parse hash '%v':\n  %w", s, err)
+	}
+	h := Hash{}
+	copy(h[:], b)
+	return h, nil
+}
+
+// The timeout for git operations if no other timeout is specified
+var DefaultTimeout = time.Minute
+
+// Git wraps the 'git' executable
+type Git struct {
+	// Path to the git executable
+	exe string
+	// Debug flag to print all command to the `git` executable
+	LogAllActions bool
+}
+
+// New returns a new Git instance
+func New(exe string) (*Git, error) {
+	if _, err := os.Stat(exe); err != nil {
+		return nil, err
+	}
+	return &Git{exe: exe}, nil
+}
+
+// Auth holds git authentication credentials
+type Auth struct {
+	Username string
+	Password string
+}
+
+// Empty return true if there's no username or password for authentication
+func (a Auth) Empty() bool {
+	return a.Username == "" && a.Password == ""
+}
+
+// ErrRepositoryDoesNotExist indicates that a repository does not exist
+var ErrRepositoryDoesNotExist = errors.New("repository does not exist")
+
+// Open opens an existing git repo at path. If the repository does not exist at
+// path then ErrRepositoryDoesNotExist is returned.
+func (g Git) Open(path string) (*Repository, error) {
+	info, err := os.Stat(filepath.Join(path, ".git"))
+	if err != nil || !info.IsDir() {
+		return nil, ErrRepositoryDoesNotExist
+	}
+	return &Repository{g, path}, nil
+}
+
+// Optional settings for Git.Clone
+type CloneOptions struct {
+	// If specified then the given branch will be cloned instead of the default
+	Branch string
+	// Timeout for the operation
+	Timeout time.Duration
+	// Authentication for the clone
+	Auth Auth
+}
+
+// Clone performs a clone of the repository at url to path.
+func (g Git) Clone(path, url string, opt *CloneOptions) (*Repository, error) {
+	if err := os.MkdirAll(path, 0777); err != nil {
+		return nil, err
+	}
+	if opt == nil {
+		opt = &CloneOptions{}
+	}
+	url, err := opt.Auth.addToURL(url)
+	if err != nil {
+		return nil, err
+	}
+	r := &Repository{g, path}
+	args := []string{"clone", url, "."}
+	if opt.Branch != "" {
+		args = append(args, "--branch", opt.Branch)
+	}
+	if _, err := r.run(opt.Timeout, args...); err != nil {
+		return nil, err
+	}
+	return r, nil
+}
+
+// Repository points to a git repository
+type Repository struct {
+	// Path to the 'git' executable
+	Git Git
+	// Repo directory
+	Path string
+}
+
+// Optional settings for Repository.Fetch
+type FetchOptions struct {
+	// The remote name. Defaults to 'origin'
+	Remote string
+	// Timeout for the operation
+	Timeout time.Duration
+	// Git authentication for the remote
+	Auth Auth
+}
+
+// Fetch performs a fetch of a reference from the remote, returning the Hash of
+// the fetched reference.
+func (r Repository) Fetch(ref string, opt *FetchOptions) (Hash, error) {
+	if opt == nil {
+		opt = &FetchOptions{}
+	}
+	if opt.Remote == "" {
+		opt.Remote = "origin"
+	}
+	if _, err := r.run(opt.Timeout, "fetch", opt.Remote, ref); err != nil {
+		return Hash{}, err
+	}
+	out, err := r.run(0, "rev-parse", "FETCH_HEAD")
+	if err != nil {
+		return Hash{}, err
+	}
+	return ParseHash(out)
+}
+
+// Optional settings for Repository.Push
+type PushOptions struct {
+	// The remote name. Defaults to 'origin'
+	Remote string
+	// Timeout for the operation
+	Timeout time.Duration
+	// Git authentication for the remote
+	Auth Auth
+}
+
+// Push performs a push of the local reference to the remote reference.
+func (r Repository) Push(localRef, remoteRef string, opt *PushOptions) error {
+	if opt == nil {
+		opt = &PushOptions{}
+	}
+	if opt.Remote == "" {
+		opt.Remote = "origin"
+	}
+	url, err := r.run(opt.Timeout, "remote", "get-url", opt.Remote)
+	if err != nil {
+		return err
+	}
+	url, err = opt.Auth.addToURL(url)
+	if err != nil {
+		return err
+	}
+	if _, err := r.run(opt.Timeout, "push", url, localRef+":"+remoteRef); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Optional settings for Repository.Add
+type AddOptions struct {
+	// Timeout for the operation
+	Timeout time.Duration
+	// Git authentication for the remote
+	Auth Auth
+}
+
+// Add stages the listed files
+func (r Repository) Add(path string, opt *AddOptions) error {
+	if opt == nil {
+		opt = &AddOptions{}
+	}
+	if _, err := r.run(opt.Timeout, "add", path); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Optional settings for Repository.Commit
+type CommitOptions struct {
+	// Timeout for the operation
+	Timeout time.Duration
+	// Author name
+	AuthorName string
+	// Author email address
+	AuthorEmail string
+}
+
+// Commit commits the staged files with the given message, returning the hash of
+// commit
+func (r Repository) Commit(msg string, opt *CommitOptions) (Hash, error) {
+	if opt == nil {
+		opt = &CommitOptions{}
+	}
+	args := []string{"commit", "-m", msg}
+	if opt.AuthorName != "" || opt.AuthorEmail != "" {
+		args = append(args, "--author", fmt.Sprintf("%v <%v>", opt.AuthorName, opt.AuthorEmail))
+	}
+	if _, err := r.run(opt.Timeout, args...); err != nil {
+		return Hash{}, err
+	}
+	out, err := r.run(0, "rev-parse", "HEAD")
+	if err != nil {
+		return Hash{}, err
+	}
+	return ParseHash(out)
+}
+
+// Optional settings for Repository.Checkout
+type CheckoutOptions struct {
+	// Timeout for the operation
+	Timeout time.Duration
+}
+
+// Checkout performs a checkout of a reference.
+func (r Repository) Checkout(ref string, opt *CheckoutOptions) error {
+	if opt == nil {
+		opt = &CheckoutOptions{}
+	}
+	if _, err := r.run(opt.Timeout, "checkout", ref); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Optional settings for Repository.Log
+type LogOptions struct {
+	// The git reference to the oldest commit in the range to query.
+	From string
+	// The git reference to the newest commit in the range to query.
+	To string
+	// The maximum number of entries to return.
+	Count int
+	// Timeout for the operation
+	Timeout time.Duration
+}
+
+// CommitInfo describes a single git commit
+type CommitInfo struct {
+	Hash        Hash
+	Date        time.Time
+	Author      string
+	Subject     string
+	Description string
+}
+
+// Log returns the list of commits between two references (inclusive).
+// The first returned commit is the most recent.
+func (r Repository) Log(opt *LogOptions) ([]CommitInfo, error) {
+	if opt == nil {
+		opt = &LogOptions{}
+	}
+	args := []string{"log"}
+	rng := "HEAD"
+	if opt.To != "" {
+		rng = opt.To
+	}
+	if opt.From != "" {
+		rng = opt.From + "^.." + rng
+	}
+	args = append(args, rng, "--pretty=format:ǁ%Hǀ%cIǀ%an <%ae>ǀ%sǀ%b")
+	if opt.Count != 0 {
+		args = append(args, fmt.Sprintf("-%d", opt.Count))
+	}
+	out, err := r.run(opt.Timeout, args...)
+	if err != nil {
+		return nil, err
+	}
+	return parseLog(out)
+}
+
+func (r Repository) run(timeout time.Duration, args ...string) (string, error) {
+	return r.Git.run(r.Path, timeout, args...)
+}
+
+func (r Repository) runAll(timeout time.Duration, args ...[]string) error {
+	for _, a := range args {
+		if _, err := r.run(timeout, a...); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (g Git) run(dir string, timeout time.Duration, args ...string) (string, error) {
+	if timeout == 0 {
+		timeout = DefaultTimeout
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+	cmd := exec.CommandContext(ctx, g.exe, args...)
+	cmd.Dir = dir
+	if g.LogAllActions {
+		fmt.Printf("%v> %v %v\n", dir, g.exe, strings.Join(args, " "))
+	}
+	out, err := cmd.CombinedOutput()
+	if g.LogAllActions {
+		fmt.Println(string(out))
+	}
+	if err != nil {
+		return string(out), fmt.Errorf("%v> %v %v failed:\n  %w\n%v",
+			dir, g.exe, strings.Join(args, " "), err, string(out))
+	}
+	return strings.TrimSpace(string(out)), nil
+}
+
+func (a Auth) addToURL(u string) (string, error) {
+	if !a.Empty() {
+		modified, err := url.Parse(u)
+		if err != nil {
+			return "", fmt.Errorf("failed to parse url '%v': %v", u, err)
+		}
+		modified.User = url.UserPassword(a.Username, a.Password)
+		u = modified.String()
+	}
+	return u, nil
+}
+
+func parseLog(str string) ([]CommitInfo, error) {
+	msgs := strings.Split(str, "ǁ")
+	cls := make([]CommitInfo, 0, len(msgs))
+	for _, s := range msgs {
+		if parts := strings.Split(s, "ǀ"); len(parts) == 5 {
+			hash, err := ParseHash(parts[0])
+			if err != nil {
+				return nil, err
+			}
+			date, err := time.Parse(time.RFC3339, parts[1])
+			if err != nil {
+				return nil, err
+			}
+			cl := CommitInfo{
+				Hash:        hash,
+				Date:        date,
+				Author:      strings.TrimSpace(parts[2]),
+				Subject:     strings.TrimSpace(parts[3]),
+				Description: strings.TrimSpace(parts[4]),
+			}
+
+			cls = append(cls, cl)
+		}
+	}
+	return cls, nil
+}

tools/src/glob/glob.go

@@ -0,0 +1,203 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package glob provides file globbing utilities
+package glob
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"dawn.googlesource.com/tint/tools/src/match"
+)
+
+// Scan walks all files and subdirectories from root, returning those
+// that Config.shouldExamine() returns true for.
+func Scan(root string, cfg Config) ([]string, error) {
+	files := []string{}
+	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		rel, err := filepath.Rel(root, path)
+		if err != nil {
+			rel = path
+		}
+
+		if rel == ".git" {
+			return filepath.SkipDir
+		}
+
+		if !cfg.shouldExamine(root, path) {
+			return nil
+		}
+
+		if !info.IsDir() {
+			files = append(files, rel)
+		}
+
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return files, nil
+}
+
+// Configs is a slice of Config.
+type Configs []Config
+
+// Config is used to parse the JSON configuration file.
+type Config struct {
+	// Paths holds a number of JSON objects that contain either a "includes" or
+	// "excludes" key to an array of path patterns.
+	// Each path pattern is considered in turn to either include or exclude the
+	// file path for license scanning. Pattern use forward-slashes '/' for
+	// directory separators, and may use the following wildcards:
+	//  ?  - matches any single non-separator character
+	//  *  - matches any sequence of non-separator characters
+	//  ** - matches any sequence of characters including separators
+	//
+	// Rules are processed in the order in which they are declared, with later
+	// rules taking precedence over earlier rules.
+	//
+	// All files are excluded before the first rule is evaluated.
+	//
+	// Example:
+	//
+	// {
+	//   "paths": [
+	// 	  { "exclude": [ "out/*", "build/*" ] },
+	// 	  { "include": [ "out/foo.txt" ] }
+	//   ],
+	// }
+	Paths searchRules
+}
+
+// LoadConfig loads a config file at path.
+func LoadConfig(path string) (Config, error) {
+	cfgBody, err := ioutil.ReadFile(path)
+	if err != nil {
+		return Config{}, err
+	}
+	return ParseConfig(string(cfgBody))
+}
+
+// ParseConfig parses the config from a JSON string.
+func ParseConfig(config string) (Config, error) {
+	d := json.NewDecoder(strings.NewReader(config))
+	cfg := Config{}
+	if err := d.Decode(&cfg); err != nil {
+		return Config{}, err
+	}
+	return cfg, nil
+}
+
+// MustParseConfig parses the config from a JSON string, panicing if the config
+// does not parse
+func MustParseConfig(config string) Config {
+	d := json.NewDecoder(strings.NewReader(config))
+	cfg := Config{}
+	if err := d.Decode(&cfg); err != nil {
+		panic(fmt.Errorf("Failed to parse config: %w\nConfig:\n%v", err, config))
+	}
+	return cfg
+}
+
+// rule is a search path predicate.
+// root is the project relative path.
+// cond is the value to return if the rule doesn't either include or exclude.
+type rule func(path string, cond bool) bool
+
+// searchRules is a ordered list of search rules.
+// searchRules is its own type as it has to perform custom JSON unmarshalling.
+type searchRules []rule
+
+// UnmarshalJSON unmarshals the array of rules in the form:
+// { "include": [ ... ] } or { "exclude": [ ... ] }
+func (l *searchRules) UnmarshalJSON(body []byte) error {
+	type parsed struct {
+		Include []string
+		Exclude []string
+	}
+
+	p := []parsed{}
+	if err := json.NewDecoder(bytes.NewReader(body)).Decode(&p); err != nil {
+		return err
+	}
+
+	*l = searchRules{}
+	for _, rule := range p {
+		rule := rule
+		switch {
+		case len(rule.Include) > 0 && len(rule.Exclude) > 0:
+			return fmt.Errorf("Rule cannot contain both include and exclude")
+		case len(rule.Include) > 0:
+			tests := make([]match.Test, len(rule.Include))
+			for i, pattern := range rule.Include {
+				test, err := match.New(pattern)
+				if err != nil {
+					return err
+				}
+				tests[i] = test
+			}
+			*l = append(*l, func(path string, cond bool) bool {
+				for _, test := range tests {
+					if test(path) {
+						return true
+					}
+				}
+				return cond
+			})
+		case len(rule.Exclude) > 0:
+			tests := make([]match.Test, len(rule.Exclude))
+			for i, pattern := range rule.Exclude {
+				test, err := match.New(pattern)
+				if err != nil {
+					return err
+				}
+				tests[i] = test
+			}
+			*l = append(*l, func(path string, cond bool) bool {
+				for _, test := range tests {
+					if test(path) {
+						return false
+					}
+				}
+				return cond
+			})
+		}
+	}
+	return nil
+}
+
+// shouldExamine returns true if the file at absPath should be scanned.
+func (c Config) shouldExamine(root, absPath string) bool {
+	root = filepath.ToSlash(root)       // Canonicalize
+	absPath = filepath.ToSlash(absPath) // Canonicalize
+	relPath, err := filepath.Rel(root, absPath)
+	if err != nil {
+		return false
+	}
+	relPath = filepath.ToSlash(relPath) // Canonicalize
+
+	res := false
+	for _, rule := range c.Paths {
+		res = rule(relPath, res)
+	}
+
+	return res
+}

tools/src/list/list.go

@@ -0,0 +1,137 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package list provides utilities for handling lists of dynamically-typed elements
+package list
+
+import (
+	"fmt"
+	"reflect"
+)
+
+// List is an interface to a list of dynamically-typed elements
+type List interface {
+	// Count returns the number if items in the list
+	Count() int
+
+	// Get returns the element at the index i
+	Get(i int) interface{}
+
+	// Set assigns the element at the index i with v
+	Set(i int, v interface{})
+
+	// Append adds a single item, list, or slice of items to this List
+	Append(v interface{})
+
+	// Copy copies the elements at [dst..dst+count) to [src..src+count)
+	Copy(dst, src, count int)
+
+	// CopyFrom copies the elements [src..src+count) from the list l to the
+	// elements [dst..dst+count) in this list
+	CopyFrom(l List, dst, src, count int)
+
+	// Reduces the size of the list to count elements
+	Resize(count int)
+
+	// ElementType returns the type of the elements of the list
+	ElementType() reflect.Type
+}
+
+// Wrap returns a List that wraps a slice pointer
+func Wrap(s interface{}) List {
+	ptr := reflect.ValueOf(s)
+	if ptr.Kind() != reflect.Ptr || ptr.Elem().Kind() != reflect.Slice {
+		panic(fmt.Errorf("Wrap() must be called with a pointer to slice. Got: %T", s))
+	}
+	return list{ptr.Elem()}
+}
+
+// New returns a new list of element type elem for n items
+func New(elem reflect.Type, count int) List {
+	slice := reflect.SliceOf(elem)
+	return list{reflect.MakeSlice(slice, count, count)}
+}
+
+// Copy makes a shallow copy of the list
+func Copy(l List) List {
+	out := New(l.ElementType(), l.Count())
+	out.CopyFrom(l, 0, 0, l.Count())
+	return out
+}
+
+type list struct{ v reflect.Value }
+
+func (l list) Count() int {
+	return l.v.Len()
+}
+
+func (l list) Get(i int) interface{} {
+	return l.v.Index(i).Interface()
+}
+
+func (l list) Set(i int, v interface{}) {
+	l.v.Index(i).Set(reflect.ValueOf(v))
+}
+
+func (l list) Append(v interface{}) {
+	switch v := v.(type) {
+	case list:
+		l.v.Set(reflect.AppendSlice(l.v, reflect.Value(v.v)))
+	case List:
+		// v implements `List`, but isn't a `list`. Need to do a piece-wise copy
+		items := make([]reflect.Value, v.Count())
+		for i := range items {
+			items[i] = reflect.ValueOf(v.Get(i))
+		}
+		l.v.Set(reflect.Append(l.v, items...))
+	default:
+		r := reflect.ValueOf(v)
+		if r.Type() == l.v.Type() {
+			l.v.Set(reflect.AppendSlice(l.v, r))
+			return
+		}
+		l.v.Set(reflect.Append(l.v, reflect.ValueOf(v)))
+	}
+}
+
+func (l list) Copy(dst, src, count int) {
+	reflect.Copy(
+		l.v.Slice(dst, dst+count),
+		l.v.Slice(src, src+count),
+	)
+}
+
+func (l list) CopyFrom(o List, dst, src, count int) {
+	if o, ok := o.(list); ok {
+		reflect.Copy(
+			l.v.Slice(dst, dst+count),
+			o.v.Slice(src, src+count),
+		)
+	}
+	// v implements `List`, but isn't a `list`. Need to do a piece-wise copy
+	items := make([]reflect.Value, count)
+	for i := range items {
+		l.Set(dst+i, o.Get(src+i))
+	}
+}
+
+func (l list) Resize(count int) {
+	new := reflect.MakeSlice(l.v.Type(), count, count)
+	reflect.Copy(new, l.v)
+	l.v.Set(new)
+}
+
+func (l list) ElementType() reflect.Type {
+	return l.v.Type().Elem()
+}

tools/src/list/list_test.go

@@ -0,0 +1,227 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either exprel or implied.
+// See the License for the specific language governing permilions and
+// limitations under the License.
+
+package list_test
+
+import (
+	"reflect"
+	"testing"
+
+	"dawn.googlesource.com/tint/tools/src/list"
+)
+
+// A simple implementation of list.List. Many methods are just stubs
+type customList struct{}
+
+func (customList) Count() int                                { return 3 }
+func (customList) Get(i int) interface{}                     { return 10 + i*10 }
+func (customList) Set(i int, v interface{})                  {}
+func (customList) Append(v interface{})                      {}
+func (customList) Copy(dst, src, count int)                  {}
+func (customList) CopyFrom(l list.List, dst, src, count int) {}
+func (customList) Resize(count int)                          {}
+func (customList) ElementType() reflect.Type                 { return nil }
+
+var _ list.List = customList{} // Interface compliance check
+
+func TestNew(t *testing.T) {
+	l := list.New(reflect.TypeOf(0), 3)
+
+	if n := l.Count(); n != 3 {
+		t.Errorf("Count(0): %v", n)
+	}
+	if n := l.Get(0); n != 0 {
+		t.Errorf("Get(0): %v", n)
+	}
+	if n := l.Get(1); n != 0 {
+		t.Errorf("Get(1): %v", n)
+	}
+	if n := l.Get(2); n != 0 {
+		t.Errorf("Get(2): %v", n)
+	}
+}
+
+func TestCopy(t *testing.T) {
+	slice := []int{1, 2, 3}
+	l := list.Wrap(&slice)
+
+	c := list.Copy(l)
+
+	if n := c.Count(); n != 3 {
+		t.Errorf("Count(0): %v", n)
+	}
+	if n := c.Get(0); n != 1 {
+		t.Errorf("Get(0): %v", n)
+	}
+	if n := c.Get(1); n != 2 {
+		t.Errorf("Get(1): %v", n)
+	}
+	if n := c.Get(2); n != 3 {
+		t.Errorf("Get(2): %v", n)
+	}
+}
+
+func TestListCount(t *testing.T) {
+	slice := make([]int, 5)
+	l := list.Wrap(&slice)
+
+	if c := l.Count(); c != 5 {
+		t.Errorf("Count() is %v", c)
+	}
+}
+
+func TestListGrow(t *testing.T) {
+	slice := []int{}
+	l := list.Wrap(&slice)
+
+	l.Resize(10)
+
+	if len(slice) != 10 {
+		t.Errorf("len(slice) after Resize(10) is %v", len(slice))
+	}
+}
+
+func TestListShrink(t *testing.T) {
+	slice := make([]int, 10)
+	l := list.Wrap(&slice)
+
+	l.Resize(5)
+
+	if len(slice) != 5 {
+		t.Errorf("len(slice) after Resize(5) is %v", len(slice))
+	}
+}
+
+func TestListCopy(t *testing.T) {
+	slice := []int{0, 10, 20, 0, 0, 0}
+	l := list.Wrap(&slice)
+
+	l.Copy(3, 1, 2)
+
+	if !reflect.DeepEqual(slice, []int{0, 10, 20, 10, 20, 0}) {
+		t.Errorf("after Copy(), slice: %v", slice)
+	}
+}
+
+func TestListCopyFromList(t *testing.T) {
+	sliceA := []int{10, 20, 30, 40, 50, 60}
+	lA := list.Wrap(&sliceA)
+
+	sliceB := []int{1, 2, 3, 4, 5, 6}
+	lB := list.Wrap(&sliceB)
+
+	lA.CopyFrom(lB, 1, 2, 3)
+
+	if !reflect.DeepEqual(sliceA, []int{10, 3, 4, 5, 50, 60}) {
+		t.Errorf("after CopyFrom(), slice: %v", sliceA)
+	}
+}
+
+func TestListCopyFromCustomList(t *testing.T) {
+	sliceA := []int{10, 20, 30, 40, 50, 60}
+	lA := list.Wrap(&sliceA)
+
+	lA.CopyFrom(customList{}, 1, 2, 3)
+
+	if !reflect.DeepEqual(sliceA, []int{10, 30, 40, 50, 50, 60}) {
+		t.Errorf("after CopyFrom(), slice: %v", sliceA)
+	}
+}
+
+func TestListGet(t *testing.T) {
+	slice := []int{0, 10, 20, 10, 20}
+	l := list.Wrap(&slice)
+
+	if n := l.Get(0); n != 0 {
+		t.Errorf("Get(0): %v", n)
+	}
+	if n := l.Get(1); n != 10 {
+		t.Errorf("Get(1): %v", n)
+	}
+	if n := l.Get(2); n != 20 {
+		t.Errorf("Get(2): %v", n)
+	}
+	if n := l.Get(3); n != 10 {
+		t.Errorf("Get(3): %v", n)
+	}
+	if n := l.Get(4); n != 20 {
+		t.Errorf("Get(4): %v", n)
+	}
+}
+
+func TestListSet(t *testing.T) {
+	slice := []int{0, 10, 20, 10, 20}
+	l := list.Wrap(&slice)
+
+	l.Set(0, 50)
+	l.Set(2, 90)
+	l.Set(4, 60)
+
+	if !reflect.DeepEqual(slice, []int{50, 10, 90, 10, 60}) {
+		t.Errorf("after Set(), slice: %v", slice)
+	}
+}
+
+func TestListAppendItem(t *testing.T) {
+	slice := []int{1, 2, 3}
+	l := list.Wrap(&slice)
+
+	l.Append(9)
+
+	if c := len(slice); c != 4 {
+		t.Errorf("len(slice): %v", 4)
+	}
+	if n := slice[3]; n != 9 {
+		t.Errorf("slice[3]: %v", n)
+	}
+}
+
+func TestListAppendItems(t *testing.T) {
+	slice := []int{1, 2, 3}
+	l := list.Wrap(&slice)
+
+	l.Append([]int{9, 8, 7})
+
+	if !reflect.DeepEqual(slice, []int{1, 2, 3, 9, 8, 7}) {
+		t.Errorf("after Append(), slice: %v", slice)
+	}
+}
+
+func TestListAppendList(t *testing.T) {
+	sliceA := []int{1, 2, 3}
+	lA := list.Wrap(&sliceA)
+
+	sliceB := []int{9, 8, 7}
+	lB := list.Wrap(&sliceB)
+
+	lA.Append(lB)
+
+	if !reflect.DeepEqual(sliceA, []int{1, 2, 3, 9, 8, 7}) {
+		t.Errorf("after Append(), sliceA: %v", sliceA)
+	}
+	if !reflect.DeepEqual(sliceB, []int{9, 8, 7}) {
+		t.Errorf("after Append(), sliceB: %v", sliceB)
+	}
+}
+
+func TestListAppendCustomList(t *testing.T) {
+	sliceA := []int{1, 2, 3}
+	lA := list.Wrap(&sliceA)
+
+	lA.Append(customList{})
+
+	if !reflect.DeepEqual(sliceA, []int{1, 2, 3, 10, 20, 30}) {
+		t.Errorf("after Append(), sliceA: %v", sliceA)
+	}
+}

tools/src/lut/lut.go

@@ -0,0 +1,245 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package lut provides a look up table, which compresses indexed data
+package lut
+
+import (
+	"sort"
+
+	"dawn.googlesource.com/tint/tools/src/list"
+)
+
+// LUT is a look up table.
+// The table holds a number of items that are stored in a linear list.
+type LUT interface {
+	// Add adds a sequence of items to the table.
+	// items can be a single element, a slice of element, or a List of element.
+	// Returns a pointer to the offset of the first item in the table's list.
+	// The sequence of items stored at [offset, offset+N), where N is the
+	// number of items added will remain equal, even after calling Compact().
+	Add(items interface{}) *int
+	// Compact reorders the table items so that the table storage is compacted
+	// by shuffling data around and de-duplicating sequences of common data.
+	// Each originally added sequence is preserved in the resulting table, with
+	// the same contiguous ordering, but with a potentially different offset.
+	// Heuristics are used to shorten the table length, by exploiting common
+	// subsequences, and removing duplicate sequences.
+	// Note that shortest common superstring is NP-hard, so heuristics are used.
+	// Compact updates pointers returned by Add().
+	Compact()
+}
+
+// New returns a new look up table
+func New(storage list.List) LUT {
+	return &lut{storage: storage}
+}
+
+// A sequence represents a span of entries in the table
+type sequence struct {
+	offset *int // Pointer to the start index of the sequence
+	count  int  // Length of the sequence
+}
+
+// lut implements LUT
+type lut struct {
+	storage   list.List  // The List that backs this LUT
+	sequences []sequence // The entries in the LUT
+}
+
+func (t *lut) Add(items interface{}) *int {
+	offset := t.storage.Count()
+	t.storage.Append(items)
+	count := t.storage.Count() - offset
+	offsetPtr := &offset
+	t.sequences = append(t.sequences, sequence{offsetPtr, count})
+	return offsetPtr
+}
+
+func (t lut) Compact() {
+	// Generate int32 identifiers for each unique item in the table.
+	// We use these to compare items instead of comparing the real data as this
+	// function is comparison-heavy, and integer compares are cheap.
+	srcIDs := t.itemIDs()
+	dstIDs := make([]int32, len(srcIDs))
+
+	// Make a copy the data held in the table, use the copy as the source, and
+	// t.storage as the destination.
+	srcData := list.Copy(t.storage)
+	dstData := t.storage
+
+	// Sort all the sequences by length, with the largest first.
+	// This helps 'seed' the compacted form with the largest items first.
+	// This can improve the compaction as small sequences can pack into larger,
+	// placed items.
+	sort.Slice(t.sequences, func(i, j int) bool {
+		return t.sequences[i].count > t.sequences[j].count
+	})
+
+	// unplaced is the list of sequences that have not yet been placed.
+	// All sequences are initially unplaced.
+	unplaced := make([]sequence, len(t.sequences))
+	copy(unplaced, t.sequences)
+
+	// placed is the list of sequences that have been placed.
+	// Nothing is initially placed.
+	placed := make([]sequence, 0, len(t.sequences))
+
+	// remove removes the sequence in unplaced with the index i.
+	remove := func(i int) {
+		placed = append(placed, unplaced[i])
+		if i > 0 {
+			if i < len(unplaced)-1 {
+				copy(unplaced[i:], unplaced[i+1:])
+			}
+			unplaced = unplaced[:len(unplaced)-1]
+		} else {
+			unplaced = unplaced[1:]
+		}
+	}
+
+	// cp copies data from [srcOffset:srcOffset+count] to [dstOffset:dstOffset+count].
+	cp := func(dstOffset, srcOffset, count int) {
+		dstData.CopyFrom(srcData, dstOffset, srcOffset, count)
+		copy(
+			dstIDs[dstOffset:dstOffset+count],
+			srcIDs[srcOffset:srcOffset+count],
+		)
+	}
+
+	// match describes a sequence that can be placed.
+	type match struct {
+		dst int      // destination offset
+		src sequence // source sequence
+		len int      // number of items that matched
+		idx int      // sequence index
+	}
+
+	// number of items that have been placed.
+	newSize := 0
+
+	// While there's sequences to place...
+	for len(unplaced) > 0 {
+		// Place the next largest, unplaced sequence at the end of the new list
+		cp(newSize, *unplaced[0].offset, unplaced[0].count)
+		*unplaced[0].offset = newSize
+		newSize += unplaced[0].count
+		remove(0)
+
+		for {
+			// Look for the sequence with the longest match against the
+			// currently placed data. Any mismatches with currently placed data
+			// will nullify the match. The head or tail of this sequence may
+			// extend the currently placed data.
+			best := match{}
+
+			// For each unplaced sequence...
+			for i := 0; i < len(unplaced); i++ {
+				seq := unplaced[i]
+
+				if best.len >= seq.count {
+					// The best match is already at least as long as this
+					// sequence and sequences are sorted by size, so best cannot
+					// be beaten. Stop searching.
+					break
+				}
+
+				// Perform a full sweep from left to right, scoring the match...
+				for shift := -seq.count + 1; shift < newSize; shift++ {
+					dstS := max(shift, 0)
+					dstE := min(shift+seq.count, newSize)
+					count := dstE - dstS
+					srcS := *seq.offset - min(shift, 0)
+					srcE := srcS + count
+
+					if best.len < count {
+						if equal(srcIDs[srcS:srcE], dstIDs[dstS:dstE]) {
+							best = match{shift, seq, count, i}
+						}
+					}
+				}
+			}
+
+			if best.src.offset == nil {
+				// Nothing matched. Not even one element.
+				// Resort to placing the next largest sequence at the end.
+				break
+			}
+
+			if best.dst < 0 {
+				// Best match wants to place the sequence to the left of the
+				// current output. We have to shuffle everything...
+				n := -best.dst
+				dstData.Copy(n, 0, newSize)
+				copy(dstIDs[n:n+newSize], dstIDs)
+				newSize += n
+				best.dst = 0
+				for _, p := range placed {
+					*p.offset += n
+				}
+			}
+
+			// Place the best matching sequence.
+			cp(best.dst, *best.src.offset, best.src.count)
+			newSize = max(newSize, best.dst+best.src.count)
+			*best.src.offset = best.dst
+			remove(best.idx)
+		}
+	}
+
+	// Shrink the output buffer to the new size.
+	dstData.Resize(newSize)
+
+	// All done.
+}
+
+// Generate a set of identifiers for all the unique items in storage
+func (t lut) itemIDs() []int32 {
+	storageSize := t.storage.Count()
+	keys := make([]int32, storageSize)
+	dataToKey := map[interface{}]int32{}
+	for i := 0; i < storageSize; i++ {
+		data := t.storage.Get(i)
+		key, found := dataToKey[data]
+		if !found {
+			key = int32(len(dataToKey))
+			dataToKey[data] = key
+		}
+		keys[i] = key
+	}
+	return keys
+}
+
+func max(a, b int) int {
+	if a < b {
+		return b
+	}
+	return a
+}
+
+func min(a, b int) int {
+	if a > b {
+		return b
+	}
+	return a
+}
+
+func equal(a, b []int32) bool {
+	for i, v := range a {
+		if b[i] != v {
+			return false
+		}
+	}
+	return true
+}

tools/src/lut/lut_test.go

@@ -0,0 +1,66 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package lut_test
+
+import (
+	"testing"
+
+	"dawn.googlesource.com/tint/tools/src/list"
+	"dawn.googlesource.com/tint/tools/src/lut"
+)
+
+func TestCompactWithFragments(t *testing.T) {
+	runes := []rune{}
+	lut := lut.New(list.Wrap(&runes))
+	indices := []*int{
+		lut.Add([]rune("the life in your")),
+		lut.Add([]rune("in your life that count")),
+		lut.Add([]rune("In the end,")),
+		lut.Add([]rune("the life in")),
+		lut.Add([]rune("count. It's the")),
+		lut.Add([]rune("years")),
+		lut.Add([]rune("in your years.")),
+		lut.Add([]rune("it's not the years in")),
+		lut.Add([]rune("not the years")),
+		lut.Add([]rune("not the years in your")),
+		lut.Add([]rune("end, it's")),
+	}
+
+	lut.Compact()
+
+	expect := "In the end, it's not the years in your life that count. It's the life in your years."
+	got := string(runes)
+	if got != expect {
+		t.Errorf("Compact result was not as expected\nExpected: '%v'\nGot:      '%v'", expect, got)
+	}
+	expectedIndices := []int{
+		61, //                                                              the life in your
+		31, //                                in your life that count
+		0,  // In the end,
+		61, //                                                              the life in
+		49, //                                                  count. It's the
+		25, //                          years
+		70, //                                                                       in your years.
+		12, //             it's not the years in
+		17, //                  not the years
+		17, //                  not the years in your
+		7,  //        end, it's
+	}
+	for i, p := range indices {
+		if expected, got := expectedIndices[i], *p; expected != got {
+			t.Errorf("Index %v was not expected. Expected %v, got %v", i, expected, got)
+		}
+	}
+}

tools/src/match/match.go

@@ -0,0 +1,76 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package match provides functions for performing filepath [?,*,**] wildcard
+// matching.
+package match
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+)
+
+// Test is the match predicate returned by New.
+type Test func(path string) bool
+
+// New returns a Test function that returns true iff the path matches the
+// provided pattern.
+//
+// pattern uses forward-slashes for directory separators '/', and may use the
+// following wildcards:
+//  ?  - matches any single non-separator character
+//  *  - matches any sequence of non-separator characters
+//  ** - matches any sequence of characters including separators
+func New(pattern string) (Test, error) {
+	// Transform pattern into a regex by replacing the uses of `?`, `*`, `**`
+	// with corresponding regex patterns.
+	// As the pattern may contain other regex sequences, the string has to be
+	// escaped. So:
+	// a) Replace the patterns of `?`, `*`, `**` with unique placeholder tokens.
+	// b) Escape the expression so that other sequences don't confuse the regex
+	//    parser.
+	// c) Replace the placeholder tokens with the corresponding regex tokens.
+
+	// Temporary placeholder tokens
+	const (
+		starstar     = "••"
+		star         = "•"
+		questionmark = "¿"
+	)
+	// Check pattern doesn't contain any of our placeholder tokens
+	for _, r := range []rune{'•', '¿'} {
+		if strings.ContainsRune(pattern, r) {
+			return nil, fmt.Errorf("Pattern must not contain '%c'", r)
+		}
+	}
+	// Replace **, * and ? with placeholder tokens
+	subbed := pattern
+	subbed = strings.ReplaceAll(subbed, "**", starstar)
+	subbed = strings.ReplaceAll(subbed, "*", star)
+	subbed = strings.ReplaceAll(subbed, "?", questionmark)
+	// Escape any remaining regex characters
+	escaped := regexp.QuoteMeta(subbed)
+	// Insert regex matchers for the substituted tokens
+	regex := "^" + escaped + "$"
+	regex = strings.ReplaceAll(regex, starstar, ".*")
+	regex = strings.ReplaceAll(regex, star, "[^/]*")
+	regex = strings.ReplaceAll(regex, questionmark, "[^/]")
+
+	re, err := regexp.Compile(regex)
+	if err != nil {
+		return nil, fmt.Errorf(`Failed to compile regex "%v" for pattern "%v": %w`, regex, pattern, err)
+	}
+	return re.MatchString, nil
+}

tools/src/match/match_test.go

@@ -0,0 +1,106 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package match_test
+
+import (
+	"strings"
+	"testing"
+
+	"dawn.googlesource.com/tint/tools/src/match"
+)
+
+func TestMatch(t *testing.T) {
+	for _, test := range []struct {
+		pattern string
+		path    string
+		expect  bool
+	}{
+		{"a", "a", true},
+		{"b", "a", false},
+
+		{"?", "a", true},
+		{"a/?/c", "a/x/c", true},
+		{"a/??/c", "a/x/c", false},
+		{"a/??/c", "a/xx/c", true},
+		{"a/???/c", "a/x z/c", true},
+		{"a/?/c", "a/xx/c", false},
+		{"a/?/?/c", "a/x/y/c", true},
+		{"a/?/?/?/c", "a/x/y/z/c", true},
+		{"a/???/c", "a/x/y/c", false},
+		{"a/?????", "a/x/y/c", false},
+
+		{"*", "a", true},
+		{"*", "abc", true},
+		{"*", "abc 123", true},
+		{"*", "xxx/yyy", false},
+		{"*/*", "xxx/yyy", true},
+		{"*/*", "xxx/yyy/zzz", false},
+		{"*/*/c", "xxx/yyy/c", true},
+		{"a/*/*", "a/xxx/yyy", true},
+		{"a/*/c", "a/xxx/c", true},
+		{"a/*/c", "a/xxx/c", true},
+		{"a/*/*/c", "a/b/c", false},
+
+		{"**", "a", true},
+		{"**", "abc", true},
+		{"**", "abc 123", true},
+		{"**", "xxx/yyy", true},
+		{"**", "xxx/yyy/zzz", true},
+		{"**/**", "xxx", false},
+		{"**/**", "xxx/yyy", true},
+		{"**/**", "xxx/yyy/zzz", true},
+		{"**/**/**", "xxx/yyy/zzz", true},
+		{"**/**/c", "xxx/yyy/c", true},
+		{"**/**/c", "xxx/yyy/c/d", false},
+		{"a/**/**", "a/xxx/yyy", true},
+		{"a/**/c", "a/xxx/c", true},
+		{"a/**/c", "a/xxx/yyy/c", true},
+		{"a/**/c", "a/xxx/y y/zzz/c", true},
+
+		{"a/**/c", "a/c", false},
+		{"a/**c", "a/c", true},
+
+		{"xxx/**.foo", "xxx/aaa.foo", true},
+		{"xxx/**.foo", "xxx/yyy/zzz/.foo", true},
+		{"xxx/**.foo", "xxx/yyy/zzz/bar.foo", true},
+	} {
+		f, err := match.New(test.pattern)
+		if err != nil {
+			t.Errorf(`match.New("%v")`, test.pattern)
+			continue
+		}
+		matched := f(test.path)
+		switch {
+		case matched && !test.expect:
+			t.Errorf(`Path "%v" matched against pattern "%v"`, test.path, test.pattern)
+		case !matched && test.expect:
+			t.Errorf(`Path "%v" did not match against pattern "%v"`, test.path, test.pattern)
+		}
+	}
+}
+
+func TestErrOnPlaceholder(t *testing.T) {
+	for _, pattern := range []string{"a/b••c", "a/b•c", "a/b/¿c"} {
+		_, err := match.New(pattern)
+		if err == nil {
+			t.Errorf(`match.New("%v") did not return an expected error`, pattern)
+			continue
+		}
+		if !strings.Contains(err.Error(), "Pattern must not contain") {
+			t.Errorf(`match.New("%v") returned unrecognised error: %v`, pattern, err)
+			continue
+		}
+	}
+}

tools/src/substr/substr.go

@@ -0,0 +1,52 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package substr
+
+import (
+	diff "github.com/sergi/go-diff/diffmatchpatch"
+)
+
+// Fix attempts to reconstruct substr by comparing it to body.
+// substr is a fuzzy substring of body.
+// Fix returns a new exact substring of body, by calculating a diff of the text.
+// If no match could be made, Fix() returns an empty string.
+func Fix(body, substr string) string {
+	dmp := diff.New()
+
+	diffs := dmp.DiffMain(body, substr, false)
+	if len(diffs) == 0 {
+		return ""
+	}
+
+	front := func() diff.Diff { return diffs[0] }
+	back := func() diff.Diff { return diffs[len(diffs)-1] }
+
+	start, end := 0, len(body)
+
+	// Trim edits that remove text from body start
+	for len(diffs) > 0 && front().Type == diff.DiffDelete {
+		start += len(front().Text)
+		diffs = diffs[1:]
+	}
+
+	// Trim edits that remove text from body end
+	for len(diffs) > 0 && back().Type == diff.DiffDelete {
+		end -= len(back().Text)
+		diffs = diffs[:len(diffs)-1]
+	}
+
+	// New substring is the span for the remainder of the edits
+	return body[start:end]
+}

tools/src/substr/substr_test.go

@@ -0,0 +1,275 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package substr
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestFixSubstr(t *testing.T) {
+	type test struct {
+		body   string
+		substr string
+		expect string
+	}
+
+	for _, test := range []test{
+		{
+			body:   "abc_def_ghi_jkl_mno",
+			substr: "def_XXX_jkl",
+			expect: "def_ghi_jkl",
+		},
+		{
+			body:   "abc\ndef\nghi\njkl\nmno",
+			substr: "def\nXXX\njkl",
+			expect: "def\nghi\njkl",
+		},
+		{
+			body:   "aaaaa12345ccccc",
+			substr: "1x345",
+			expect: "12345",
+		},
+		{
+			body:   "aaaaa12345ccccc",
+			substr: "12x45",
+			expect: "12345",
+		},
+		{
+			body:   "aaaaa12345ccccc",
+			substr: "123x5",
+			expect: "12345",
+		},
+		{
+			body:   "aaaaaaaaaaaaa",
+			substr: "bbbbbbbbbbbbb",
+			expect: "", // cannot produce a sensible diff
+		}, { ///////////////////////////////////////////////////////////////////
+			body: `Return{
+  {
+    ScalarConstructor[not set]{42u}
+  }
+}
+`,
+			substr: `Return{
+  {
+    ScalarConstructor[not set]{42}
+  }
+}`,
+			expect: `Return{
+  {
+    ScalarConstructor[not set]{42u}
+  }
+}`,
+		}, { ///////////////////////////////////////////////////////////////////
+			body: `VariableDeclStatement{
+  Variable{
+    x_1
+    function
+    __u32
+  }
+}
+Assignment{
+  Identifier[not set]{x_1}
+  ScalarConstructor[not set]{42u}
+}
+Assignment{
+  Identifier[not set]{x_1}
+  ScalarConstructor[not set]{0u}
+}
+Return{}
+`,
+			substr: `Assignment{
+  Identifier[not set]{x_1}
+  ScalarConstructor[not set]{42}
+}
+Assignment{
+  Identifier[not set]{x_1}
+  ScalarConstructor[not set]{0}
+}`,
+			expect: `Assignment{
+  Identifier[not set]{x_1}
+  ScalarConstructor[not set]{42u}
+}
+Assignment{
+  Identifier[not set]{x_1}
+  ScalarConstructor[not set]{0u}
+}`,
+		}, { ///////////////////////////////////////////////////////////////////
+			body: `VariableDeclStatement{
+  Variable{
+    a
+    function
+    __bool
+    {
+      ScalarConstructor[not set]{true}
+    }
+  }
+}
+VariableDeclStatement{
+  Variable{
+    b
+    function
+    __bool
+    {
+      ScalarConstructor[not set]{false}
+    }
+  }
+}
+VariableDeclStatement{
+  Variable{
+    c
+    function
+    __i32
+    {
+      ScalarConstructor[not set]{-1}
+    }
+  }
+}
+VariableDeclStatement{
+  Variable{
+    d
+    function
+    __u32
+    {
+      ScalarConstructor[not set]{1u}
+    }
+  }
+}
+VariableDeclStatement{
+  Variable{
+    e
+    function
+    __f32
+    {
+      ScalarConstructor[not set]{1.500000}
+    }
+  }
+}
+`,
+			substr: `VariableDeclStatement{
+  Variable{
+    a
+    function
+    __bool
+    {
+      ScalarConstructor[not set]{true}
+    }
+  }
+}
+VariableDeclStatement{
+  Variable{
+    b
+    function
+    __bool
+    {
+      ScalarConstructor[not set]{false}
+    }
+  }
+}
+VariableDeclStatement{
+  Variable{
+    c
+    function
+    __i32
+    {
+      ScalarConstructor[not set]{-1}
+    }
+  }
+}
+VariableDeclStatement{
+  Variable{
+    d
+    function
+    __u32
+    {
+      ScalarConstructor[not set]{1}
+    }
+  }
+}
+VariableDeclStatement{
+  Variable{
+    e
+    function
+    __f32
+    {
+      ScalarConstructor[not set]{1.500000}
+    }
+  }
+}
+`,
+			expect: `VariableDeclStatement{
+  Variable{
+    a
+    function
+    __bool
+    {
+      ScalarConstructor[not set]{true}
+    }
+  }
+}
+VariableDeclStatement{
+  Variable{
+    b
+    function
+    __bool
+    {
+      ScalarConstructor[not set]{false}
+    }
+  }
+}
+VariableDeclStatement{
+  Variable{
+    c
+    function
+    __i32
+    {
+      ScalarConstructor[not set]{-1}
+    }
+  }
+}
+VariableDeclStatement{
+  Variable{
+    d
+    function
+    __u32
+    {
+      ScalarConstructor[not set]{1u}
+    }
+  }
+}
+VariableDeclStatement{
+  Variable{
+    e
+    function
+    __f32
+    {
+      ScalarConstructor[not set]{1.500000}
+    }
+  }
+}
+`,
+		},
+	} {
+		body := strings.ReplaceAll(test.body, "\n", "␤")
+		substr := strings.ReplaceAll(test.substr, "\n", "␤")
+		expect := strings.ReplaceAll(test.expect, "\n", `␤`)
+		got := strings.ReplaceAll(Fix(test.body, test.substr), "\n", "␤")
+		if got != expect {
+			t.Errorf("Test failure:\nbody:   '%v'\nsubstr: '%v'\nexpect: '%v'\ngot:    '%v'\n\n", body, substr, expect, got)
+		}
+
+	}
+}