benchmarks: Add a basic set of benchmarks

Add google benchmark to the DEPs.

Implement a basic set of benchmarks for each of the writers and the WGSL parser.

Add build rules for CMake. GN build rules TODO.

Add a simple go tool (ported from Marl) to diff two benchmarks. Less
noisy than the one provided by google benchmark.

Bug: tint:1378
Change-Id: I73cf92c5d9fd2d3bfac8f264864fd774afbd5d01
Reviewed-on: https://dawn-review.googlesource.com/c/tint/+/76840
Kokoro: Kokoro <noreply+kokoro@google.com>
Reviewed-by: Ryan Harrison <rharrison@chromium.org>
Commit-Queue: Ben Clayton <bclayton@chromium.org>

tools/benchdiff

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Copyright 2022 The Tint Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e # Fail on any error.
+
+if [ ! -x "$(which go)" ] ; then
+    echo "error: go needs to be on \$PATH to use $0"
+    exit 1
+fi
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )"
+ROOT_DIR="$( cd "${SCRIPT_DIR}/.." >/dev/null 2>&1 && pwd )"
+BINARY="${SCRIPT_DIR}/bin/benchdiff"
+
+# Rebuild the binary.
+# Note, go caches build artifacts, so this is quick for repeat calls
+pushd "${SCRIPT_DIR}/src/cmd/benchdiff" > /dev/null
+    go build -o "${BINARY}" main.go
+popd > /dev/null
+
+"${BINARY}" "$@"

tools/src/bench/bench.go

@@ -0,0 +1,150 @@
+// Copyright 2022 The Tint Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package bench provides types and methods for parsing Google benchmark results.
+package bench
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Test holds the results of a single benchmark test.
+type Test struct {
+	Name       string
+	NumTasks   uint
+	NumThreads uint
+	Duration   time.Duration
+	Iterations uint
+}
+
+var testVarRE = regexp.MustCompile(`([\w])+:([0-9]+)`)
+
+func (t *Test) parseName() {
+	for _, match := range testVarRE.FindAllStringSubmatch(t.Name, -1) {
+		if len(match) != 3 {
+			continue
+		}
+		n, err := strconv.Atoi(match[2])
+		if err != nil {
+			continue
+		}
+		switch match[1] {
+		case "threads":
+			t.NumThreads = uint(n)
+		case "tasks":
+			t.NumTasks = uint(n)
+		}
+	}
+}
+
+// Benchmark holds a set of benchmark test results.
+type Benchmark struct {
+	Tests []Test
+}
+
+// Parse parses the benchmark results from the string s.
+// Parse will handle the json and 'console' formats.
+func Parse(s string) (Benchmark, error) {
+	type Parser = func(s string) (Benchmark, error)
+	for _, parser := range []Parser{parseConsole, parseJSON} {
+		b, err := parser(s)
+		switch err {
+		case nil:
+			return b, nil
+		case errWrongFormat:
+		default:
+			return Benchmark{}, err
+		}
+	}
+
+	return Benchmark{}, errors.New("Unrecognised file format")
+}
+
+var errWrongFormat = errors.New("Wrong format")
+var consoleLineRE = regexp.MustCompile(`([\w/:]+)\s+([0-9]+(?:.[0-9]+)?) ns\s+[0-9]+(?:.[0-9]+) ns\s+([0-9]+)`)
+
+func parseConsole(s string) (Benchmark, error) {
+	blocks := strings.Split(s, "------------------------------------------------------------------------------------------")
+	if len(blocks) != 3 {
+		return Benchmark{}, errWrongFormat
+	}
+
+	lines := strings.Split(blocks[2], "\n")
+	b := Benchmark{
+		Tests: make([]Test, 0, len(lines)),
+	}
+	for _, line := range lines {
+		if len(line) == 0 {
+			continue
+		}
+		matches := consoleLineRE.FindStringSubmatch(line)
+		if len(matches) != 4 {
+			return Benchmark{}, fmt.Errorf("Unable to parse the line:\n" + line)
+		}
+		ns, err := strconv.ParseFloat(matches[2], 64)
+		if err != nil {
+			return Benchmark{}, fmt.Errorf("Unable to parse the duration: " + matches[2])
+		}
+		iterations, err := strconv.Atoi(matches[3])
+		if err != nil {
+			return Benchmark{}, fmt.Errorf("Unable to parse the number of iterations: " + matches[3])
+		}
+
+		t := Test{
+			Name:       matches[1],
+			Duration:   time.Nanosecond * time.Duration(ns),
+			Iterations: uint(iterations),
+		}
+		t.parseName()
+		b.Tests = append(b.Tests, t)
+	}
+	return b, nil
+}
+
+func parseJSON(s string) (Benchmark, error) {
+	type T struct {
+		Name       string  `json:"name"`
+		Iterations uint    `json:"iterations"`
+		Time       float64 `json:"real_time"`
+	}
+	type B struct {
+		Tests []T `json:"benchmarks"`
+	}
+	b := B{}
+	d := json.NewDecoder(strings.NewReader(s))
+	if err := d.Decode(&b); err != nil {
+		return Benchmark{}, err
+	}
+
+	out := Benchmark{
+		Tests: make([]Test, len(b.Tests)),
+	}
+	for i, test := range b.Tests {
+		t := Test{
+			Name:       test.Name,
+			Duration:   time.Nanosecond * time.Duration(int64(test.Time)),
+			Iterations: test.Iterations,
+		}
+		t.parseName()
+		out.Tests[i] = t
+	}
+
+	return out, nil
+}

tools/src/cmd/benchdiff/main.go

@@ -0,0 +1,177 @@
+// Copyright 2022 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// benchdiff is a tool that compares two Google benchmark results and displays
+// sorted performance differences.
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"text/tabwriter"
+	"time"
+
+	"dawn.googlesource.com/tint/tools/src/bench"
+)
+
+var (
+	minDiff    = flag.Duration("min-diff", time.Microsecond*10, "Filter away time diffs less than this duration")
+	minRelDiff = flag.Float64("min-rel-diff", 0.01, "Filter away absolute relative diffs between [1, 1+x]")
+)
+
+func main() {
+	flag.ErrHelp = errors.New("benchdiff is a tool to compare two benchmark results")
+	flag.Parse()
+	flag.Usage = func() {
+		fmt.Fprintln(os.Stderr, "benchdiff <benchmark-a> <benchmark-b>")
+		flag.PrintDefaults()
+	}
+
+	args := flag.Args()
+	if len(args) < 2 {
+		flag.Usage()
+		os.Exit(1)
+	}
+
+	pathA, pathB := args[0], args[1]
+
+	if err := run(pathA, pathB); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(-1)
+	}
+}
+
+func run(pathA, pathB string) error {
+	fileA, err := ioutil.ReadFile(pathA)
+	if err != nil {
+		return err
+	}
+	benchA, err := bench.Parse(string(fileA))
+	if err != nil {
+		return err
+	}
+
+	fileB, err := ioutil.ReadFile(pathB)
+	if err != nil {
+		return err
+	}
+	benchB, err := bench.Parse(string(fileB))
+	if err != nil {
+		return err
+	}
+
+	compare(benchA, benchB, fileName(pathA), fileName(pathB))
+
+	return nil
+}
+
+func fileName(path string) string {
+	_, name := filepath.Split(path)
+	return name
+}
+
+func compare(benchA, benchB bench.Benchmark, nameA, nameB string) {
+	type times struct {
+		a time.Duration
+		b time.Duration
+	}
+	byName := map[string]times{}
+	for _, test := range benchA.Tests {
+		byName[test.Name] = times{a: test.Duration}
+	}
+	for _, test := range benchB.Tests {
+		t := byName[test.Name]
+		t.b = test.Duration
+		byName[test.Name] = t
+	}
+
+	type delta struct {
+		name       string
+		times      times
+		relDiff    float64
+		absRelDiff float64
+	}
+	deltas := []delta{}
+	for name, times := range byName {
+		if times.a == 0 || times.b == 0 {
+			continue // Assuming test was missing from a or b
+		}
+		diff := times.b - times.a
+		absDiff := diff
+		if absDiff < 0 {
+			absDiff = -absDiff
+		}
+		if absDiff < *minDiff {
+			continue
+		}
+
+		relDiff := float64(times.b) / float64(times.a)
+		absRelDiff := relDiff
+		if absRelDiff < 1 {
+			absRelDiff = 1.0 / absRelDiff
+		}
+		if absRelDiff < (1.0 + *minRelDiff) {
+			continue
+		}
+
+		d := delta{
+			name:       name,
+			times:      times,
+			relDiff:    relDiff,
+			absRelDiff: absRelDiff,
+		}
+		deltas = append(deltas, d)
+	}
+
+	sort.Slice(deltas, func(i, j int) bool { return deltas[j].relDiff < deltas[i].relDiff })
+
+	fmt.Println("A:", nameA)
+	fmt.Println("B:", nameB)
+	fmt.Println()
+
+	buf := strings.Builder{}
+	{
+		w := tabwriter.NewWriter(&buf, 1, 1, 0, ' ', 0)
+		fmt.Fprintln(w, "Test name\t | Δ (A → B)\t | % (A → B)\t | % (B → A)\t | × (A → B)\t | × (B → A)\t | A \t | B")
+		fmt.Fprintln(w, "\t-+\t-+\t-+\t-+\t-+\t-+\t-+\t-")
+		for _, delta := range deltas {
+			a2b := delta.times.b - delta.times.a
+			fmt.Fprintf(w, "%v \t | %v \t | %+2.1f%% \t | %+2.1f%% \t | %+.4f \t | %+.4f \t | %v \t | %v \t|\n",
+				delta.name,
+				a2b, // Δ (A → B)
+				100*float64(a2b)/float64(delta.times.a),       // % (A → B)
+				100*float64(-a2b)/float64(delta.times.b),      // % (B → A)
+				float64(delta.times.b)/float64(delta.times.a), // × (A → B)
+				float64(delta.times.a)/float64(delta.times.b), // × (B → A)
+				delta.times.a, // A
+				delta.times.b, // B
+			)
+		}
+		w.Flush()
+	}
+
+	// Split the table by line so we can add in a header line
+	lines := strings.Split(buf.String(), "\n")
+	fmt.Println(lines[0])
+	fmt.Println(strings.ReplaceAll(lines[1], " ", "-"))
+	for _, l := range lines[2:] {
+		fmt.Println(l)
+	}
+}