tools/src/cmd/perfmon: Fixes / improvements

• Fix up various references of 'tint' to 'dawn' post-migration.
• Round the CPU speed to 100MHz. This appears to fluctuate on reboots
  for some machines.
• Fix git.Log() - the Count was trimming from the wrong end, leading
  to perfmon spamming the github repo with repeat results.
• Instead of using google-benchmark's `--benchmark_repetitions` flag
  for repeating benchmarks, calculate averages by re-running the
  benchmark executable. Use '--benchmark_enable_random_interleaving'
  to randomize the order in which benchmarks are run, which greatly
  helps reduce noise in the averaged results.
• If the host machine supports CPU temperature sensors, wait for
  thermals to stabilise before running the benchmarks. Further helps
  reduce result noise.
• Breakout of the historic benchmarking loop every 15 minutes to check
  for new Gerrit changes to benchmark.
• When idle, attempt to re-benchmark historic results that are 'spiky'
  to reduce noise in the graphs.
• Specify more CMake flags to avoid building non-benchmark executables.
• Update the default base change for historic results. Attempting to
  use the old hash, which was prior to the tint -> dawn  merge makes,
  the git log go back to T0 on the dawn branch. We don't want to
  benchmark nearly 3k changes.

Change-Id: I8e59c7838720eb8bd11f217e9bd3104ba1eb51c3
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/87642
Kokoro: Kokoro <noreply+kokoro@google.com>
Reviewed-by: Antonio Maiorano <amaiorano@google.com>
Commit-Queue: Ben Clayton <bclayton@google.com>

tools/src/cmd/perfmon/main.go

@@ -24,10 +24,12 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"math"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"reflect"
+	"regexp"
 	"sort"
 	"strings"
 	"time"
@@ -35,8 +37,6 @@ import (
 	"dawn.googlesource.com/dawn/tools/src/bench"
 	"dawn.googlesource.com/dawn/tools/src/git"
 	"github.com/andygrunwald/go-gerrit"
-	"github.com/go-git/go-git/v5/plumbing/transport"
-	"github.com/go-git/go-git/v5/plumbing/transport/http"
 	"github.com/shirou/gopsutil/cpu"
 )
 
@@ -73,11 +73,11 @@ func run(cfgPath string) error {
 		return err
 	}
 
-	tintDir, resultsDir, err := makeWorkingDirs(cfg)
+	dawnDir, resultsDir, err := makeWorkingDirs(cfg)
 	if err != nil {
 		return err
 	}
-	tintRepo, err := createOrOpenGitRepo(g, tintDir, cfg.Tint)
+	dawnRepo, err := createOrOpenGitRepo(g, dawnDir, cfg.Dawn)
 	if err != nil {
 		return err
 	}
@@ -96,25 +96,32 @@ func run(cfgPath string) error {
 		return fmt.Errorf("failed to obtain system info:\n  %v", err)
 	}
 
+	// Some machines report slightly different CPU clock speeds each reboot
+	// To work around this, quantize the reported speed to the nearest 100MHz
+	for i, s := range sysInfo {
+		sysInfo[i].Mhz = math.Round(s.Mhz/100) * 100
+	}
+
 	e := env{
 		cfg:         cfg,
 		git:         g,
 		system:      sysInfo,
 		systemID:    hash(sysInfo)[:8],
-		tintDir:     tintDir,
+		dawnDir:     dawnDir,
-		buildDir:    filepath.Join(tintDir, "out"),
+		buildDir:    filepath.Join(dawnDir, "out"),
 		resultsDir:  resultsDir,
-		tintRepo:    tintRepo,
+		dawnRepo:    dawnRepo,
 		resultsRepo: resultsRepo,
 		gerrit:      gerritClient,
 
 		benchmarkCache: map[git.Hash]*bench.Run{},
 	}
 
-	for true {
+	for {
 		didSomething, err := e.doSomeWork()
 		if err != nil {
 			log.Printf("ERROR: %v", err)
+			log.Printf("Pausing...")
 			time.Sleep(time.Minute * 10)
 			continue
 		}
@@ -123,20 +130,20 @@ func run(cfgPath string) error {
 			time.Sleep(time.Minute * 5)
 		}
 	}
-
-	return nil
 }
 
 // Config holds the root configuration options for the perfmon tool
 type Config struct {
 	WorkingDir           string
 	RootChange           git.Hash
-	Tint                 GitConfig
+	Dawn                 GitConfig
 	Results              GitConfig
 	Gerrit               GerritConfig
 	Timeouts             TimeoutsConfig
 	ExternalAccounts     []string
 	BenchmarkRepetitions int
+	BenchmarkMaxTemp     float32 // celsius
+	CPUTempSensorName    string  // Name of the sensor to use for CPU temp
 }
 
 // GitConfig holds the configuration options for accessing a git repo
@@ -168,7 +175,7 @@ type HistoricResults struct {
 	Commits []CommitResults
 }
 
-// CommitResults holds the results of a single tint commit
+// CommitResults holds the results of a single dawn commit
 type CommitResults struct {
 	Commit            string
 	CommitTime        time.Time
@@ -178,22 +185,30 @@ type CommitResults struct {
 
 // Benchmark holds the benchmark results for a single test
 type Benchmark struct {
-	Name   string
+	Name    string
-	Mean   float64
+	Time    float64
-	Median float64
+	Repeats int `json:",omitempty"`
-	Stddev float64
+}
+
+// AuthConfig holds the authentication options for accessing a git repo
+type AuthConfig struct {
+	Username string
+	Password string
 }
 
 // setDefaults assigns default values to unassigned fields of cfg
 func (cfg *Config) setDefaults() {
 	if cfg.RootChange.IsZero() {
-		cfg.RootChange, _ = git.ParseHash("be2362b18c792364c6bf5744db6d3837fbc655a0")
+		cfg.RootChange, _ = git.ParseHash("e72e42d9e0c851311512ca6da4d7b59f0bcc60d9")
 	}
-	cfg.Tint.setDefaults()
+	cfg.Dawn.setDefaults()
 	cfg.Results.setDefaults()
 	cfg.Timeouts.setDefaults()
-	if cfg.BenchmarkRepetitions < 2 {
+	if cfg.BenchmarkRepetitions < 1 {
-		cfg.BenchmarkRepetitions = 2
+		cfg.BenchmarkRepetitions = 1
+	}
+	if cfg.BenchmarkMaxTemp == 0 {
+		cfg.BenchmarkMaxTemp = 50
 	}
 }
 
@@ -217,30 +232,58 @@ func (cfg *TimeoutsConfig) setDefaults() {
 	}
 }
 
-// AuthConfig holds the authentication options for accessing a git repo
+// findCommitResults looks for a CommitResult with the given commit id,
-type AuthConfig struct {
+// returning a pointer to the CommitResult if found, otherwise nil
-	Username string
+func (h *HistoricResults) findCommitResults(commit string) *CommitResults {
-	Password string
+	for i, c := range h.Commits {
-}
+		if c.Commit == commit {
-
+			return &h.Commits[i]
-// authMethod returns a http.BasicAuth constructed from the AuthConfig
+		}
-func (cfg AuthConfig) authMethod() transport.AuthMethod {
-	if cfg.Username != "" || cfg.Password != "" {
-		return &http.BasicAuth{Username: cfg.Username, Password: cfg.Password}
 	}
 	return nil
 }
 
+// sorts all the benchmarks by commit date
+func (h *HistoricResults) sort() {
+	sort.Slice(h.Commits, func(i, j int) bool {
+		if h.Commits[i].CommitTime.Before(h.Commits[j].CommitTime) {
+			return true
+		}
+		if h.Commits[j].CommitTime.Before(h.Commits[i].CommitTime) {
+			return false
+		}
+		return h.Commits[i].CommitDescription < h.Commits[j].CommitDescription
+	})
+}
+
+// findBenchmark looks for a Benchmark with the given commit id,
+// returning a pointer to the Benchmark if found, otherwise nil
+func (r *CommitResults) findBenchmark(name string) *Benchmark {
+	for i, b := range r.Benchmarks {
+		if b.Name == name {
+			return &r.Benchmarks[i]
+		}
+	}
+	return nil
+}
+
+// sorts all the benchmarks by name
+func (r *CommitResults) sort() {
+	sort.Slice(r.Benchmarks, func(i, j int) bool {
+		return r.Benchmarks[i].Name < r.Benchmarks[j].Name
+	})
+}
+
 // env holds the perfmon main environment state
 type env struct {
 	cfg         Config
 	git         *git.Git
 	system      []cpu.InfoStat
 	systemID    string
-	tintDir     string
+	dawnDir     string
 	buildDir    string
 	resultsDir  string
-	tintRepo    *git.Repository
+	dawnRepo    *git.Repository
 	resultsRepo *git.Repository
 	gerrit      *gerrit.Client
 
@@ -272,12 +315,21 @@ func (e env) doSomeWork() (bool, error) {
 		}
 
 		if len(changesToBenchmark) > 0 {
-			log.Printf("benchmarking %v changes...", len(changesToBenchmark))
+			log.Printf("%v submitted changes to benchmark...", len(changesToBenchmark))
+
+			start := time.Now()
 			for i, c := range changesToBenchmark {
-				log.Printf("benchmarking %v/%v....", i+1, len(changesToBenchmark))
+				if time.Since(start) > time.Minute*15 {
+					// It's been a while since we scanned for review changes.
+					// Take a break from benchmarking submitted changes so we
+					// can scan for review changes to benchmark.
+					log.Printf("benchmarked %v changes", i)
+					return true, nil
+				}
 				benchRes, err := e.benchmarkTintChange(c)
 				if err != nil {
-					return true, err
+					log.Printf("benchmarking failed: %v", err)
+					benchRes = &bench.Run{}
 				}
 				commitRes, err := e.benchmarksToCommitResults(c, *benchRes)
 				if err != nil {
@@ -291,25 +343,51 @@ func (e env) doSomeWork() (bool, error) {
 			return true, nil
 		}
 	}
+
+	{
+		log.Println("scanning for benchmarks to refine...")
+		changeToBenchmark, err := e.changeToRefineBenchmarks()
+		if err != nil {
+			return true, err
+		}
+
+		if changeToBenchmark != nil {
+			log.Printf("re-benchmarking change '%v'", *changeToBenchmark)
+			benchRes, err := e.benchmarkTintChange(*changeToBenchmark)
+			if err != nil {
+				log.Printf("benchmarking failed: %v", err)
+				benchRes = &bench.Run{}
+			}
+			commitRes, err := e.benchmarksToCommitResults(*changeToBenchmark, *benchRes)
+			if err != nil {
+				return true, err
+			}
+			log.Printf("pushing results...")
+			if err := e.pushUpdatedResults(*commitRes); err != nil {
+				return true, err
+			}
+			return true, nil
+		}
+	}
 	return false, nil
 }
 
 // changesToBenchmark fetches the list of changes that do not currently have
 // benchmark results, which should be benchmarked.
 func (e env) changesToBenchmark() ([]git.Hash, error) {
-	log.Println("syncing tint repo...")
+	log.Println("syncing dawn repo...")
-	latest, err := e.tintRepo.Fetch(e.cfg.Tint.Branch, &git.FetchOptions{
+	latest, err := e.dawnRepo.Fetch(e.cfg.Dawn.Branch, &git.FetchOptions{
-		Credentials: e.cfg.Tint.Credentials,
+		Credentials: e.cfg.Dawn.Credentials,
 	})
 	if err != nil {
 		return nil, err
 	}
-	allChanges, err := e.tintRepo.Log(&git.LogOptions{
+	allChanges, err := e.dawnRepo.Log(&git.LogOptions{
 		From: e.cfg.RootChange.String(),
 		To:   latest.String(),
 	})
 	if err != nil {
-		return nil, fmt.Errorf("failed to obtain tint log:\n  %w", err)
+		return nil, fmt.Errorf("failed to obtain dawn log:\n  %w", err)
 	}
 	changesWithBenchmarks, err := e.changesWithBenchmarks()
 	if err != nil {
@@ -330,28 +408,103 @@ func (e env) changesToBenchmark() ([]git.Hash, error) {
 	return changesToBenchmark, nil
 }
 
-// benchmarkTintChange checks out the given commit, fetches the tint third party
+// changeToRefineBenchmarks scans for the most suitable historic commit to
-// dependencies, builds tint, then runs the benchmarks, returning the results.
+// re-benchmark and refine the results. Returns nil if there are no suitable
-func (e env) benchmarkTintChange(hash git.Hash) (*bench.Run, error) {
+// changes.
+func (e env) changeToRefineBenchmarks() (*git.Hash, error) {
+	log.Println("syncing results repo...")
+	if err := fetchAndCheckoutLatest(e.resultsRepo, e.cfg.Results); err != nil {
+		return nil, err
+	}
+
+	_, absPath, err := e.resultsFilePaths()
+	if err != nil {
+		return nil, err
+	}
+
+	results, err := e.loadHistoricResults(absPath)
+	if err != nil {
+		log.Println(fmt.Errorf("WARNING: failed to open result file '%v':\n  %w", absPath, err))
+		return nil, nil
+	}
+
+	if len(results.Commits) == 0 {
+		return nil, nil
+	}
+
+	type hashDelta struct {
+		hash  git.Hash
+		delta float64
+	}
+	hashDeltas := make([]hashDelta, 0, len(results.Commits))
+	for i, c := range results.Commits {
+		hash, err := git.ParseHash(c.Commit)
+		if err != nil {
+			return nil, err
+		}
+
+		prev := results.Commits[max(0, i-1)]
+		next := results.Commits[min(len(results.Commits)-1, i+1)]
+		delta, count := 0.0, 0
+		for _, b := range c.Benchmarks {
+			if b.Time == 0 {
+				continue
+			}
+			p, n := b.Time, b.Time
+			if pb := prev.findBenchmark(b.Name); pb != nil {
+				p = pb.Time
+			}
+			if nb := next.findBenchmark(b.Name); nb != nil {
+				n = nb.Time
+			}
+			avr := (p + n) / 2
+			confidence := math.Pow(2, float64(b.Repeats))
+			delta += math.Abs(avr-b.Time) / (b.Time * confidence)
+			count++
+		}
+		if count > 0 {
+			delta = delta / float64(count)
+			hashDeltas = append(hashDeltas, hashDelta{hash, delta})
+		}
+	}
+
+	sort.Slice(hashDeltas, func(i, j int) bool { return hashDeltas[i].delta > hashDeltas[j].delta })
+
+	return &hashDeltas[0].hash, nil
+}
+
+// benchmarkTintChangeIfNotCached first checks the results cache for existing
+// benchmark values for the given change, returning those cached values if hit.
+// If the cache does not contain results for the change, then
+// e.benchmarkTintChange() is called.
+func (e env) benchmarkTintChangeIfNotCached(hash git.Hash) (*bench.Run, error) {
 	if cached, ok := e.benchmarkCache[hash]; ok {
 		log.Printf("reusing cached benchmark results of '%v'...", hash)
 		return cached, nil
 	}
+	return e.benchmarkTintChange(hash)
+}
 
-	log.Printf("checking out tint at '%v'...", hash)
+// benchmarkTintChange checks out the given commit, fetches the dawn third party
-	if err := checkout(hash, e.tintRepo); err != nil {
+// dependencies, builds tint, then runs the benchmarks, returning the results.
+func (e env) benchmarkTintChange(hash git.Hash) (*bench.Run, error) {
+	log.Printf("checking out dawn at '%v'...", hash)
+	if err := checkout(hash, e.dawnRepo); err != nil {
 		return nil, err
 	}
-	log.Println("fetching tint dependencies...")
+	log.Println("fetching dawn dependencies...")
-	if err := e.fetchTintDeps(); err != nil {
+	if err := e.fetchDawnDeps(); err != nil {
 		return nil, err
 	}
 	log.Println("building tint...")
 	if err := e.buildTint(); err != nil {
 		return nil, err
 	}
+	if err := e.waitForTempsToSettle(); err != nil {
+		return nil, err
+	}
 	log.Println("benchmarking tint...")
-	run, err := e.benchmarkTint()
+	run, err := e.repeatedlyBenchmarkTint()
 	if err != nil {
 		return nil, err
 	}
@@ -363,45 +516,40 @@ func (e env) benchmarkTintChange(hash git.Hash) (*bench.Run, error) {
 // benchmarksToCommitResults converts the benchmarks in the provided bench.Run
 // to a CommitResults.
 func (e env) benchmarksToCommitResults(hash git.Hash, results bench.Run) (*CommitResults, error) {
-	commits, err := e.tintRepo.Log(&git.LogOptions{
+	commits, err := e.dawnRepo.Log(&git.LogOptions{
-		From:  hash.String(),
+		From: hash.String(),
-		Count: 1,
 	})
-	if err != nil || len(commits) != 1 {
+	if err != nil || len(commits) == 0 {
-		return nil, fmt.Errorf("failed to get commit object '%v' of tint repo:\n  %w", hash, err)
+		return nil, fmt.Errorf("failed to get commit object '%v' of dawn repo:\n  %w", hash, err)
+	}
+	commit := commits[len(commits)-1]
+	if commit.Hash != hash {
+		panic(fmt.Errorf("git.Repository.Log({From: %v}) returned:\n%+v", hash, commits))
 	}
-	commit := commits[0]
 
 	m := map[string]Benchmark{}
 	for _, b := range results.Benchmarks {
-		benchmark := m[b.Name]
+		m[b.Name] = Benchmark{
-		benchmark.Name = b.Name
+			Name: b.Name,
-		switch b.AggregateType {
+			Time: float64(b.Duration) / float64(time.Second),
-		case bench.Mean:
-			benchmark.Mean = float64(b.Duration) / float64(time.Second)
-		case bench.Median:
-			benchmark.Median = float64(b.Duration) / float64(time.Second)
-		case bench.Stddev:
-			benchmark.Stddev = float64(b.Duration) / float64(time.Second)
 		}
-		m[b.Name] = benchmark
 	}
 
-	sorted := make([]Benchmark, 0, len(m))
+	out := &CommitResults{
-	for _, b := range m {
-		sorted = append(sorted, b)
-	}
-	sort.Slice(sorted, func(i, j int) bool { return sorted[i].Name < sorted[i].Name })
-
-	return &CommitResults{
 		Commit:            commit.Hash.String(),
 		CommitDescription: commit.Subject,
 		CommitTime:        commit.Date,
-		Benchmarks:        sorted,
+		Benchmarks:        make([]Benchmark, 0, len(m)),
-	}, nil
+	}
+	for _, b := range m {
+		out.Benchmarks = append(out.Benchmarks, b)
+	}
+	out.sort()
+
+	return out, nil
 }
 
-// changesWithBenchmarks returns a set of tint changes that we already have
+// changesWithBenchmarks returns a set of dawn changes that we already have
 // benchmarks for.
 func (e env) changesWithBenchmarks() (map[git.Hash]struct{}, error) {
 	log.Println("syncing results repo...")
@@ -431,6 +579,9 @@ func (e env) changesWithBenchmarks() (map[git.Hash]struct{}, error) {
 	return m, nil
 }
 
+// pushUpdatedResults fetches and loads the latest benchmark results, adds or
+// merges the new results 'res' to the file, and then pushes the new results to
+// the server.
 func (e env) pushUpdatedResults(res CommitResults) error {
 	log.Println("syncing results repo...")
 	if err := fetchAndCheckoutLatest(e.resultsRepo, e.cfg.Results); err != nil {
@@ -448,10 +599,27 @@ func (e env) pushUpdatedResults(res CommitResults) error {
 		h = &HistoricResults{System: e.system}
 	}
 
-	h.Commits = append(h.Commits, res)
+	// Are there existing benchmark results for this commit?
+	if existing := h.findCommitResults(res.Commit); existing != nil {
+		// Yes: merge in the new results
+		for _, b := range res.Benchmarks {
+			if e := existing.findBenchmark(b.Name); e != nil {
+				// Benchmark found to merge. Add a weighted contribution to the benchmark value.
+				e.Time = (e.Time*float64(e.Repeats+1) + b.Time) / float64(e.Repeats+2)
+				e.Repeats++
+			} else {
+				// New benchmark? Just append.
+				existing.Benchmarks = append(existing.Benchmarks, b)
+			}
+		}
+		existing.sort()
+	} else {
+		// New benchmark results for this commit. Just append.
+		h.Commits = append(h.Commits, res)
+	}
 
 	// Sort the commits by timestamp
-	sort.Slice(h.Commits, func(i, j int) bool { return h.Commits[i].CommitTime.Before(h.Commits[j].CommitTime) })
+	h.sort()
 
 	// Write the new results to the file
 	f, err := os.Create(absPath)
@@ -527,20 +695,20 @@ System: %+v`, path, res.System, e.system)
 	return res, nil
 }
 
-// fetchTintDeps fetches the third party tint dependencies using gclient.
+// fetchDawnDeps fetches the third party dawn dependencies using gclient.
-func (e env) fetchTintDeps() error {
+func (e env) fetchDawnDeps() error {
-	gclientConfig := filepath.Join(e.tintDir, ".gclient")
+	gclientConfig := filepath.Join(e.dawnDir, ".gclient")
 	if _, err := os.Stat(gclientConfig); errors.Is(err, os.ErrNotExist) {
-		standalone := filepath.Join(e.tintDir, "scripts", "standalone.gclient")
+		standalone := filepath.Join(e.dawnDir, "scripts", "standalone.gclient")
 		if err := copyFile(gclientConfig, standalone); err != nil {
 			return fmt.Errorf("failed to copy '%v' to '%v':\n  %w", standalone, gclientConfig, err)
 		}
 	}
-	if _, err := call(tools.gclient, e.tintDir, e.cfg.Timeouts.Sync,
+	if _, err := call(tools.gclient, e.dawnDir, e.cfg.Timeouts.Sync,
 		"sync",
 		"--force",
 	); err != nil {
-		return fmt.Errorf("failed to fetch tint dependencies:\n  %w", err)
+		return fmt.Errorf("failed to fetch dawn dependencies:\n  %w", err)
 	}
 	return nil
 }
@@ -551,10 +719,15 @@ func (e env) buildTint() error {
 		return fmt.Errorf("failed to create build directory at '%v':\n  %w", e.buildDir, err)
 	}
 	if _, err := call(tools.cmake, e.buildDir, e.cfg.Timeouts.Build,
-		e.tintDir,
+		e.dawnDir,
 		"-GNinja",
 		"-DCMAKE_CXX_COMPILER_LAUNCHER=ccache",
 		"-DCMAKE_BUILD_TYPE=Release",
+		"-DCMAKE_BUILD_TESTS=0",
+		"-DCMAKE_BUILD_SAMPLES=0",
+		"-DTINT_BUILD_DOCS=0",
+		"-DTINT_BUILD_SAMPLES=0",
+		"-DTINT_BUILD_TESTS=0",
 		"-DTINT_BUILD_SPV_READER=1",
 		"-DTINT_BUILD_WGSL_READER=1",
 		"-DTINT_BUILD_GLSL_WRITER=1",
@@ -563,8 +736,9 @@ func (e env) buildTint() error {
 		"-DTINT_BUILD_SPV_WRITER=1",
 		"-DTINT_BUILD_WGSL_WRITER=1",
 		"-DTINT_BUILD_BENCHMARKS=1",
+		"-DDAWN_BUILD_SAMPLES=0",
 	); err != nil {
-		return errFailedToBuild{fmt.Errorf("failed to generate tint build config:\n  %w", err)}
+		return errFailedToBuild{fmt.Errorf("failed to generate dawn build config:\n  %w", err)}
 	}
 	if _, err := call(tools.ninja, e.buildDir, e.cfg.Timeouts.Build); err != nil {
 		return errFailedToBuild{err}
@@ -582,20 +756,71 @@ func (e errFailedToBuild) Error() string {
 	return fmt.Sprintf("failed to build: %v", e.reason)
 }
 
-// benchmarkTint runs the tint benchmarks, returning the results.
+// errFailedToBenchmark is the error returned by benchmarkTint() if the benchmark failed
+type errFailedToBenchmark struct {
+	// The reason
+	reason error
+}
+
+func (e errFailedToBenchmark) Error() string {
+	return fmt.Sprintf("failed to benchmark: %v", e.reason)
+}
+
+// benchmarkTint runs the tint benchmarks e.cfg.BenchmarkRepetitions times,
+// returning the averaged results.
+func (e env) repeatedlyBenchmarkTint() (*bench.Run, error) {
+	type durationAndCount struct {
+		duration time.Duration
+		count    int
+	}
+
+	var ctx *bench.Context
+	acc := map[string]durationAndCount{}
+	for i := 0; i < e.cfg.BenchmarkRepetitions; i++ {
+		if err := e.waitForTempsToSettle(); err != nil {
+			return nil, err
+		}
+		log.Printf("benchmark pass %v/%v...", (i + 1), e.cfg.BenchmarkRepetitions)
+		run, err := e.benchmarkTint()
+		if err != nil {
+			return nil, err
+		}
+		for _, b := range run.Benchmarks {
+			v := acc[b.Name]
+			v.duration += b.Duration
+			v.count++
+			acc[b.Name] = v
+		}
+		if ctx == nil {
+			ctx = run.Context
+		}
+	}
+
+	out := bench.Run{Context: ctx}
+	for name, dc := range acc {
+		out.Benchmarks = append(out.Benchmarks, bench.Benchmark{
+			Name:     name,
+			Duration: dc.duration / time.Duration(dc.count),
+		})
+	}
+
+	return &out, nil
+}
+
+// benchmarkTint runs the tint benchmarks once, returning the results.
 func (e env) benchmarkTint() (*bench.Run, error) {
 	exe := filepath.Join(e.buildDir, "tint-benchmark")
 	out, err := call(exe, e.buildDir, e.cfg.Timeouts.Benchmark,
 		"--benchmark_format=json",
-		fmt.Sprintf("--benchmark_repetitions=%v", e.cfg.BenchmarkRepetitions),
+		"--benchmark_enable_random_interleaving=true",
 	)
 	if err != nil {
-		return nil, fmt.Errorf("failed to benchmark tint:\n  %w", err)
+		return nil, errFailedToBenchmark{err}
 	}
 
 	results, err := bench.Parse(out)
 	if err != nil {
-		return nil, fmt.Errorf("failed to parse benchmark results:\n  %w", err)
+		return nil, errFailedToBenchmark{err}
 	}
 	return &results, nil
 }
@@ -605,7 +830,7 @@ func (e env) findGerritChangeToBenchmark() (*gerrit.ChangeInfo, error) {
 	log.Println("querying gerrit for changes...")
 	results, _, err := e.gerrit.Changes.QueryChanges(&gerrit.QueryChangeOptions{
 		QueryOptions: gerrit.QueryOptions{
-			Query: []string{"project:tint status:open+-age:3d"},
+			Query: []string{"project:dawn status:open+-age:3d"},
 			Limit: 100,
 		},
 		ChangeOptions: gerrit.ChangeOptions{
@@ -643,7 +868,7 @@ func (e env) findGerritChangeToBenchmark() (*gerrit.ChangeInfo, error) {
 				strings.HasSuffix(change.Labels["Presubmit-Ready"].Approved.Email, "@google.com")) {
 				permitted := false
 				for _, email := range e.cfg.ExternalAccounts {
-					if strings.ToLower(current.Commit.Committer.Email) == strings.ToLower(email) {
+					if strings.EqualFold(current.Commit.Committer.Email, email) {
 						permitted = true
 						break
 					}
@@ -704,8 +929,8 @@ func (e env) findGerritChangeToBenchmark() (*gerrit.ChangeInfo, error) {
 func (e env) benchmarkGerritChange(change gerrit.ChangeInfo) error {
 	current := change.Revisions[change.CurrentRevision]
 	log.Printf("fetching '%v'...", current.Ref)
-	currentHash, err := e.tintRepo.Fetch(current.Ref, &git.FetchOptions{
+	currentHash, err := e.dawnRepo.Fetch(current.Ref, &git.FetchOptions{
-		Credentials: e.cfg.Tint.Credentials,
+		Credentials: e.cfg.Dawn.Credentials,
 	})
 	if err != nil {
 		return err
@@ -730,39 +955,30 @@ func (e env) benchmarkGerritChange(change gerrit.ChangeInfo) error {
 
 	newRun, err := e.benchmarkTintChange(currentHash)
 	if err != nil {
-		var ftb errFailedToBuild
+		log.Printf("ERROR: %v", err)
-		if errors.As(err, &ftb) {
+		buildErr := errFailedToBuild{}
+		if errors.As(err, &buildErr) {
 			return postMsg("OWNER", fmt.Sprintf("patchset %v failed to build", current.Number))
 		}
+		benchErr := errFailedToBenchmark{}
+		if errors.As(err, &benchErr) {
+			return postMsg("OWNER", fmt.Sprintf("patchset %v failed to benchmark", current.Number))
+		}
 		return err
 	}
-	if _, err := e.tintRepo.Fetch(parent, &git.FetchOptions{
+	if _, err := e.dawnRepo.Fetch(parent, &git.FetchOptions{
-		Credentials: e.cfg.Tint.Credentials,
+		Credentials: e.cfg.Dawn.Credentials,
 	}); err != nil {
 		return err
 	}
-	parentRun, err := e.benchmarkTintChange(parentHash)
+	parentRun, err := e.benchmarkTintChangeIfNotCached(parentHash)
 	if err != nil {
 		return err
 	}
 
-	// filters the benchmark results to only the mean aggregate values
-	meanBenchmarkResults := func(in []bench.Benchmark) []bench.Benchmark {
-		out := make([]bench.Benchmark, 0, len(in))
-		for _, b := range in {
-			if b.AggregateType == bench.Mean {
-				out = append(out, b)
-			}
-		}
-		return out
-	}
-
-	newResults := meanBenchmarkResults(newRun.Benchmarks)
-	parentResults := meanBenchmarkResults(parentRun.Benchmarks)
-
 	const minDiff = time.Microsecond * 50 // Ignore time diffs less than this duration
 	const minRelDiff = 0.01               // Ignore absolute relative diffs between [1, 1+x]
-	diff := bench.Compare(parentResults, newResults, minDiff, minRelDiff)
+	diff := bench.Compare(parentRun.Benchmarks, newRun.Benchmarks, minDiff, minRelDiff)
 	diffFmt := bench.DiffFormat{
 		TestName:        true,
 		Delta:           true,
@@ -772,7 +988,7 @@ func (e env) benchmarkGerritChange(change gerrit.ChangeInfo) error {
 	}
 
 	msg := &strings.Builder{}
-	fmt.Fprintf(msg, "Tint perfmon analysis:\n")
+	fmt.Fprintf(msg, "Perfmon analysis:\n")
 	fmt.Fprintf(msg, "  \n")
 	fmt.Fprintf(msg, "  A: parent change (%v) -> B: patchset %v\n", parent[:7], current.Number)
 	fmt.Fprintf(msg, "  \n")
@@ -787,6 +1003,33 @@ func (e env) benchmarkGerritChange(change gerrit.ChangeInfo) error {
 	return postMsg(notify, msg.String())
 }
 
+// waitForTempsToSettle waits for the maximum temperature of all sensors to drop
+// below the threshold value specified by the config.
+func (e env) waitForTempsToSettle() error {
+	if e.cfg.CPUTempSensorName == "" {
+		time.Sleep(time.Second * 30)
+		return nil
+	}
+	const timeout = 5 * time.Minute
+	start := time.Now()
+	for {
+		temp, err := maxTemp(e.cfg.CPUTempSensorName)
+		if err != nil {
+			return fmt.Errorf("failed to obtain system temeratures: %v", err)
+		}
+		if temp < e.cfg.BenchmarkMaxTemp {
+			log.Printf("temperatures settled. current: %v°C", temp)
+			return nil
+		}
+		if time.Since(start) > timeout {
+			log.Printf("timeout waiting for temperatures to settle. current: %v°C", temp)
+			return nil
+		}
+		log.Printf("waiting for temperatures to settle. current: %v°C, max: %v°C", temp, e.cfg.BenchmarkMaxTemp)
+		time.Sleep(time.Second * 10)
+	}
+}
+
 // createOrOpenGitRepo creates a new local repo by cloning cfg.URL into
 // filepath, or opens the existing repo at filepath.
 func createOrOpenGitRepo(g *git.Git, filepath string, cfg GitConfig) (*git.Repository, error) {
@@ -818,8 +1061,8 @@ func loadConfig(path string) (Config, error) {
 	return cfg, nil
 }
 
-// makeWorkingDirs builds the tint repo and results repo directories.
+// makeWorkingDirs creates the dawn repo and results repo directories.
-func makeWorkingDirs(cfg Config) (tintDir, resultsDir string, err error) {
+func makeWorkingDirs(cfg Config) (dawnDir, resultsDir string, err error) {
 	wd, err := expandHomeDir(cfg.WorkingDir)
 	if err != nil {
 		return "", "", err
@@ -827,15 +1070,15 @@ func makeWorkingDirs(cfg Config) (tintDir, resultsDir string, err error) {
 	if err := os.MkdirAll(wd, 0777); err != nil {
 		return "", "", fmt.Errorf("failed to create working directory '%v':\n  %w", wd, err)
 	}
-	tintDir = filepath.Join(wd, "tint")
+	dawnDir = filepath.Join(wd, "dawn")
-	if err := os.MkdirAll(tintDir, 0777); err != nil {
+	if err := os.MkdirAll(dawnDir, 0777); err != nil {
-		return "", "", fmt.Errorf("failed to create working tint directory '%v':\n  %w", tintDir, err)
+		return "", "", fmt.Errorf("failed to create working dawn directory '%v':\n  %w", dawnDir, err)
 	}
 	resultsDir = filepath.Join(wd, "results")
 	if err := os.MkdirAll(resultsDir, 0777); err != nil {
 		return "", "", fmt.Errorf("failed to create working results directory '%v':\n  %w", resultsDir, err)
 	}
-	return tintDir, resultsDir, nil
+	return dawnDir, resultsDir, nil
 }
 
 // fetchAndCheckoutLatest calls fetch(cfg.Branch) followed by checkoutLatest().
@@ -882,6 +1125,7 @@ var tools struct {
 	gclient string
 	git     string
 	ninja   string
+	sensors string
 }
 
 // findTools looks for the file paths for executables used by this tool,
@@ -896,6 +1140,7 @@ func findTools() error {
 		{"gclient", &tools.gclient},
 		{"git", &tools.git},
 		{"ninja", &tools.ninja},
+		{"sensors", &tools.sensors},
 	} {
 		path, err := exec.LookPath(tool.name)
 		if err != nil {
@@ -922,6 +1167,27 @@ func copyFile(dstPath, srcPath string) error {
 	return err
 }
 
+// The regular expression to parse a temperature from 'sensors'
+var reTemp = regexp.MustCompile("([0-9]+.[0-9])°C")
+
+// maxTemp returns the maximum sensor temperature in celsius returned by 'sensors'
+func maxTemp(sensorName string) (float32, error) {
+	output, err := call(tools.sensors, "", time.Second*2, sensorName)
+	if err != nil {
+		return 0, err
+	}
+	var maxTemp float32
+	for _, match := range reTemp.FindAllStringSubmatch(output, -1) {
+		var temp float32
+		if _, err := fmt.Sscanf(match[1], "%f", &temp); err == nil {
+			if temp > maxTemp {
+				maxTemp = temp
+			}
+		}
+	}
+	return maxTemp, nil
+}
+
 // call invokes the executable exe in the current working directory wd, with
 // the provided arguments.
 // If the executable does not complete within the timeout duration, then an
@@ -945,3 +1211,17 @@ func hash(o interface{}) string {
 	hash.Write([]byte(str))
 	return hex.EncodeToString(hash.Sum(nil))[:8]
 }
+
+func min(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}
+
+func max(a, b int) int {
+	if a > b {
+		return a
+	}
+	return b
+}

tools/src/git/git.go

@@ -133,7 +133,7 @@ func (g Git) Clone(path, url string, opt *CloneOptions) (*Repository, error) {
 	if opt.Branch != "" {
 		args = append(args, "--branch", opt.Branch)
 	}
-	if _, err := r.run(opt.Timeout, args...); err != nil {
+	if _, err := r.run(nil, opt.Timeout, args...); err != nil {
 		return nil, err
 	}
 	return r, nil
@@ -166,10 +166,10 @@ func (r Repository) Fetch(ref string, opt *FetchOptions) (Hash, error) {
 	if opt.Remote == "" {
 		opt.Remote = "origin"
 	}
-	if _, err := r.run(opt.Timeout, "fetch", opt.Remote, ref); err != nil {
+	if _, err := r.run(nil, opt.Timeout, "fetch", opt.Remote, ref); err != nil {
 		return Hash{}, err
 	}
-	out, err := r.run(0, "rev-parse", "FETCH_HEAD")
+	out, err := r.run(nil, 0, "rev-parse", "FETCH_HEAD")
 	if err != nil {
 		return Hash{}, err
 	}
@@ -194,7 +194,7 @@ func (r Repository) Push(localRef, remoteRef string, opt *PushOptions) error {
 	if opt.Remote == "" {
 		opt.Remote = "origin"
 	}
-	url, err := r.run(opt.Timeout, "remote", "get-url", opt.Remote)
+	url, err := r.run(nil, opt.Timeout, "remote", "get-url", opt.Remote)
 	if err != nil {
 		return err
 	}
@@ -202,7 +202,7 @@ func (r Repository) Push(localRef, remoteRef string, opt *PushOptions) error {
 	if err != nil {
 		return err
 	}
-	if _, err := r.run(opt.Timeout, "push", url, localRef+":"+remoteRef); err != nil {
+	if _, err := r.run(nil, opt.Timeout, "push", url, localRef+":"+remoteRef); err != nil {
 		return err
 	}
 	return nil
@@ -221,7 +221,7 @@ func (r Repository) Add(path string, opt *AddOptions) error {
 	if opt == nil {
 		opt = &AddOptions{}
 	}
-	if _, err := r.run(opt.Timeout, "add", path); err != nil {
+	if _, err := r.run(nil, opt.Timeout, "add", path); err != nil {
 		return err
 	}
 	return nil
@@ -245,19 +245,27 @@ func (r Repository) Commit(msg string, opt *CommitOptions) (Hash, error) {
 	if opt == nil {
 		opt = &CommitOptions{}
 	}
+
 	args := []string{"commit"}
 	if opt.Amend {
 		args = append(args, "--amend")
 	} else {
 		args = append(args, "-m", msg)
 	}
+
+	var env []string
 	if opt.AuthorName != "" || opt.AuthorEmail != "" {
-		args = append(args, "--author", fmt.Sprintf("%v <%v>", opt.AuthorName, opt.AuthorEmail))
+		env = []string{
+			fmt.Sprintf("GIT_AUTHOR_NAME=%v", opt.AuthorName),
+			fmt.Sprintf("GIT_AUTHOR_EMAIL=%v", opt.AuthorEmail),
+			fmt.Sprintf("GIT_COMMITTER_NAME=%v", opt.AuthorName),
+			fmt.Sprintf("GIT_COMMITTER_EMAIL=%v", opt.AuthorEmail),
+		}
 	}
-	if _, err := r.run(opt.Timeout, args...); err != nil {
+	if _, err := r.run(env, opt.Timeout, "commit", "-m", msg); err != nil {
 		return Hash{}, err
 	}
-	out, err := r.run(0, "rev-parse", "HEAD")
+	out, err := r.run(nil, 0, "rev-parse", "HEAD")
 	if err != nil {
 		return Hash{}, err
 	}
@@ -275,7 +283,7 @@ func (r Repository) Checkout(ref string, opt *CheckoutOptions) error {
 	if opt == nil {
 		opt = &CheckoutOptions{}
 	}
-	if _, err := r.run(opt.Timeout, "checkout", ref); err != nil {
+	if _, err := r.run(nil, opt.Timeout, "checkout", ref); err != nil {
 		return err
 	}
 	return nil
@@ -287,8 +295,6 @@ type LogOptions struct {
 	From string
 	// The git reference to the newest commit in the range to query.
 	To string
-	// The maximum number of entries to return.
-	Count int
 	// Timeout for the operation
 	Timeout time.Duration
 }
@@ -317,10 +323,7 @@ func (r Repository) Log(opt *LogOptions) ([]CommitInfo, error) {
 		rng = opt.From + "^.." + rng
 	}
 	args = append(args, rng, "--pretty=format:ǁ%Hǀ%cIǀ%an <%ae>ǀ%sǀ%b")
-	if opt.Count != 0 {
+	out, err := r.run(nil, opt.Timeout, args...)
-		args = append(args, fmt.Sprintf("-%d", opt.Count))
-	}
-	out, err := r.run(opt.Timeout, args...)
 	if err != nil {
 		return nil, err
 	}
@@ -338,7 +341,7 @@ func (r Repository) Config(opt *ConfigOptions) (map[string]string, error) {
 	if opt == nil {
 		opt = &ConfigOptions{}
 	}
-	text, err := r.run(opt.Timeout, "config", "-l")
+	text, err := r.run(nil, opt.Timeout, "config", "-l")
 	if err != nil {
 		return nil, err
 	}
@@ -354,20 +357,11 @@ func (r Repository) Config(opt *ConfigOptions) (map[string]string, error) {
 	return out, nil
 }
 
-func (r Repository) run(timeout time.Duration, args ...string) (string, error) {
+func (r Repository) run(env []string, timeout time.Duration, args ...string) (string, error) {
-	return r.Git.run(r.Path, timeout, args...)
+	return r.Git.run(r.Path, env, timeout, args...)
 }
 
-func (r Repository) runAll(timeout time.Duration, args ...[]string) error {
+func (g Git) run(dir string, env []string, timeout time.Duration, args ...string) (string, error) {
-	for _, a := range args {
-		if _, err := r.run(timeout, a...); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (g Git) run(dir string, timeout time.Duration, args ...string) (string, error) {
 	if timeout == 0 {
 		timeout = DefaultTimeout
 	}
@@ -375,6 +369,12 @@ func (g Git) run(dir string, timeout time.Duration, args ...string) (string, err
 	defer cancel()
 	cmd := exec.CommandContext(ctx, g.exe, args...)
 	cmd.Dir = dir
+	if env != nil {
+		// Godocs for exec.Cmd.Env:
+		// "If Env contains duplicate environment keys, only the last value in
+		// the slice for each duplicate key is used.
+		cmd.Env = append(os.Environ(), env...)
+	}
 	if g.LogAllActions {
 		fmt.Printf("%v> %v %v\n", dir, g.exe, strings.Join(args, " "))
 	}