diff --git a/tools/src/bench/bench.go b/tools/src/bench/bench.go index 8bb1d99b49..a2836381fb 100644 --- a/tools/src/bench/bench.go +++ b/tools/src/bench/bench.go @@ -20,131 +20,398 @@ import ( "errors" "fmt" "regexp" + "sort" "strconv" "strings" "time" + "unicode/utf8" ) -// Test holds the results of a single benchmark test. -type Test struct { - Name string - NumTasks uint - NumThreads uint - Duration time.Duration - Iterations uint +// Run holds all the benchmark results for a run, along with the context +// information for the run. +type Run struct { + Benchmarks []Benchmark + Context *Context } -var testVarRE = regexp.MustCompile(`([\w])+:([0-9]+)`) - -func (t *Test) parseName() { - for _, match := range testVarRE.FindAllStringSubmatch(t.Name, -1) { - if len(match) != 3 { - continue - } - n, err := strconv.Atoi(match[2]) - if err != nil { - continue - } - switch match[1] { - case "threads": - t.NumThreads = uint(n) - case "tasks": - t.NumTasks = uint(n) - } - } +// Context provides information about the environment used to perform the +// benchmark. +type Context struct { + Date time.Time + HostName string + Executable string + NumCPUs int + MhzPerCPU int + CPUScalingEnabled bool + Caches []ContextCache + LoadAvg []float32 + LibraryBuildType string } -// Benchmark holds a set of benchmark test results. +// ContextCache holds information about one of the system caches. +type ContextCache struct { + Type string + Level int + Size int + NumSharing int +} + +// Benchmark holds the results of a single benchmark test. type Benchmark struct { - Tests []Test + Name string + Duration time.Duration + AggregateType AggregateType } +// AggregateType is an enumerator of benchmark aggregate types. +type AggregateType string + +// Enumerator values of AggregateType +const ( + NonAggregate AggregateType = "NonAggregate" + Mean AggregateType = "mean" + Median AggregateType = "median" + Stddev AggregateType = "stddev" +) + // Parse parses the benchmark results from the string s. // Parse will handle the json and 'console' formats. -func Parse(s string) (Benchmark, error) { - type Parser = func(s string) (Benchmark, error) +func Parse(s string) (Run, error) { + type Parser = func(s string) (Run, error) for _, parser := range []Parser{parseConsole, parseJSON} { - b, err := parser(s) + r, err := parser(s) switch err { case nil: - return b, nil + return r, nil case errWrongFormat: default: - return Benchmark{}, err + return Run{}, err } } - return Benchmark{}, errors.New("Unrecognised file format") + return Run{}, errors.New("Unrecognised file format") } var errWrongFormat = errors.New("Wrong format") var consoleLineRE = regexp.MustCompile(`([\w/:]+)\s+([0-9]+(?:.[0-9]+)?) ns\s+[0-9]+(?:.[0-9]+) ns\s+([0-9]+)`) -func parseConsole(s string) (Benchmark, error) { +func parseConsole(s string) (Run, error) { blocks := strings.Split(s, "------------------------------------------------------------------------------------------") if len(blocks) != 3 { - return Benchmark{}, errWrongFormat + return Run{}, errWrongFormat } lines := strings.Split(blocks[2], "\n") - b := Benchmark{ - Tests: make([]Test, 0, len(lines)), - } + b := make([]Benchmark, 0, len(lines)) + for _, line := range lines { if len(line) == 0 { continue } matches := consoleLineRE.FindStringSubmatch(line) if len(matches) != 4 { - return Benchmark{}, fmt.Errorf("Unable to parse the line:\n" + line) + return Run{}, fmt.Errorf("Unable to parse the line:\n" + line) } ns, err := strconv.ParseFloat(matches[2], 64) if err != nil { - return Benchmark{}, fmt.Errorf("Unable to parse the duration: " + matches[2]) - } - iterations, err := strconv.Atoi(matches[3]) - if err != nil { - return Benchmark{}, fmt.Errorf("Unable to parse the number of iterations: " + matches[3]) + return Run{}, fmt.Errorf("Unable to parse the duration: " + matches[2]) } - t := Test{ - Name: matches[1], - Duration: time.Nanosecond * time.Duration(ns), - Iterations: uint(iterations), - } - t.parseName() - b.Tests = append(b.Tests, t) + b = append(b, Benchmark{ + Name: trimAggregateSuffix(matches[1]), + Duration: time.Nanosecond * time.Duration(ns), + }) } - return b, nil + return Run{Benchmarks: b}, nil } -func parseJSON(s string) (Benchmark, error) { - type T struct { - Name string `json:"name"` - Iterations uint `json:"iterations"` - Time float64 `json:"real_time"` +func parseJSON(s string) (Run, error) { + type Data struct { + Context struct { + Date time.Time `json:"date"` + HostName string `json:"host_name"` + Executable string `json:"executable"` + NumCPUs int `json:"num_cpus"` + MhzPerCPU int `json:"mhz_per_cpu"` + CPUScalingEnabled bool `json:"cpu_scaling_enabled"` + LoadAvg []float32 `json:"load_avg"` + LibraryBuildType string `json:"library_build_type"` + Caches []struct { + Type string `json:"type"` + Level int `json:"level"` + Size int `json:"size"` + NumSharing int `json:"num_sharing"` + } `json:"caches"` + } `json:"context"` + Benchmarks []struct { + Name string `json:"name"` + Time float64 `json:"real_time"` + AggregateType AggregateType `json:"aggregate_name"` + } `json:"benchmarks"` } - type B struct { - Tests []T `json:"benchmarks"` - } - b := B{} + data := Data{} d := json.NewDecoder(strings.NewReader(s)) - if err := d.Decode(&b); err != nil { - return Benchmark{}, err + if err := d.Decode(&data); err != nil { + return Run{}, err } - out := Benchmark{ - Tests: make([]Test, len(b.Tests)), + out := Run{ + Benchmarks: make([]Benchmark, len(data.Benchmarks)), + Context: &Context{ + Date: data.Context.Date, + HostName: data.Context.HostName, + Executable: data.Context.Executable, + NumCPUs: data.Context.NumCPUs, + MhzPerCPU: data.Context.MhzPerCPU, + CPUScalingEnabled: data.Context.CPUScalingEnabled, + LoadAvg: data.Context.LoadAvg, + LibraryBuildType: data.Context.LibraryBuildType, + Caches: make([]ContextCache, len(data.Context.Caches)), + }, } - for i, test := range b.Tests { - t := Test{ - Name: test.Name, - Duration: time.Nanosecond * time.Duration(int64(test.Time)), - Iterations: test.Iterations, + for i, c := range data.Context.Caches { + out.Context.Caches[i] = ContextCache{ + Type: c.Type, + Level: c.Level, + Size: c.Size, + NumSharing: c.NumSharing, + } + } + for i, b := range data.Benchmarks { + out.Benchmarks[i] = Benchmark{ + Name: trimAggregateSuffix(b.Name), + Duration: time.Nanosecond * time.Duration(int64(b.Time)), + AggregateType: b.AggregateType, } - t.parseName() - out.Tests[i] = t } return out, nil } + +// Diff describes the difference between two benchmarks +type Diff struct { + TestName string + Delta time.Duration // Δ (A → B) + PercentChangeAB float64 // % (A → B) + PercentChangeBA float64 // % (A → B) + MultiplierChangeAB float64 // × (A → B) + MultiplierChangeBA float64 // × (A → B) + TimeA time.Duration // A + TimeB time.Duration // B +} + +// Diffs is a list of Diff +type Diffs []Diff + +// DiffFormat describes how a list of diffs should be formatted +type DiffFormat struct { + TestName bool + Delta bool + PercentChangeAB bool + PercentChangeBA bool + MultiplierChangeAB bool + MultiplierChangeBA bool + TimeA bool + TimeB bool +} + +func (diffs Diffs) Format(f DiffFormat) string { + if len(diffs) == 0 { + return "" + } + + type row []string + + header := row{} + if f.TestName { + header = append(header, "Test name") + } + if f.Delta { + header = append(header, "Δ (A → B)") + } + if f.PercentChangeAB { + header = append(header, "% (A → B)") + } + if f.PercentChangeBA { + header = append(header, "% (B → A)") + } + if f.MultiplierChangeAB { + header = append(header, "× (A → B)") + } + if f.MultiplierChangeBA { + header = append(header, "× (B → A)") + } + if f.TimeA { + header = append(header, "A") + } + if f.TimeB { + header = append(header, "B") + } + if len(header) == 0 { + return "" + } + + columns := []row{} + for _, d := range diffs { + r := make(row, 0, len(header)) + if f.TestName { + r = append(r, d.TestName) + } + if f.Delta { + r = append(r, fmt.Sprintf("%v", d.Delta)) + } + if f.PercentChangeAB { + r = append(r, fmt.Sprintf("%+2.1f%%", d.PercentChangeAB)) + } + if f.PercentChangeBA { + r = append(r, fmt.Sprintf("%+2.1f%%", d.PercentChangeBA)) + } + if f.MultiplierChangeAB { + r = append(r, fmt.Sprintf("%+.4f", d.MultiplierChangeAB)) + } + if f.MultiplierChangeBA { + r = append(r, fmt.Sprintf("%+.4f", d.MultiplierChangeBA)) + } + if f.TimeA { + r = append(r, fmt.Sprintf("%v", d.TimeA)) + } + if f.TimeB { + r = append(r, fmt.Sprintf("%v", d.TimeB)) + } + columns = append(columns, r) + } + + // measure + widths := make([]int, len(header)) + for i, h := range header { + widths[i] = utf8.RuneCountInString(h) + } + for _, row := range columns { + for i, cell := range row { + l := utf8.RuneCountInString(cell) + if widths[i] < l { + widths[i] = l + } + } + } + + pad := func(s string, i int) string { + if n := i - utf8.RuneCountInString(s); n > 0 { + return s + strings.Repeat(" ", n) + } + return s + } + + // Draw table + b := &strings.Builder{} + + horizontal_bar := func() { + for i := range header { + fmt.Fprintf(b, "+%v", strings.Repeat("-", 2+widths[i])) + } + fmt.Fprintln(b, "+") + } + + horizontal_bar() + + for i, h := range header { + fmt.Fprintf(b, "| %v ", pad(h, widths[i])) + } + fmt.Fprintln(b, "|") + + horizontal_bar() + + for _, row := range columns { + for i, cell := range row { + fmt.Fprintf(b, "| %v ", pad(cell, widths[i])) + } + fmt.Fprintln(b, "|") + } + + horizontal_bar() + + return b.String() +} + +// Compare returns a string describing differences in the two benchmarks +// Absolute benchmark differences less than minDiff are omitted +// Absolute relative differences between [1, 1+x] are omitted +func Compare(a, b []Benchmark, minDiff time.Duration, minRelDiff float64) Diffs { + type times struct { + a time.Duration + b time.Duration + } + byName := map[string]times{} + for _, test := range a { + byName[test.Name] = times{a: test.Duration} + } + for _, test := range b { + t := byName[test.Name] + t.b = test.Duration + byName[test.Name] = t + } + + type delta struct { + name string + times times + relDiff float64 + absRelDiff float64 + } + deltas := []delta{} + for name, times := range byName { + if times.a == 0 || times.b == 0 { + continue // Assuming test was missing from a or b + } + diff := times.b - times.a + absDiff := diff + if absDiff < 0 { + absDiff = -absDiff + } + if absDiff < minDiff { + continue + } + + relDiff := float64(times.b) / float64(times.a) + absRelDiff := relDiff + if absRelDiff < 1 { + absRelDiff = 1.0 / absRelDiff + } + if absRelDiff < (1.0 + minRelDiff) { + continue + } + + d := delta{ + name: name, + times: times, + relDiff: relDiff, + absRelDiff: absRelDiff, + } + deltas = append(deltas, d) + } + + sort.Slice(deltas, func(i, j int) bool { return deltas[j].relDiff < deltas[i].relDiff }) + + out := make(Diffs, len(deltas)) + + for i, delta := range deltas { + a2b := delta.times.b - delta.times.a + out[i] = Diff{ + TestName: delta.name, + Delta: a2b, + PercentChangeAB: 100 * float64(a2b) / float64(delta.times.a), + PercentChangeBA: 100 * float64(-a2b) / float64(delta.times.b), + MultiplierChangeAB: float64(delta.times.b) / float64(delta.times.a), + MultiplierChangeBA: float64(delta.times.a) / float64(delta.times.b), + TimeA: delta.times.a, + TimeB: delta.times.b, + } + } + return out +} + +func trimAggregateSuffix(name string) string { + name = strings.TrimSuffix(name, "_stddev") + name = strings.TrimSuffix(name, "_mean") + name = strings.TrimSuffix(name, "_median") + return name +} diff --git a/tools/src/bench/bench_test.go b/tools/src/bench/bench_test.go new file mode 100644 index 0000000000..1078f86284 --- /dev/null +++ b/tools/src/bench/bench_test.go @@ -0,0 +1,244 @@ +// Copyright 2022 The Tint Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bench_test + +import ( + "reflect" + "testing" + "time" + + "dawn.googlesource.com/tint/tools/src/bench" +) + +func TestParseJson(t *testing.T) { + json := ` + { + "context": { + "date": "2022-01-24T10:28:13+00:00", + "host_name": "hostname", + "executable": "./myexe", + "num_cpus": 16, + "mhz_per_cpu": 2400, + "cpu_scaling_enabled": false, + "caches": [ + { + "type": "Data", + "level": 1, + "size": 32768, + "num_sharing": 2 + }, + { + "type": "Instruction", + "level": 1, + "size": 32768, + "num_sharing": 2 + }, + { + "type": "Unified", + "level": 2, + "size": 262144, + "num_sharing": 2 + }, + { + "type": "Unified", + "level": 3, + "size": 16777216, + "num_sharing": 16 + } + ], + "load_avg": [2.60938,2.59863,2.55566], + "library_build_type": "release" + }, + "benchmarks": [ + { + "name": "MyBenchmark", + "family_index": 0, + "per_family_instance_index": 0, + "run_name": "MyBenchmark", + "run_type": "iteration", + "repetitions": 2, + "repetition_index": 0, + "threads": 1, + "iterations": 402, + "real_time": 1.6392272438353568e+06, + "cpu_time": 1.6387412935323382e+06, + "time_unit": "ns" + }, + { + "name": "MyBenchmark", + "family_index": 0, + "per_family_instance_index": 0, + "run_name": "MyBenchmark", + "run_type": "iteration", + "repetitions": 2, + "repetition_index": 1, + "threads": 1, + "iterations": 402, + "real_time": 1.7143936117703272e+06, + "cpu_time": 1.7124004975124374e+06, + "time_unit": "ns" + }, + { + "name": "MyBenchmark_mean", + "family_index": 0, + "per_family_instance_index": 0, + "run_name": "MyBenchmark", + "run_type": "aggregate", + "repetitions": 2, + "threads": 1, + "aggregate_name": "mean", + "iterations": 2, + "real_time": 1.6768104278028419e+06, + "cpu_time": 1.6755708955223879e+06, + "time_unit": "ns" + }, + { + "name": "MyBenchmark_median", + "family_index": 0, + "per_family_instance_index": 0, + "run_name": "MyBenchmark", + "run_type": "aggregate", + "repetitions": 2, + "threads": 1, + "aggregate_name": "median", + "iterations": 2, + "real_time": 1.6768104278028419e+06, + "cpu_time": 1.6755708955223879e+06, + "time_unit": "ns" + }, + { + "name": "MyBenchmark_stddev", + "family_index": 0, + "per_family_instance_index": 0, + "run_name": "MyBenchmark", + "run_type": "aggregate", + "repetitions": 2, + "threads": 1, + "aggregate_name": "stddev", + "iterations": 2, + "real_time": 5.3150648483981553e+04, + "cpu_time": 5.2084922631119407e+04, + "time_unit": "ns" + } + ] + } +` + got, err := bench.Parse(json) + if err != nil { + t.Errorf("bench.Parse() returned %v", err) + return + } + + expectedDate, err := time.Parse(time.RFC1123, "Mon, 24 Jan 2022 10:28:13 GMT") + if err != nil { + t.Errorf("time.Parse() returned %v", err) + return + } + + expect := bench.Run{ + Benchmarks: []bench.Benchmark{ + {Name: "MyBenchmark", Duration: time.Nanosecond * 1639227, AggregateType: ""}, + {Name: "MyBenchmark", Duration: time.Nanosecond * 1714393, AggregateType: ""}, + {Name: "MyBenchmark", Duration: time.Nanosecond * 1676810, AggregateType: "mean"}, + {Name: "MyBenchmark", Duration: time.Nanosecond * 1676810, AggregateType: "median"}, + {Name: "MyBenchmark", Duration: time.Nanosecond * 53150, AggregateType: "stddev"}, + }, + Context: &bench.Context{ + Date: expectedDate, + HostName: "hostname", + Executable: "./myexe", + NumCPUs: 16, + MhzPerCPU: 2400, CPUScalingEnabled: false, + Caches: []bench.ContextCache{ + {Type: "Data", Level: 1, Size: 32768, NumSharing: 2}, + {Type: "Instruction", Level: 1, Size: 32768, NumSharing: 2}, + {Type: "Unified", Level: 2, Size: 262144, NumSharing: 2}, + {Type: "Unified", Level: 3, Size: 16777216, NumSharing: 16}, + }, + LoadAvg: []float32{2.60938, 2.59863, 2.55566}, LibraryBuildType: "release"}, + } + + expectEqual(t, "bench.Parse().Benchmarks", got.Benchmarks, expect.Benchmarks) + expectEqual(t, "bench.Parse().Context", got.Benchmarks, expect.Benchmarks) +} + +func TestCompare(t *testing.T) { + a := []bench.Benchmark{ + {Name: "MyBenchmark1", Duration: time.Nanosecond * 1714393}, + {Name: "MyBenchmark0", Duration: time.Nanosecond * 1639227}, + {Name: "MyBenchmark3", Duration: time.Nanosecond * 1676810}, + {Name: "MyBenchmark4", Duration: time.Nanosecond * 53150}, + {Name: "MyBenchmark2", Duration: time.Nanosecond * 1676810}, + } + b := []bench.Benchmark{ + {Name: "MyBenchmark1", Duration: time.Nanosecond * 56747654}, + {Name: "MyBenchmark0", Duration: time.Nanosecond * 236246}, + {Name: "MyBenchmark2", Duration: time.Nanosecond * 675865}, + {Name: "MyBenchmark4", Duration: time.Nanosecond * 2352336}, + {Name: "MyBenchmark3", Duration: time.Nanosecond * 87657868}, + } + + minDiff := time.Millisecond * 2 + minRelDiff := 35.0 + + cmp := bench.Compare(a, b, minDiff, minRelDiff) + + expectEqual(t, "bench.Compare().Format", cmp.Format(bench.DiffFormat{}), "") + expectEqual(t, "bench.Compare().Format", "\n"+cmp.Format(bench.DiffFormat{TimeA: true}), ` +| A | +|-----------| +| 1.67681ms | +| 53.15µs | +`) + expectEqual(t, "bench.Compare().Format", "\n"+cmp.Format(bench.DiffFormat{TimeA: true, TimeB: true}), ` +| A | B | +|-----------+-------------| +| 1.67681ms | 87.657868ms | +| 53.15µs | 2.352336ms | +`) + expectEqual(t, "bench.Compare().Format", "\n"+cmp.Format(bench.DiffFormat{ + TestName: true, + Delta: true, + PercentChangeAB: true, + TimeA: true, + TimeB: true, + }), ` +| Test name | Δ (A → B) | % (A → B) | A | B | +|--------------+-------------+-----------+-----------+-------------| +| MyBenchmark3 | 85.981058ms | +5127.7% | 1.67681ms | 87.657868ms | +| MyBenchmark4 | 2.299186ms | +4325.8% | 53.15µs | 2.352336ms | +`) + expectEqual(t, "bench.Compare().Format", "\n"+cmp.Format(bench.DiffFormat{ + TestName: true, + Delta: true, + PercentChangeAB: true, + PercentChangeBA: true, + MultiplierChangeAB: true, + MultiplierChangeBA: true, + TimeA: true, + TimeB: true, + }), ` +| Test name | Δ (A → B) | % (A → B) | % (B → A) | × (A → B) | × (B → A) | A | B | +|--------------+-------------+-----------+-----------+-----------+-----------+-----------+-------------| +| MyBenchmark3 | 85.981058ms | +5127.7% | -98.1% | +52.2766 | +0.0191 | 1.67681ms | 87.657868ms | +| MyBenchmark4 | 2.299186ms | +4325.8% | -97.7% | +44.2584 | +0.0226 | 53.15µs | 2.352336ms | +`) +} + +func expectEqual(t *testing.T, desc string, got, expect interface{}) { + if !reflect.DeepEqual(got, expect) { + t.Errorf("%v was not expected:\nGot:\n%+v\nExpected:\n%+v", desc, got, expect) + } +} diff --git a/tools/src/cmd/benchdiff/main.go b/tools/src/cmd/benchdiff/main.go index 14ec4f3d6e..0f7f9492f0 100644 --- a/tools/src/cmd/benchdiff/main.go +++ b/tools/src/cmd/benchdiff/main.go @@ -23,9 +23,6 @@ import ( "io/ioutil" "os" "path/filepath" - "sort" - "strings" - "text/tabwriter" "time" "dawn.googlesource.com/tint/tools/src/bench" @@ -77,7 +74,18 @@ func run(pathA, pathB string) error { return err } - compare(benchA, benchB, fileName(pathA), fileName(pathB)) + cmp := bench.Compare(benchA.Benchmarks, benchB.Benchmarks, *minDiff, *minRelDiff) + diff := cmp.Format(bench.DiffFormat{ + TestName: true, + Delta: true, + PercentChangeAB: true, + TimeA: true, + TimeB: true, + }) + + fmt.Println("A:", pathA, " B:", pathB) + fmt.Println() + fmt.Println(diff) return nil } @@ -86,92 +94,3 @@ func fileName(path string) string { _, name := filepath.Split(path) return name } - -func compare(benchA, benchB bench.Benchmark, nameA, nameB string) { - type times struct { - a time.Duration - b time.Duration - } - byName := map[string]times{} - for _, test := range benchA.Tests { - byName[test.Name] = times{a: test.Duration} - } - for _, test := range benchB.Tests { - t := byName[test.Name] - t.b = test.Duration - byName[test.Name] = t - } - - type delta struct { - name string - times times - relDiff float64 - absRelDiff float64 - } - deltas := []delta{} - for name, times := range byName { - if times.a == 0 || times.b == 0 { - continue // Assuming test was missing from a or b - } - diff := times.b - times.a - absDiff := diff - if absDiff < 0 { - absDiff = -absDiff - } - if absDiff < *minDiff { - continue - } - - relDiff := float64(times.b) / float64(times.a) - absRelDiff := relDiff - if absRelDiff < 1 { - absRelDiff = 1.0 / absRelDiff - } - if absRelDiff < (1.0 + *minRelDiff) { - continue - } - - d := delta{ - name: name, - times: times, - relDiff: relDiff, - absRelDiff: absRelDiff, - } - deltas = append(deltas, d) - } - - sort.Slice(deltas, func(i, j int) bool { return deltas[j].relDiff < deltas[i].relDiff }) - - fmt.Println("A:", nameA) - fmt.Println("B:", nameB) - fmt.Println() - - buf := strings.Builder{} - { - w := tabwriter.NewWriter(&buf, 1, 1, 0, ' ', 0) - fmt.Fprintln(w, "Test name\t | Δ (A → B)\t | % (A → B)\t | % (B → A)\t | × (A → B)\t | × (B → A)\t | A \t | B") - fmt.Fprintln(w, "\t-+\t-+\t-+\t-+\t-+\t-+\t-+\t-") - for _, delta := range deltas { - a2b := delta.times.b - delta.times.a - fmt.Fprintf(w, "%v \t | %v \t | %+2.1f%% \t | %+2.1f%% \t | %+.4f \t | %+.4f \t | %v \t | %v \t|\n", - delta.name, - a2b, // Δ (A → B) - 100*float64(a2b)/float64(delta.times.a), // % (A → B) - 100*float64(-a2b)/float64(delta.times.b), // % (B → A) - float64(delta.times.b)/float64(delta.times.a), // × (A → B) - float64(delta.times.a)/float64(delta.times.b), // × (B → A) - delta.times.a, // A - delta.times.b, // B - ) - } - w.Flush() - } - - // Split the table by line so we can add in a header line - lines := strings.Split(buf.String(), "\n") - fmt.Println(lines[0]) - fmt.Println(strings.ReplaceAll(lines[1], " ", "-")) - for _, l := range lines[2:] { - fmt.Println(l) - } -} diff --git a/tools/src/cmd/perfmon/main.go b/tools/src/cmd/perfmon/main.go new file mode 100644 index 0000000000..21f1057105 --- /dev/null +++ b/tools/src/cmd/perfmon/main.go @@ -0,0 +1,904 @@ +// Copyright 2022 The Tint Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "log" + "os" + "os/exec" + "path/filepath" + "reflect" + "sort" + "strings" + "time" + + "dawn.googlesource.com/tint/tools/src/bench" + "dawn.googlesource.com/tint/tools/src/git" + "github.com/andygrunwald/go-gerrit" + "github.com/go-git/go-git/v5/plumbing/transport" + "github.com/go-git/go-git/v5/plumbing/transport/http" + "github.com/shirou/gopsutil/cpu" +) + +// main entry point +func main() { + var cfgPath string + flag.StringVar(&cfgPath, "c", "~/.config/perfmon/config.json", "the config file") + flag.Parse() + + if err := run(cfgPath); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +// run starts the perfmon tool with the given config path +func run(cfgPath string) error { + cfgPath, err := expandHomeDir(cfgPath) + if err != nil { + return err + } + + if err := findTools(); err != nil { + return err + } + + g, err := git.New(tools.git) + if err != nil { + return err + } + + cfg, err := loadConfig(cfgPath) + if err != nil { + return err + } + + tintDir, resultsDir, err := makeWorkingDirs(cfg) + if err != nil { + return err + } + tintRepo, err := createOrOpenGitRepo(g, tintDir, cfg.Tint) + if err != nil { + return err + } + resultsRepo, err := createOrOpenGitRepo(g, resultsDir, cfg.Results) + if err != nil { + return err + } + gerritClient, err := gerrit.NewClient(cfg.Gerrit.URL, nil) + if err != nil { + return err + } + gerritClient.Authentication.SetBasicAuth(cfg.Gerrit.Username, cfg.Gerrit.Password) + + sysInfo, err := cpu.Info() + if err != nil { + return fmt.Errorf("failed to obtain system info:\n %v", err) + } + + e := env{ + cfg: cfg, + git: g, + system: sysInfo, + systemID: hash(sysInfo)[:8], + tintDir: tintDir, + buildDir: filepath.Join(tintDir, "out"), + resultsDir: resultsDir, + tintRepo: tintRepo, + resultsRepo: resultsRepo, + gerrit: gerritClient, + + benchmarkCache: map[git.Hash]*bench.Run{}, + } + + for true { + { + log.Println("scanning for review changes to benchmark...") + change, err := e.findGerritChangeToBenchmark() + if err != nil { + return err + } + if change != nil { + if err := e.benchmarkGerritChange(*change); err != nil { + return err + } + continue + } + } + + { + log.Println("scanning for submitted changes to benchmark...") + changesToBenchmark, err := e.changesToBenchmark() + if err != nil { + return err + } + + if len(changesToBenchmark) > 0 { + log.Printf("benchmarking %v changes...\n", len(changesToBenchmark)) + for i, c := range changesToBenchmark { + log.Printf("benchmarking %v/%v....\n", i+1, len(changesToBenchmark)) + benchRes, err := e.benchmarkTintChange(c) + if err != nil { + return err + } + commitRes, err := e.benchmarksToCommitResults(c, *benchRes) + if err != nil { + return err + } + log.Printf("pushing results...\n") + if err := e.pushUpdatedResults(*commitRes); err != nil { + return err + } + } + continue + } + } + + log.Println("nothing to do. Sleeping...") + time.Sleep(time.Minute * 5) + } + + return nil +} + +// Config holds the root configuration options for the perfmon tool +type Config struct { + WorkingDir string + RootChange git.Hash + Tint GitConfig + Results GitConfig + Gerrit GerritConfig + Timeouts TimeoutsConfig + ExternalAccounts []string + BenchmarkRepetitions int +} + +// GitConfig holds the configuration options for accessing a git repo +type GitConfig struct { + URL string + Branch string + Auth git.Auth +} + +// GerritConfig holds the configuration options for accessing gerrit +type GerritConfig struct { + URL string + Username string + Email string + Password string +} + +// TimeoutsConfig holds the configuration options for timeouts +type TimeoutsConfig struct { + Sync time.Duration + Build time.Duration + Benchmark time.Duration +} + +// HistoricResults contains the full set of historic benchmark results for a +// given system +type HistoricResults struct { + System []cpu.InfoStat + Commits []CommitResults +} + +// CommitResults holds the results of a single tint commit +type CommitResults struct { + Commit string + CommitTime time.Time + CommitDescription string + Benchmarks []Benchmark +} + +// Benchmark holds the benchmark results for a single test +type Benchmark struct { + Name string + Mean float64 + Median float64 + Stddev float64 +} + +// setDefaults assigns default values to unassigned fields of cfg +func (cfg *Config) setDefaults() { + if cfg.RootChange.IsZero() { + cfg.RootChange, _ = git.ParseHash("be2362b18c792364c6bf5744db6d3837fbc655a0") + } + cfg.Tint.setDefaults() + cfg.Results.setDefaults() + cfg.Timeouts.setDefaults() + if cfg.BenchmarkRepetitions < 2 { + cfg.BenchmarkRepetitions = 2 + } +} + +// setDefaults assigns default values to unassigned fields of cfg +func (cfg *GitConfig) setDefaults() { + if cfg.Branch == "" { + cfg.Branch = "main" + } +} + +// setDefaults assigns default values to unassigned fields of cfg +func (cfg *TimeoutsConfig) setDefaults() { + if cfg.Sync == 0 { + cfg.Sync = time.Minute * 10 + } + if cfg.Build == 0 { + cfg.Build = time.Minute * 10 + } + if cfg.Benchmark == 0 { + cfg.Benchmark = time.Minute * 30 + } +} + +// AuthConfig holds the authentication options for accessing a git repo +type AuthConfig struct { + Username string + Password string +} + +// authMethod returns a http.BasicAuth constructed from the AuthConfig +func (cfg AuthConfig) authMethod() transport.AuthMethod { + if cfg.Username != "" || cfg.Password != "" { + return &http.BasicAuth{Username: cfg.Username, Password: cfg.Password} + } + return nil +} + +// env holds the perfmon main environment state +type env struct { + cfg Config + git *git.Git + system []cpu.InfoStat + systemID string + tintDir string + buildDir string + resultsDir string + tintRepo *git.Repository + resultsRepo *git.Repository + gerrit *gerrit.Client + + benchmarkCache map[git.Hash]*bench.Run +} + +// changesToBenchmark fetches the list of changes that do not currently have +// benchmark results, which should be benchmarked. +func (e env) changesToBenchmark() ([]git.Hash, error) { + log.Println("syncing tint repo...") + latest, err := e.tintRepo.Fetch(e.cfg.Tint.Branch, &git.FetchOptions{Auth: e.cfg.Tint.Auth}) + if err != nil { + return nil, err + } + allChanges, err := e.tintRepo.Log(&git.LogOptions{ + From: e.cfg.RootChange.String(), + To: latest.String(), + }) + if err != nil { + return nil, fmt.Errorf("failed to obtain tint log:\n %w", err) + } + changesWithBenchmarks, err := e.changesWithBenchmarks() + if err != nil { + return nil, fmt.Errorf("failed to gather changes with existing benchmarks:\n %w", err) + } + changesToBenchmark := make([]git.Hash, 0, len(allChanges)) + for _, c := range allChanges { + if _, exists := changesWithBenchmarks[c.Hash]; !exists { + changesToBenchmark = append(changesToBenchmark, c.Hash) + } + } + // Reverse the order of changesToBenchmark, so that the oldest comes first. + for i := len(changesToBenchmark)/2 - 1; i >= 0; i-- { + j := len(changesToBenchmark) - 1 - i + changesToBenchmark[i], changesToBenchmark[j] = changesToBenchmark[j], changesToBenchmark[i] + } + + return changesToBenchmark, nil +} + +// benchmarkTintChange checks out the given commit, fetches the tint third party +// dependencies, builds tint, then runs the benchmarks, returning the results. +func (e env) benchmarkTintChange(hash git.Hash) (*bench.Run, error) { + if cached, ok := e.benchmarkCache[hash]; ok { + log.Printf("reusing cached benchmark results of '%v'...\n", hash) + return cached, nil + } + + log.Printf("checking out tint at '%v'...\n", hash) + if err := checkout(hash, e.tintRepo); err != nil { + return nil, err + } + log.Println("fetching tint dependencies...") + if err := e.fetchTintDeps(); err != nil { + return nil, err + } + log.Println("building tint...") + if err := e.buildTint(); err != nil { + return nil, err + } + log.Println("benchmarking tint...") + run, err := e.benchmarkTint() + if err != nil { + return nil, err + } + + e.benchmarkCache[hash] = run + return run, nil +} + +func (e env) benchmarksToCommitResults(hash git.Hash, results bench.Run) (*CommitResults, error) { + commits, err := e.tintRepo.Log(&git.LogOptions{ + From: hash.String(), + Count: 1, + }) + if err != nil || len(commits) != 1 { + return nil, fmt.Errorf("failed to get commit object '%v' of tint repo:\n %w", hash, err) + } + commit := commits[0] + + m := map[string]Benchmark{} + for _, b := range results.Benchmarks { + benchmark := m[b.Name] + benchmark.Name = b.Name + switch b.AggregateType { + case bench.Mean: + benchmark.Mean = float64(b.Duration) / float64(time.Second) + case bench.Median: + benchmark.Median = float64(b.Duration) / float64(time.Second) + case bench.Stddev: + benchmark.Stddev = float64(b.Duration) / float64(time.Second) + } + m[b.Name] = benchmark + } + + sorted := make([]Benchmark, 0, len(m)) + for _, b := range m { + sorted = append(sorted, b) + } + sort.Slice(sorted, func(i, j int) bool { return sorted[i].Name < sorted[i].Name }) + + return &CommitResults{ + Commit: commit.Hash.String(), + CommitDescription: commit.Subject, + CommitTime: commit.Date, + Benchmarks: sorted, + }, nil +} + +// changesWithBenchmarks returns a set of tint changes that we already have +// benchmarks for. +func (e env) changesWithBenchmarks() (map[git.Hash]struct{}, error) { + log.Println("syncing results repo...") + if err := fetchAndCheckoutLatest(e.resultsRepo, e.cfg.Results); err != nil { + return nil, err + } + + _, absPath, err := e.resultsFilePaths() + if err != nil { + return nil, err + } + + results, err := e.loadHistoricResults(absPath) + if err != nil { + log.Println(fmt.Errorf("WARNING: failed to open result file '%v':\n %w", absPath, err)) + return nil, nil + } + + m := make(map[git.Hash]struct{}, len(results.Commits)) + for _, c := range results.Commits { + hash, err := git.ParseHash(c.Commit) + if err != nil { + return nil, err + } + m[hash] = struct{}{} + } + return m, nil +} + +func (e env) pushUpdatedResults(res CommitResults) error { + log.Println("syncing results repo...") + if err := fetchAndCheckoutLatest(e.resultsRepo, e.cfg.Results); err != nil { + return err + } + + relPath, absPath, err := e.resultsFilePaths() + if err != nil { + return err + } + + h, err := e.loadHistoricResults(absPath) + if err != nil { + log.Println(fmt.Errorf("failed to open result file '%v'. Creating new file\n %w", absPath, err)) + h = &HistoricResults{System: e.system} + } + + h.Commits = append(h.Commits, res) + + // Sort the commits by timestamp + sort.Slice(h.Commits, func(i, j int) bool { return h.Commits[i].CommitTime.Before(h.Commits[j].CommitTime) }) + + // Write the new results to the file + f, err := os.Create(absPath) + if err != nil { + return fmt.Errorf("failed to create updated results file '%v':\n %w", absPath, err) + } + defer f.Close() + + enc := json.NewEncoder(f) + enc.SetIndent("", " ") + if err := enc.Encode(h); err != nil { + return fmt.Errorf("failed to encode updated results file '%v':\n %w", absPath, err) + } + + // Stage the file + if err := e.resultsRepo.Add(relPath, nil); err != nil { + return fmt.Errorf("failed to stage updated results file '%v':\n %w", relPath, err) + } + + // Commit the change + msg := fmt.Sprintf("Add benchmark results for '%v'", res.Commit[:6]) + hash, err := e.resultsRepo.Commit(msg, &git.CommitOptions{ + AuthorName: "tint perfmon bot", + AuthorEmail: "tint-perfmon-bot@gmail.com", + }) + if err != nil { + return fmt.Errorf("failed to commit updated results file '%v':\n %w", absPath, err) + } + + // Push the change + log.Println("pushing updated results to results repo...") + if err := e.resultsRepo.Push(hash.String(), e.cfg.Results.Branch, &git.PushOptions{Auth: e.cfg.Results.Auth}); err != nil { + return fmt.Errorf("failed to push updated results file '%v':\n %w", absPath, err) + } + + return nil +} + +// resultsFilePaths returns the paths to the results.json file, holding the +// benchmarks for the given system. +func (e env) resultsFilePaths() (relPath string, absPath string, err error) { + dir := filepath.Join(e.resultsDir, "results") + if err = os.MkdirAll(dir, 0777); err != nil { + err = fmt.Errorf("failed to create results directory '%v':\n %w", dir, err) + return + } + relPath = filepath.Join("results", e.systemID+".json") + absPath = filepath.Join(dir, e.systemID+".json") + return +} + +// loadHistoricResults loads and returns the results.json file for the given +// system. +func (e env) loadHistoricResults(path string) (*HistoricResults, error) { + file, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("failed to open result file '%v':\n %w", path, err) + } + defer file.Close() + res := &HistoricResults{} + if err := json.NewDecoder(file).Decode(res); err != nil { + return nil, fmt.Errorf("failed to parse result file '%v':\n %w", path, err) + } + + if !reflect.DeepEqual(res.System, e.system) { + log.Printf(`WARNING: results file '%v' has different system information! +File: %+v +System: %+v +`, path, res.System, e.system) + } + + return res, nil +} + +// fetchTintDeps fetches the third party tint dependencies using gclient. +func (e env) fetchTintDeps() error { + gclientConfig := filepath.Join(e.tintDir, ".gclient") + if _, err := os.Stat(gclientConfig); errors.Is(err, os.ErrNotExist) { + standalone := filepath.Join(e.tintDir, "standalone.gclient") + if err := copyFile(gclientConfig, standalone); err != nil { + return fmt.Errorf("failed to copy '%v' to '%v':\n %w", standalone, gclientConfig, err) + } + } + if _, err := call(tools.gclient, e.tintDir, e.cfg.Timeouts.Sync, + "sync", + "--force", + ); err != nil { + return fmt.Errorf("failed to fetch tint dependencies:\n %w", err) + } + return nil +} + +// buildTint builds the tint benchmarks. +func (e env) buildTint() error { + if err := os.MkdirAll(e.buildDir, 0777); err != nil { + return fmt.Errorf("failed to create build directory at '%v':\n %w", e.buildDir, err) + } + if _, err := call(tools.cmake, e.buildDir, e.cfg.Timeouts.Build, + e.tintDir, + "-GNinja", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", + "-DCMAKE_BUILD_TYPE=Release", + "-DTINT_BUILD_SPV_READER=1", + "-DTINT_BUILD_WGSL_READER=1", + "-DTINT_BUILD_GLSL_WRITER=1", + "-DTINT_BUILD_HLSL_WRITER=1", + "-DTINT_BUILD_MSL_WRITER=1", + "-DTINT_BUILD_SPV_WRITER=1", + "-DTINT_BUILD_WGSL_WRITER=1", + "-DTINT_BUILD_BENCHMARKS=1", + ); err != nil { + return fmt.Errorf("failed to generate tint build config:\n %w", err) + } + if _, err := call(tools.ninja, e.buildDir, e.cfg.Timeouts.Build); err != nil { + return fmt.Errorf("failed to build tint:\n %w", err) + } + return nil +} + +// benchmarkTint runs the tint benchmarks, returning the results. +func (e env) benchmarkTint() (*bench.Run, error) { + exe := filepath.Join(e.buildDir, "tint-benchmark") + out, err := call(exe, e.buildDir, e.cfg.Timeouts.Benchmark, + "--benchmark_format=json", + fmt.Sprintf("--benchmark_repetitions=%v", e.cfg.BenchmarkRepetitions), + ) + if err != nil { + return nil, fmt.Errorf("failed to benchmark tint:\n %w", err) + } + + results, err := bench.Parse(out) + if err != nil { + return nil, fmt.Errorf("failed to parse benchmark results:\n %w", err) + } + return &results, nil +} + +// findGerritChangeToBenchmark queries gerrit for a change to benchmark. +func (e env) findGerritChangeToBenchmark() (*gerrit.ChangeInfo, error) { + log.Println("querying gerrit for changes...") + results, _, err := e.gerrit.Changes.QueryChanges(&gerrit.QueryChangeOptions{ + QueryOptions: gerrit.QueryOptions{ + Query: []string{"project:tint status:open+-age:3d"}, + Limit: 100, + }, + ChangeOptions: gerrit.ChangeOptions{ + AdditionalFields: []string{"CURRENT_REVISION", "CURRENT_COMMIT", "MESSAGES", "LABELS", "DETAILED_ACCOUNTS"}, + }, + }) + if err != nil { + return nil, fmt.Errorf("failed to get list of changes:\n %w", err) + } + + type candidate struct { + change gerrit.ChangeInfo + priority int + } + + candidates := make([]candidate, 0, len(*results)) + + for _, change := range *results { + kokoroApproved := change.Labels["Kokoro"].Approved.AccountID != 0 + codeReviewScore := change.Labels["Code-Review"].Value + codeReviewApproved := change.Labels["Code-Review"].Approved.AccountID != 0 + presubmitReady := change.Labels["Presubmit-Ready"].Approved.AccountID != 0 + verifiedScore := change.Labels["Verified"].Value + + current, ok := change.Revisions[change.CurrentRevision] + if !ok { + log.Printf("WARNING: couldn't find current revision for change '%s'", change.ChangeID) + } + + canBenchmark := func() bool { + // Is the change from a Googler, reviewed by a Googler or is from a allow-listed external developer? + if !(strings.HasSuffix(current.Commit.Committer.Email, "@google.com") || + strings.HasSuffix(change.Labels["Code-Review"].Approved.Email, "@google.com") || + strings.HasSuffix(change.Labels["Code-Review"].Recommended.Email, "@google.com") || + strings.HasSuffix(change.Labels["Presubmit-Ready"].Approved.Email, "@google.com")) { + permitted := false + for _, email := range e.cfg.ExternalAccounts { + if strings.ToLower(current.Commit.Committer.Email) == strings.ToLower(email) { + permitted = true + break + } + + } + if !permitted { + return false + } + } + + // Don't benchmark if the change has negative scores. + if codeReviewScore < 0 || verifiedScore < 0 { + return false + } + + // Has the latest patchset already been benchmarked? + for _, msg := range change.Messages { + if msg.RevisionNumber == current.Number && + msg.Author.Email == e.cfg.Gerrit.Email { + return false + } + } + + return true + }() + if !canBenchmark { + continue + } + + priority := 0 + if presubmitReady { + priority += 10 + } + priority += codeReviewScore + if codeReviewApproved { + priority += 2 + } + if kokoroApproved { + priority++ + } + + candidates = append(candidates, candidate{change, priority}) + } + + // Sort the candidates + sort.Slice(candidates, func(i, j int) bool { + return candidates[i].priority > candidates[j].priority + }) + + if len(candidates) > 0 { + log.Printf("%d gerrit changes to benchmark\n", len(candidates)) + return &candidates[0].change, nil + } + return nil, nil +} + +// benchmarks the gerrit change, posting the findings to the change +func (e env) benchmarkGerritChange(change gerrit.ChangeInfo) error { + current := change.Revisions[change.CurrentRevision] + log.Printf("fetching '%v'...\n", current.Ref) + currentHash, err := e.tintRepo.Fetch(current.Ref, &git.FetchOptions{Auth: e.cfg.Tint.Auth}) + if err != nil { + return err + } + parent := current.Commit.Parents[0].Commit + parentHash, err := git.ParseHash(parent) + if err != nil { + return fmt.Errorf("failed to parse parent hash '%v':\n %v", parent, err) + } + newRun, err := e.benchmarkTintChange(currentHash) + if err != nil { + return err + } + if _, err := e.tintRepo.Fetch(parent, &git.FetchOptions{Auth: e.cfg.Tint.Auth}); err != nil { + return err + } + parentRun, err := e.benchmarkTintChange(parentHash) + if err != nil { + return err + } + + // filters the benchmark results to only the mean aggregate values + meanBenchmarkResults := func(in []bench.Benchmark) []bench.Benchmark { + out := make([]bench.Benchmark, 0, len(in)) + for _, b := range in { + if b.AggregateType == bench.Mean { + out = append(out, b) + } + } + return out + } + + newResults := meanBenchmarkResults(newRun.Benchmarks) + parentResults := meanBenchmarkResults(parentRun.Benchmarks) + + const minDiff = time.Microsecond * 50 // Ignore time diffs less than this duration + const minRelDiff = 0.01 // Ignore absolute relative diffs between [1, 1+x] + diff := bench.Compare(parentResults, newResults, minDiff, minRelDiff) + diffFmt := bench.DiffFormat{ + TestName: true, + Delta: true, + PercentChangeAB: true, + TimeA: true, + TimeB: true, + } + + msg := &strings.Builder{} + fmt.Fprintf(msg, "Tint perfmon analysis:\n") + fmt.Fprintf(msg, " \n") + fmt.Fprintf(msg, " A: parent change (%v) -> B: patchset %v\n", parent[:7], current.Number) + fmt.Fprintf(msg, " \n") + for _, line := range strings.Split(diff.Format(diffFmt), "\n") { + fmt.Fprintf(msg, " %v\n", line) + } + + notify := "OWNER" + if len(diff) > 0 { + notify = "OWNER_REVIEWERS" + } + _, _, err = e.gerrit.Changes.SetReview(change.ChangeID, currentHash.String(), &gerrit.ReviewInput{ + Message: msg.String(), + Tag: "autogenerated:perfmon", + Notify: notify, + }) + if err != nil { + return fmt.Errorf("failed to post benchmark results to gerrit change:\n %v", err) + } + return nil +} + +// createOrOpenGitRepo creates a new local repo by cloning cfg.URL into +// filepath, or opens the existing repo at filepath. +func createOrOpenGitRepo(g *git.Git, filepath string, cfg GitConfig) (*git.Repository, error) { + repo, err := g.Open(filepath) + if errors.Is(err, git.ErrRepositoryDoesNotExist) { + log.Printf("cloning '%v' branch '%v' to '%v'...", cfg.URL, cfg.Branch, filepath) + repo, err = g.Clone(filepath, cfg.URL, &git.CloneOptions{ + Branch: cfg.Branch, + Auth: cfg.Auth, + }) + } + if err != nil { + return nil, fmt.Errorf("failed to open git repository '%v':\n %w", filepath, err) + } + return repo, err +} + +// loadConfig loads the perfmon config file. +func loadConfig(path string) (Config, error) { + f, err := os.Open(path) + if err != nil { + return Config{}, fmt.Errorf("failed to open config file at '%v':\n %w", path, err) + } + cfg := Config{} + if err := json.NewDecoder(f).Decode(&cfg); err != nil { + return Config{}, fmt.Errorf("failed to load config file at '%v':\n %w", path, err) + } + cfg.setDefaults() + return cfg, nil +} + +// makeWorkingDirs builds the tint repo and results repo directories. +func makeWorkingDirs(cfg Config) (tintDir, resultsDir string, err error) { + wd, err := expandHomeDir(cfg.WorkingDir) + if err != nil { + return "", "", err + } + if err := os.MkdirAll(wd, 0777); err != nil { + return "", "", fmt.Errorf("failed to create working directory '%v':\n %w", wd, err) + } + tintDir = filepath.Join(wd, "tint") + if err := os.MkdirAll(tintDir, 0777); err != nil { + return "", "", fmt.Errorf("failed to create working tint directory '%v':\n %w", tintDir, err) + } + resultsDir = filepath.Join(wd, "results") + if err := os.MkdirAll(resultsDir, 0777); err != nil { + return "", "", fmt.Errorf("failed to create working results directory '%v':\n %w", resultsDir, err) + } + return tintDir, resultsDir, nil +} + +// fetchAndCheckoutLatest calls fetch(cfg.Branch) followed by checkoutLatest(). +func fetchAndCheckoutLatest(repo *git.Repository, cfg GitConfig) error { + hash, err := repo.Fetch(cfg.Branch, &git.FetchOptions{Auth: cfg.Auth}) + if err != nil { + return err + } + if err := repo.Checkout(hash.String(), nil); err != nil { + return err + } + return checkout(hash, repo) +} + +// checkout checks out the change with the given hash. +// Note: call fetch() to ensure that this is the latest change on the +// branch. +func checkout(hash git.Hash, repo *git.Repository) error { + if err := repo.Checkout(hash.String(), nil); err != nil { + return fmt.Errorf("failed to checkout '%v':\n %w", hash, err) + } + return nil +} + +// expandHomeDir returns path with all occurrences of '~' replaced with the user +// home directory. +func expandHomeDir(path string) (string, error) { + if strings.ContainsRune(path, '~') { + home, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("failed to expand home dir:\n %w", err) + } + path = strings.ReplaceAll(path, "~", home) + } + return path, nil +} + +// tools holds the file paths to the executables used by this tool +var tools struct { + ccache string + cmake string + gclient string + git string + ninja string +} + +// findTools looks for the file paths for executables used by this tool, +// returning an error if any could not be found. +func findTools() error { + for _, tool := range []struct { + name string + path *string + }{ + {"ccache", &tools.ccache}, + {"cmake", &tools.cmake}, + {"gclient", &tools.gclient}, + {"git", &tools.git}, + {"ninja", &tools.ninja}, + } { + path, err := exec.LookPath(tool.name) + if err != nil { + return fmt.Errorf("failed to find path to '%v':\n %w", tool.name, err) + } + *tool.path = path + } + return nil +} + +// copyFile copies the file at srcPath to dstPath. +func copyFile(dstPath, srcPath string) error { + src, err := os.Open(srcPath) + if err != nil { + return fmt.Errorf("failed to open file '%v':\n %w", srcPath, err) + } + defer src.Close() + dst, err := os.Create(dstPath) + if err != nil { + return fmt.Errorf("failed to create file '%v':\n %w", dstPath, err) + } + defer dst.Close() + _, err = io.Copy(dst, src) + return err +} + +// call invokes the executable exe in the current working directory wd, with +// the provided arguments. +// If the executable does not complete within the timeout duration, then an +// error is returned. +func call(exe, wd string, timeout time.Duration, args ...string) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + cmd := exec.CommandContext(ctx, exe, args...) + cmd.Dir = wd + out, err := cmd.CombinedOutput() + if err != nil { + return string(out), fmt.Errorf("'%v %v' failed:\n %w\n%v", exe, args, err, string(out)) + } + return string(out), nil +} + +// hash returns a hash of the string representation of 'o'. +func hash(o interface{}) string { + str := fmt.Sprintf("%+v", o) + hash := sha256.New() + hash.Write([]byte(str)) + return hex.EncodeToString(hash.Sum(nil))[:8] +} diff --git a/tools/src/cmd/roll-release/main.go b/tools/src/cmd/roll-release/main.go index a727130510..cb6e261c2c 100644 --- a/tools/src/cmd/roll-release/main.go +++ b/tools/src/cmd/roll-release/main.go @@ -90,7 +90,7 @@ func run() error { // This tool uses a mix of 'go-git' and the command line git. // go-git has the benefit of keeping the git information entirely in-memory, - // but has issues working with chromiums tools and gerrit. + // but has issues working with chromium's tools and gerrit. // To create new release branches in Tint, we use 'go-git', so we need to // dig out the username and password. var auth transport.AuthMethod diff --git a/tools/src/git/git.go b/tools/src/git/git.go new file mode 100644 index 0000000000..f81f328457 --- /dev/null +++ b/tools/src/git/git.go @@ -0,0 +1,381 @@ +// Copyright 2022 The Tint Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package git + +import ( + "context" + "encoding/hex" + "errors" + "fmt" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "time" +) + +// Hash is a 20 byte, git object hash. +type Hash [20]byte + +func (h Hash) String() string { return hex.EncodeToString(h[:]) } + +// IsZero returns true if the hash h is all zeros +func (h Hash) IsZero() bool { + zero := Hash{} + return h == zero +} + +// ParseHash returns a Hash from a hexadecimal string. +func ParseHash(s string) (Hash, error) { + b, err := hex.DecodeString(s) + if err != nil { + return Hash{}, fmt.Errorf("failed to parse hash '%v':\n %w", s, err) + } + h := Hash{} + copy(h[:], b) + return h, nil +} + +// The timeout for git operations if no other timeout is specified +var DefaultTimeout = time.Minute + +// Git wraps the 'git' executable +type Git struct { + // Path to the git executable + exe string + // Debug flag to print all command to the `git` executable + LogAllActions bool +} + +// New returns a new Git instance +func New(exe string) (*Git, error) { + if _, err := os.Stat(exe); err != nil { + return nil, err + } + return &Git{exe: exe}, nil +} + +// Auth holds git authentication credentials +type Auth struct { + Username string + Password string +} + +// Empty return true if there's no username or password for authentication +func (a Auth) Empty() bool { + return a.Username == "" && a.Password == "" +} + +// ErrRepositoryDoesNotExist indicates that a repository does not exist +var ErrRepositoryDoesNotExist = errors.New("repository does not exist") + +// Open opens an existing git repo at path. If the repository does not exist at +// path then ErrRepositoryDoesNotExist is returned. +func (g Git) Open(path string) (*Repository, error) { + info, err := os.Stat(filepath.Join(path, ".git")) + if err != nil || !info.IsDir() { + return nil, ErrRepositoryDoesNotExist + } + return &Repository{g, path}, nil +} + +// Optional settings for Git.Clone +type CloneOptions struct { + // If specified then the given branch will be cloned instead of the default + Branch string + // Timeout for the operation + Timeout time.Duration + // Authentication for the clone + Auth Auth +} + +// Clone performs a clone of the repository at url to path. +func (g Git) Clone(path, url string, opt *CloneOptions) (*Repository, error) { + if err := os.MkdirAll(path, 0777); err != nil { + return nil, err + } + if opt == nil { + opt = &CloneOptions{} + } + url, err := opt.Auth.addToURL(url) + if err != nil { + return nil, err + } + r := &Repository{g, path} + args := []string{"clone", url, "."} + if opt.Branch != "" { + args = append(args, "--branch", opt.Branch) + } + if _, err := r.run(opt.Timeout, args...); err != nil { + return nil, err + } + return r, nil +} + +// Repository points to a git repository +type Repository struct { + // Path to the 'git' executable + Git Git + // Repo directory + Path string +} + +// Optional settings for Repository.Fetch +type FetchOptions struct { + // The remote name. Defaults to 'origin' + Remote string + // Timeout for the operation + Timeout time.Duration + // Git authentication for the remote + Auth Auth +} + +// Fetch performs a fetch of a reference from the remote, returning the Hash of +// the fetched reference. +func (r Repository) Fetch(ref string, opt *FetchOptions) (Hash, error) { + if opt == nil { + opt = &FetchOptions{} + } + if opt.Remote == "" { + opt.Remote = "origin" + } + if _, err := r.run(opt.Timeout, "fetch", opt.Remote, ref); err != nil { + return Hash{}, err + } + out, err := r.run(0, "rev-parse", "FETCH_HEAD") + if err != nil { + return Hash{}, err + } + return ParseHash(out) +} + +// Optional settings for Repository.Push +type PushOptions struct { + // The remote name. Defaults to 'origin' + Remote string + // Timeout for the operation + Timeout time.Duration + // Git authentication for the remote + Auth Auth +} + +// Push performs a push of the local reference to the remote reference. +func (r Repository) Push(localRef, remoteRef string, opt *PushOptions) error { + if opt == nil { + opt = &PushOptions{} + } + if opt.Remote == "" { + opt.Remote = "origin" + } + url, err := r.run(opt.Timeout, "remote", "get-url", opt.Remote) + if err != nil { + return err + } + url, err = opt.Auth.addToURL(url) + if err != nil { + return err + } + if _, err := r.run(opt.Timeout, "push", url, localRef+":"+remoteRef); err != nil { + return err + } + return nil +} + +// Optional settings for Repository.Add +type AddOptions struct { + // Timeout for the operation + Timeout time.Duration + // Git authentication for the remote + Auth Auth +} + +// Add stages the listed files +func (r Repository) Add(path string, opt *AddOptions) error { + if opt == nil { + opt = &AddOptions{} + } + if _, err := r.run(opt.Timeout, "add", path); err != nil { + return err + } + return nil +} + +// Optional settings for Repository.Commit +type CommitOptions struct { + // Timeout for the operation + Timeout time.Duration + // Author name + AuthorName string + // Author email address + AuthorEmail string +} + +// Commit commits the staged files with the given message, returning the hash of +// commit +func (r Repository) Commit(msg string, opt *CommitOptions) (Hash, error) { + if opt == nil { + opt = &CommitOptions{} + } + args := []string{"commit", "-m", msg} + if opt.AuthorName != "" || opt.AuthorEmail != "" { + args = append(args, "--author", fmt.Sprintf("%v <%v>", opt.AuthorName, opt.AuthorEmail)) + } + if _, err := r.run(opt.Timeout, args...); err != nil { + return Hash{}, err + } + out, err := r.run(0, "rev-parse", "HEAD") + if err != nil { + return Hash{}, err + } + return ParseHash(out) +} + +// Optional settings for Repository.Checkout +type CheckoutOptions struct { + // Timeout for the operation + Timeout time.Duration +} + +// Checkout performs a checkout of a reference. +func (r Repository) Checkout(ref string, opt *CheckoutOptions) error { + if opt == nil { + opt = &CheckoutOptions{} + } + if _, err := r.run(opt.Timeout, "checkout", ref); err != nil { + return err + } + return nil +} + +// Optional settings for Repository.Log +type LogOptions struct { + // The git reference to the oldest commit in the range to query. + From string + // The git reference to the newest commit in the range to query. + To string + // The maximum number of entries to return. + Count int + // Timeout for the operation + Timeout time.Duration +} + +// CommitInfo describes a single git commit +type CommitInfo struct { + Hash Hash + Date time.Time + Author string + Subject string + Description string +} + +// Log returns the list of commits between two references (inclusive). +// The first returned commit is the most recent. +func (r Repository) Log(opt *LogOptions) ([]CommitInfo, error) { + if opt == nil { + opt = &LogOptions{} + } + args := []string{"log"} + rng := "HEAD" + if opt.To != "" { + rng = opt.To + } + if opt.From != "" { + rng = opt.From + "^.." + rng + } + args = append(args, rng, "--pretty=format:ǁ%Hǀ%cIǀ%an <%ae>ǀ%sǀ%b") + if opt.Count != 0 { + args = append(args, fmt.Sprintf("-%d", opt.Count)) + } + out, err := r.run(opt.Timeout, args...) + if err != nil { + return nil, err + } + return parseLog(out) +} + +func (r Repository) run(timeout time.Duration, args ...string) (string, error) { + return r.Git.run(r.Path, timeout, args...) +} + +func (r Repository) runAll(timeout time.Duration, args ...[]string) error { + for _, a := range args { + if _, err := r.run(timeout, a...); err != nil { + return err + } + } + return nil +} + +func (g Git) run(dir string, timeout time.Duration, args ...string) (string, error) { + if timeout == 0 { + timeout = DefaultTimeout + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + cmd := exec.CommandContext(ctx, g.exe, args...) + cmd.Dir = dir + if g.LogAllActions { + fmt.Printf("%v> %v %v\n", dir, g.exe, strings.Join(args, " ")) + } + out, err := cmd.CombinedOutput() + if g.LogAllActions { + fmt.Println(string(out)) + } + if err != nil { + return string(out), fmt.Errorf("%v> %v %v failed:\n %w\n%v", + dir, g.exe, strings.Join(args, " "), err, string(out)) + } + return strings.TrimSpace(string(out)), nil +} + +func (a Auth) addToURL(u string) (string, error) { + if !a.Empty() { + modified, err := url.Parse(u) + if err != nil { + return "", fmt.Errorf("failed to parse url '%v': %v", u, err) + } + modified.User = url.UserPassword(a.Username, a.Password) + u = modified.String() + } + return u, nil +} + +func parseLog(str string) ([]CommitInfo, error) { + msgs := strings.Split(str, "ǁ") + cls := make([]CommitInfo, 0, len(msgs)) + for _, s := range msgs { + if parts := strings.Split(s, "ǀ"); len(parts) == 5 { + hash, err := ParseHash(parts[0]) + if err != nil { + return nil, err + } + date, err := time.Parse(time.RFC3339, parts[1]) + if err != nil { + return nil, err + } + cl := CommitInfo{ + Hash: hash, + Date: date, + Author: strings.TrimSpace(parts[2]), + Subject: strings.TrimSpace(parts[3]), + Description: strings.TrimSpace(parts[4]), + } + + cls = append(cls, cl) + } + } + return cls, nil +} diff --git a/tools/src/go.mod b/tools/src/go.mod index d3c85af2cc..cbb7157ea2 100644 --- a/tools/src/go.mod +++ b/tools/src/go.mod @@ -7,5 +7,8 @@ require ( github.com/fatih/color v1.10.0 github.com/go-git/go-git/v5 v5.4.2 github.com/sergi/go-diff v1.2.0 + github.com/shirou/gopsutil v3.21.11+incompatible + github.com/tklauser/go-sysconf v0.3.9 // indirect + github.com/yusufpapurcu/wmi v1.2.2 // indirect golang.org/x/net v0.0.0-20210614182718-04defd469f4e ) diff --git a/tools/src/go.sum b/tools/src/go.sum index 14f9171b67..9929b0c431 100644 --- a/tools/src/go.sum +++ b/tools/src/go.sum @@ -31,6 +31,8 @@ github.com/go-git/go-git-fixtures/v4 v4.2.1 h1:n9gGL1Ct/yIw+nfsfr8s4+sbhT+Ncu2Su github.com/go-git/go-git-fixtures/v4 v4.2.1/go.mod h1:K8zd3kDUAykwTdDCr+I0per6Y6vMiRR/nnVTBtavnB0= github.com/go-git/go-git/v5 v5.4.2 h1:BXyZu9t0VkbiHtqrsvdq39UDhGJTl1h55VW6CSC4aY4= github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti4ihgckDc= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -68,6 +70,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -75,8 +79,14 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo= +github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= +github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ= +github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI= github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= +github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= +github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b h1:7mWr3k41Qtv8XlltBkDkl8LoP3mpSgBW8BUoxtEdbXg= @@ -95,8 +105,9 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79 h1:RX8C8PRZc2hTIod4ds8ij+/4RQX3AqhYj3uOHmyaz4E= golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71 h1:ikCpsnYR+Ew0vu99XlDp55lGgDJdIMx3f4a18jfse/s= +golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=