tools: Add perfmon
A tool to continually automatically generate performance metrics for tint CLs. perfmon monitors gerrit changes, benchmarks them and posts results to the gerrit change. Commit changes are also benchmarked, and results are automatically posted to: https://tint-perfmon-bot.github.io/tint-perf Bug: tint:1383 Change-Id: I3470b170046e1d9af456f5e3a1d6ff76c305898a Reviewed-on: https://dawn-review.googlesource.com/c/tint/+/77940 Kokoro: Kokoro <noreply+kokoro@google.com> Reviewed-by: Ryan Harrison <rharrison@chromium.org> Reviewed-by: Antonio Maiorano <amaiorano@google.com> Commit-Queue: Ben Clayton <bclayton@google.com>
This commit is contained in:
Ben Clayton
Tint LUCI CQ
parent
3cdb8e3c3e
commit
c126bc95df
|
|
@ -20,131 +20,398 @@ import (
|
|||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// Test holds the results of a single benchmark test.
|
||||
type Test struct {
|
||||
Name string
|
||||
NumTasks uint
|
||||
NumThreads uint
|
||||
Duration time.Duration
|
||||
Iterations uint
|
||||
// Run holds all the benchmark results for a run, along with the context
|
||||
// information for the run.
|
||||
type Run struct {
|
||||
Benchmarks []Benchmark
|
||||
Context *Context
|
||||
}
|
||||
|
||||
var testVarRE = regexp.MustCompile(`([\w])+:([0-9]+)`)
|
||||
|
||||
func (t *Test) parseName() {
|
||||
for _, match := range testVarRE.FindAllStringSubmatch(t.Name, -1) {
|
||||
if len(match) != 3 {
|
||||
continue
|
||||
}
|
||||
n, err := strconv.Atoi(match[2])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
switch match[1] {
|
||||
case "threads":
|
||||
t.NumThreads = uint(n)
|
||||
case "tasks":
|
||||
t.NumTasks = uint(n)
|
||||
}
|
||||
}
|
||||
// Context provides information about the environment used to perform the
|
||||
// benchmark.
|
||||
type Context struct {
|
||||
Date time.Time
|
||||
HostName string
|
||||
Executable string
|
||||
NumCPUs int
|
||||
MhzPerCPU int
|
||||
CPUScalingEnabled bool
|
||||
Caches []ContextCache
|
||||
LoadAvg []float32
|
||||
LibraryBuildType string
|
||||
}
|
||||
|
||||
// Benchmark holds a set of benchmark test results.
|
||||
// ContextCache holds information about one of the system caches.
|
||||
type ContextCache struct {
|
||||
Type string
|
||||
Level int
|
||||
Size int
|
||||
NumSharing int
|
||||
}
|
||||
|
||||
// Benchmark holds the results of a single benchmark test.
|
||||
type Benchmark struct {
|
||||
Tests []Test
|
||||
Name string
|
||||
Duration time.Duration
|
||||
AggregateType AggregateType
|
||||
}
|
||||
|
||||
// AggregateType is an enumerator of benchmark aggregate types.
|
||||
type AggregateType string
|
||||
|
||||
// Enumerator values of AggregateType
|
||||
const (
|
||||
NonAggregate AggregateType = "NonAggregate"
|
||||
Mean AggregateType = "mean"
|
||||
Median AggregateType = "median"
|
||||
Stddev AggregateType = "stddev"
|
||||
)
|
||||
|
||||
// Parse parses the benchmark results from the string s.
|
||||
// Parse will handle the json and 'console' formats.
|
||||
func Parse(s string) (Benchmark, error) {
|
||||
type Parser = func(s string) (Benchmark, error)
|
||||
func Parse(s string) (Run, error) {
|
||||
type Parser = func(s string) (Run, error)
|
||||
for _, parser := range []Parser{parseConsole, parseJSON} {
|
||||
b, err := parser(s)
|
||||
r, err := parser(s)
|
||||
switch err {
|
||||
case nil:
|
||||
return b, nil
|
||||
return r, nil
|
||||
case errWrongFormat:
|
||||
default:
|
||||
return Benchmark{}, err
|
||||
return Run{}, err
|
||||
}
|
||||
}
|
||||
|
||||
return Benchmark{}, errors.New("Unrecognised file format")
|
||||
return Run{}, errors.New("Unrecognised file format")
|
||||
}
|
||||
|
||||
var errWrongFormat = errors.New("Wrong format")
|
||||
var consoleLineRE = regexp.MustCompile(`([\w/:]+)\s+([0-9]+(?:.[0-9]+)?) ns\s+[0-9]+(?:.[0-9]+) ns\s+([0-9]+)`)
|
||||
|
||||
func parseConsole(s string) (Benchmark, error) {
|
||||
func parseConsole(s string) (Run, error) {
|
||||
blocks := strings.Split(s, "------------------------------------------------------------------------------------------")
|
||||
if len(blocks) != 3 {
|
||||
return Benchmark{}, errWrongFormat
|
||||
return Run{}, errWrongFormat
|
||||
}
|
||||
|
||||
lines := strings.Split(blocks[2], "\n")
|
||||
b := Benchmark{
|
||||
Tests: make([]Test, 0, len(lines)),
|
||||
}
|
||||
b := make([]Benchmark, 0, len(lines))
|
||||
|
||||
for _, line := range lines {
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
}
|
||||
matches := consoleLineRE.FindStringSubmatch(line)
|
||||
if len(matches) != 4 {
|
||||
return Benchmark{}, fmt.Errorf("Unable to parse the line:\n" + line)
|
||||
return Run{}, fmt.Errorf("Unable to parse the line:\n" + line)
|
||||
}
|
||||
ns, err := strconv.ParseFloat(matches[2], 64)
|
||||
if err != nil {
|
||||
return Benchmark{}, fmt.Errorf("Unable to parse the duration: " + matches[2])
|
||||
}
|
||||
iterations, err := strconv.Atoi(matches[3])
|
||||
if err != nil {
|
||||
return Benchmark{}, fmt.Errorf("Unable to parse the number of iterations: " + matches[3])
|
||||
return Run{}, fmt.Errorf("Unable to parse the duration: " + matches[2])
|
||||
}
|
||||
|
||||
t := Test{
|
||||
Name: matches[1],
|
||||
Duration: time.Nanosecond * time.Duration(ns),
|
||||
Iterations: uint(iterations),
|
||||
}
|
||||
t.parseName()
|
||||
b.Tests = append(b.Tests, t)
|
||||
b = append(b, Benchmark{
|
||||
Name: trimAggregateSuffix(matches[1]),
|
||||
Duration: time.Nanosecond * time.Duration(ns),
|
||||
})
|
||||
}
|
||||
return b, nil
|
||||
return Run{Benchmarks: b}, nil
|
||||
}
|
||||
|
||||
func parseJSON(s string) (Benchmark, error) {
|
||||
type T struct {
|
||||
Name string `json:"name"`
|
||||
Iterations uint `json:"iterations"`
|
||||
Time float64 `json:"real_time"`
|
||||
func parseJSON(s string) (Run, error) {
|
||||
type Data struct {
|
||||
Context struct {
|
||||
Date time.Time `json:"date"`
|
||||
HostName string `json:"host_name"`
|
||||
Executable string `json:"executable"`
|
||||
NumCPUs int `json:"num_cpus"`
|
||||
MhzPerCPU int `json:"mhz_per_cpu"`
|
||||
CPUScalingEnabled bool `json:"cpu_scaling_enabled"`
|
||||
LoadAvg []float32 `json:"load_avg"`
|
||||
LibraryBuildType string `json:"library_build_type"`
|
||||
Caches []struct {
|
||||
Type string `json:"type"`
|
||||
Level int `json:"level"`
|
||||
Size int `json:"size"`
|
||||
NumSharing int `json:"num_sharing"`
|
||||
} `json:"caches"`
|
||||
} `json:"context"`
|
||||
Benchmarks []struct {
|
||||
Name string `json:"name"`
|
||||
Time float64 `json:"real_time"`
|
||||
AggregateType AggregateType `json:"aggregate_name"`
|
||||
} `json:"benchmarks"`
|
||||
}
|
||||
type B struct {
|
||||
Tests []T `json:"benchmarks"`
|
||||
}
|
||||
b := B{}
|
||||
data := Data{}
|
||||
d := json.NewDecoder(strings.NewReader(s))
|
||||
if err := d.Decode(&b); err != nil {
|
||||
return Benchmark{}, err
|
||||
if err := d.Decode(&data); err != nil {
|
||||
return Run{}, err
|
||||
}
|
||||
|
||||
out := Benchmark{
|
||||
Tests: make([]Test, len(b.Tests)),
|
||||
out := Run{
|
||||
Benchmarks: make([]Benchmark, len(data.Benchmarks)),
|
||||
Context: &Context{
|
||||
Date: data.Context.Date,
|
||||
HostName: data.Context.HostName,
|
||||
Executable: data.Context.Executable,
|
||||
NumCPUs: data.Context.NumCPUs,
|
||||
MhzPerCPU: data.Context.MhzPerCPU,
|
||||
CPUScalingEnabled: data.Context.CPUScalingEnabled,
|
||||
LoadAvg: data.Context.LoadAvg,
|
||||
LibraryBuildType: data.Context.LibraryBuildType,
|
||||
Caches: make([]ContextCache, len(data.Context.Caches)),
|
||||
},
|
||||
}
|
||||
for i, test := range b.Tests {
|
||||
t := Test{
|
||||
Name: test.Name,
|
||||
Duration: time.Nanosecond * time.Duration(int64(test.Time)),
|
||||
Iterations: test.Iterations,
|
||||
for i, c := range data.Context.Caches {
|
||||
out.Context.Caches[i] = ContextCache{
|
||||
Type: c.Type,
|
||||
Level: c.Level,
|
||||
Size: c.Size,
|
||||
NumSharing: c.NumSharing,
|
||||
}
|
||||
}
|
||||
for i, b := range data.Benchmarks {
|
||||
out.Benchmarks[i] = Benchmark{
|
||||
Name: trimAggregateSuffix(b.Name),
|
||||
Duration: time.Nanosecond * time.Duration(int64(b.Time)),
|
||||
AggregateType: b.AggregateType,
|
||||
}
|
||||
t.parseName()
|
||||
out.Tests[i] = t
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Diff describes the difference between two benchmarks
|
||||
type Diff struct {
|
||||
TestName string
|
||||
Delta time.Duration // Δ (A → B)
|
||||
PercentChangeAB float64 // % (A → B)
|
||||
PercentChangeBA float64 // % (A → B)
|
||||
MultiplierChangeAB float64 // × (A → B)
|
||||
MultiplierChangeBA float64 // × (A → B)
|
||||
TimeA time.Duration // A
|
||||
TimeB time.Duration // B
|
||||
}
|
||||
|
||||
// Diffs is a list of Diff
|
||||
type Diffs []Diff
|
||||
|
||||
// DiffFormat describes how a list of diffs should be formatted
|
||||
type DiffFormat struct {
|
||||
TestName bool
|
||||
Delta bool
|
||||
PercentChangeAB bool
|
||||
PercentChangeBA bool
|
||||
MultiplierChangeAB bool
|
||||
MultiplierChangeBA bool
|
||||
TimeA bool
|
||||
TimeB bool
|
||||
}
|
||||
|
||||
func (diffs Diffs) Format(f DiffFormat) string {
|
||||
if len(diffs) == 0 {
|
||||
return "<no changes>"
|
||||
}
|
||||
|
||||
type row []string
|
||||
|
||||
header := row{}
|
||||
if f.TestName {
|
||||
header = append(header, "Test name")
|
||||
}
|
||||
if f.Delta {
|
||||
header = append(header, "Δ (A → B)")
|
||||
}
|
||||
if f.PercentChangeAB {
|
||||
header = append(header, "% (A → B)")
|
||||
}
|
||||
if f.PercentChangeBA {
|
||||
header = append(header, "% (B → A)")
|
||||
}
|
||||
if f.MultiplierChangeAB {
|
||||
header = append(header, "× (A → B)")
|
||||
}
|
||||
if f.MultiplierChangeBA {
|
||||
header = append(header, "× (B → A)")
|
||||
}
|
||||
if f.TimeA {
|
||||
header = append(header, "A")
|
||||
}
|
||||
if f.TimeB {
|
||||
header = append(header, "B")
|
||||
}
|
||||
if len(header) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
columns := []row{}
|
||||
for _, d := range diffs {
|
||||
r := make(row, 0, len(header))
|
||||
if f.TestName {
|
||||
r = append(r, d.TestName)
|
||||
}
|
||||
if f.Delta {
|
||||
r = append(r, fmt.Sprintf("%v", d.Delta))
|
||||
}
|
||||
if f.PercentChangeAB {
|
||||
r = append(r, fmt.Sprintf("%+2.1f%%", d.PercentChangeAB))
|
||||
}
|
||||
if f.PercentChangeBA {
|
||||
r = append(r, fmt.Sprintf("%+2.1f%%", d.PercentChangeBA))
|
||||
}
|
||||
if f.MultiplierChangeAB {
|
||||
r = append(r, fmt.Sprintf("%+.4f", d.MultiplierChangeAB))
|
||||
}
|
||||
if f.MultiplierChangeBA {
|
||||
r = append(r, fmt.Sprintf("%+.4f", d.MultiplierChangeBA))
|
||||
}
|
||||
if f.TimeA {
|
||||
r = append(r, fmt.Sprintf("%v", d.TimeA))
|
||||
}
|
||||
if f.TimeB {
|
||||
r = append(r, fmt.Sprintf("%v", d.TimeB))
|
||||
}
|
||||
columns = append(columns, r)
|
||||
}
|
||||
|
||||
// measure
|
||||
widths := make([]int, len(header))
|
||||
for i, h := range header {
|
||||
widths[i] = utf8.RuneCountInString(h)
|
||||
}
|
||||
for _, row := range columns {
|
||||
for i, cell := range row {
|
||||
l := utf8.RuneCountInString(cell)
|
||||
if widths[i] < l {
|
||||
widths[i] = l
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pad := func(s string, i int) string {
|
||||
if n := i - utf8.RuneCountInString(s); n > 0 {
|
||||
return s + strings.Repeat(" ", n)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Draw table
|
||||
b := &strings.Builder{}
|
||||
|
||||
horizontal_bar := func() {
|
||||
for i := range header {
|
||||
fmt.Fprintf(b, "+%v", strings.Repeat("-", 2+widths[i]))
|
||||
}
|
||||
fmt.Fprintln(b, "+")
|
||||
}
|
||||
|
||||
horizontal_bar()
|
||||
|
||||
for i, h := range header {
|
||||
fmt.Fprintf(b, "| %v ", pad(h, widths[i]))
|
||||
}
|
||||
fmt.Fprintln(b, "|")
|
||||
|
||||
horizontal_bar()
|
||||
|
||||
for _, row := range columns {
|
||||
for i, cell := range row {
|
||||
fmt.Fprintf(b, "| %v ", pad(cell, widths[i]))
|
||||
}
|
||||
fmt.Fprintln(b, "|")
|
||||
}
|
||||
|
||||
horizontal_bar()
|
||||
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// Compare returns a string describing differences in the two benchmarks
|
||||
// Absolute benchmark differences less than minDiff are omitted
|
||||
// Absolute relative differences between [1, 1+x] are omitted
|
||||
func Compare(a, b []Benchmark, minDiff time.Duration, minRelDiff float64) Diffs {
|
||||
type times struct {
|
||||
a time.Duration
|
||||
b time.Duration
|
||||
}
|
||||
byName := map[string]times{}
|
||||
for _, test := range a {
|
||||
byName[test.Name] = times{a: test.Duration}
|
||||
}
|
||||
for _, test := range b {
|
||||
t := byName[test.Name]
|
||||
t.b = test.Duration
|
||||
byName[test.Name] = t
|
||||
}
|
||||
|
||||
type delta struct {
|
||||
name string
|
||||
times times
|
||||
relDiff float64
|
||||
absRelDiff float64
|
||||
}
|
||||
deltas := []delta{}
|
||||
for name, times := range byName {
|
||||
if times.a == 0 || times.b == 0 {
|
||||
continue // Assuming test was missing from a or b
|
||||
}
|
||||
diff := times.b - times.a
|
||||
absDiff := diff
|
||||
if absDiff < 0 {
|
||||
absDiff = -absDiff
|
||||
}
|
||||
if absDiff < minDiff {
|
||||
continue
|
||||
}
|
||||
|
||||
relDiff := float64(times.b) / float64(times.a)
|
||||
absRelDiff := relDiff
|
||||
if absRelDiff < 1 {
|
||||
absRelDiff = 1.0 / absRelDiff
|
||||
}
|
||||
if absRelDiff < (1.0 + minRelDiff) {
|
||||
continue
|
||||
}
|
||||
|
||||
d := delta{
|
||||
name: name,
|
||||
times: times,
|
||||
relDiff: relDiff,
|
||||
absRelDiff: absRelDiff,
|
||||
}
|
||||
deltas = append(deltas, d)
|
||||
}
|
||||
|
||||
sort.Slice(deltas, func(i, j int) bool { return deltas[j].relDiff < deltas[i].relDiff })
|
||||
|
||||
out := make(Diffs, len(deltas))
|
||||
|
||||
for i, delta := range deltas {
|
||||
a2b := delta.times.b - delta.times.a
|
||||
out[i] = Diff{
|
||||
TestName: delta.name,
|
||||
Delta: a2b,
|
||||
PercentChangeAB: 100 * float64(a2b) / float64(delta.times.a),
|
||||
PercentChangeBA: 100 * float64(-a2b) / float64(delta.times.b),
|
||||
MultiplierChangeAB: float64(delta.times.b) / float64(delta.times.a),
|
||||
MultiplierChangeBA: float64(delta.times.a) / float64(delta.times.b),
|
||||
TimeA: delta.times.a,
|
||||
TimeB: delta.times.b,
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func trimAggregateSuffix(name string) string {
|
||||
name = strings.TrimSuffix(name, "_stddev")
|
||||
name = strings.TrimSuffix(name, "_mean")
|
||||
name = strings.TrimSuffix(name, "_median")
|
||||
return name
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,244 @@
|
|||
// Copyright 2022 The Tint Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// https://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package bench_test
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"dawn.googlesource.com/tint/tools/src/bench"
|
||||
)
|
||||
|
||||
func TestParseJson(t *testing.T) {
|
||||
json := `
|
||||
{
|
||||
"context": {
|
||||
"date": "2022-01-24T10:28:13+00:00",
|
||||
"host_name": "hostname",
|
||||
"executable": "./myexe",
|
||||
"num_cpus": 16,
|
||||
"mhz_per_cpu": 2400,
|
||||
"cpu_scaling_enabled": false,
|
||||
"caches": [
|
||||
{
|
||||
"type": "Data",
|
||||
"level": 1,
|
||||
"size": 32768,
|
||||
"num_sharing": 2
|
||||
},
|
||||
{
|
||||
"type": "Instruction",
|
||||
"level": 1,
|
||||
"size": 32768,
|
||||
"num_sharing": 2
|
||||
},
|
||||
{
|
||||
"type": "Unified",
|
||||
"level": 2,
|
||||
"size": 262144,
|
||||
"num_sharing": 2
|
||||
},
|
||||
{
|
||||
"type": "Unified",
|
||||
"level": 3,
|
||||
"size": 16777216,
|
||||
"num_sharing": 16
|
||||
}
|
||||
],
|
||||
"load_avg": [2.60938,2.59863,2.55566],
|
||||
"library_build_type": "release"
|
||||
},
|
||||
"benchmarks": [
|
||||
{
|
||||
"name": "MyBenchmark",
|
||||
"family_index": 0,
|
||||
"per_family_instance_index": 0,
|
||||
"run_name": "MyBenchmark",
|
||||
"run_type": "iteration",
|
||||
"repetitions": 2,
|
||||
"repetition_index": 0,
|
||||
"threads": 1,
|
||||
"iterations": 402,
|
||||
"real_time": 1.6392272438353568e+06,
|
||||
"cpu_time": 1.6387412935323382e+06,
|
||||
"time_unit": "ns"
|
||||
},
|
||||
{
|
||||
"name": "MyBenchmark",
|
||||
"family_index": 0,
|
||||
"per_family_instance_index": 0,
|
||||
"run_name": "MyBenchmark",
|
||||
"run_type": "iteration",
|
||||
"repetitions": 2,
|
||||
"repetition_index": 1,
|
||||
"threads": 1,
|
||||
"iterations": 402,
|
||||
"real_time": 1.7143936117703272e+06,
|
||||
"cpu_time": 1.7124004975124374e+06,
|
||||
"time_unit": "ns"
|
||||
},
|
||||
{
|
||||
"name": "MyBenchmark_mean",
|
||||
"family_index": 0,
|
||||
"per_family_instance_index": 0,
|
||||
"run_name": "MyBenchmark",
|
||||
"run_type": "aggregate",
|
||||
"repetitions": 2,
|
||||
"threads": 1,
|
||||
"aggregate_name": "mean",
|
||||
"iterations": 2,
|
||||
"real_time": 1.6768104278028419e+06,
|
||||
"cpu_time": 1.6755708955223879e+06,
|
||||
"time_unit": "ns"
|
||||
},
|
||||
{
|
||||
"name": "MyBenchmark_median",
|
||||
"family_index": 0,
|
||||
"per_family_instance_index": 0,
|
||||
"run_name": "MyBenchmark",
|
||||
"run_type": "aggregate",
|
||||
"repetitions": 2,
|
||||
"threads": 1,
|
||||
"aggregate_name": "median",
|
||||
"iterations": 2,
|
||||
"real_time": 1.6768104278028419e+06,
|
||||
"cpu_time": 1.6755708955223879e+06,
|
||||
"time_unit": "ns"
|
||||
},
|
||||
{
|
||||
"name": "MyBenchmark_stddev",
|
||||
"family_index": 0,
|
||||
"per_family_instance_index": 0,
|
||||
"run_name": "MyBenchmark",
|
||||
"run_type": "aggregate",
|
||||
"repetitions": 2,
|
||||
"threads": 1,
|
||||
"aggregate_name": "stddev",
|
||||
"iterations": 2,
|
||||
"real_time": 5.3150648483981553e+04,
|
||||
"cpu_time": 5.2084922631119407e+04,
|
||||
"time_unit": "ns"
|
||||
}
|
||||
]
|
||||
}
|
||||
`
|
||||
got, err := bench.Parse(json)
|
||||
if err != nil {
|
||||
t.Errorf("bench.Parse() returned %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
expectedDate, err := time.Parse(time.RFC1123, "Mon, 24 Jan 2022 10:28:13 GMT")
|
||||
if err != nil {
|
||||
t.Errorf("time.Parse() returned %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
expect := bench.Run{
|
||||
Benchmarks: []bench.Benchmark{
|
||||
{Name: "MyBenchmark", Duration: time.Nanosecond * 1639227, AggregateType: ""},
|
||||
{Name: "MyBenchmark", Duration: time.Nanosecond * 1714393, AggregateType: ""},
|
||||
{Name: "MyBenchmark", Duration: time.Nanosecond * 1676810, AggregateType: "mean"},
|
||||
{Name: "MyBenchmark", Duration: time.Nanosecond * 1676810, AggregateType: "median"},
|
||||
{Name: "MyBenchmark", Duration: time.Nanosecond * 53150, AggregateType: "stddev"},
|
||||
},
|
||||
Context: &bench.Context{
|
||||
Date: expectedDate,
|
||||
HostName: "hostname",
|
||||
Executable: "./myexe",
|
||||
NumCPUs: 16,
|
||||
MhzPerCPU: 2400, CPUScalingEnabled: false,
|
||||
Caches: []bench.ContextCache{
|
||||
{Type: "Data", Level: 1, Size: 32768, NumSharing: 2},
|
||||
{Type: "Instruction", Level: 1, Size: 32768, NumSharing: 2},
|
||||
{Type: "Unified", Level: 2, Size: 262144, NumSharing: 2},
|
||||
{Type: "Unified", Level: 3, Size: 16777216, NumSharing: 16},
|
||||
},
|
||||
LoadAvg: []float32{2.60938, 2.59863, 2.55566}, LibraryBuildType: "release"},
|
||||
}
|
||||
|
||||
expectEqual(t, "bench.Parse().Benchmarks", got.Benchmarks, expect.Benchmarks)
|
||||
expectEqual(t, "bench.Parse().Context", got.Benchmarks, expect.Benchmarks)
|
||||
}
|
||||
|
||||
func TestCompare(t *testing.T) {
|
||||
a := []bench.Benchmark{
|
||||
{Name: "MyBenchmark1", Duration: time.Nanosecond * 1714393},
|
||||
{Name: "MyBenchmark0", Duration: time.Nanosecond * 1639227},
|
||||
{Name: "MyBenchmark3", Duration: time.Nanosecond * 1676810},
|
||||
{Name: "MyBenchmark4", Duration: time.Nanosecond * 53150},
|
||||
{Name: "MyBenchmark2", Duration: time.Nanosecond * 1676810},
|
||||
}
|
||||
b := []bench.Benchmark{
|
||||
{Name: "MyBenchmark1", Duration: time.Nanosecond * 56747654},
|
||||
{Name: "MyBenchmark0", Duration: time.Nanosecond * 236246},
|
||||
{Name: "MyBenchmark2", Duration: time.Nanosecond * 675865},
|
||||
{Name: "MyBenchmark4", Duration: time.Nanosecond * 2352336},
|
||||
{Name: "MyBenchmark3", Duration: time.Nanosecond * 87657868},
|
||||
}
|
||||
|
||||
minDiff := time.Millisecond * 2
|
||||
minRelDiff := 35.0
|
||||
|
||||
cmp := bench.Compare(a, b, minDiff, minRelDiff)
|
||||
|
||||
expectEqual(t, "bench.Compare().Format", cmp.Format(bench.DiffFormat{}), "")
|
||||
expectEqual(t, "bench.Compare().Format", "\n"+cmp.Format(bench.DiffFormat{TimeA: true}), `
|
||||
| A |
|
||||
|-----------|
|
||||
| 1.67681ms |
|
||||
| 53.15µs |
|
||||
`)
|
||||
expectEqual(t, "bench.Compare().Format", "\n"+cmp.Format(bench.DiffFormat{TimeA: true, TimeB: true}), `
|
||||
| A | B |
|
||||
|-----------+-------------|
|
||||
| 1.67681ms | 87.657868ms |
|
||||
| 53.15µs | 2.352336ms |
|
||||
`)
|
||||
expectEqual(t, "bench.Compare().Format", "\n"+cmp.Format(bench.DiffFormat{
|
||||
TestName: true,
|
||||
Delta: true,
|
||||
PercentChangeAB: true,
|
||||
TimeA: true,
|
||||
TimeB: true,
|
||||
}), `
|
||||
| Test name | Δ (A → B) | % (A → B) | A | B |
|
||||
|--------------+-------------+-----------+-----------+-------------|
|
||||
| MyBenchmark3 | 85.981058ms | +5127.7% | 1.67681ms | 87.657868ms |
|
||||
| MyBenchmark4 | 2.299186ms | +4325.8% | 53.15µs | 2.352336ms |
|
||||
`)
|
||||
expectEqual(t, "bench.Compare().Format", "\n"+cmp.Format(bench.DiffFormat{
|
||||
TestName: true,
|
||||
Delta: true,
|
||||
PercentChangeAB: true,
|
||||
PercentChangeBA: true,
|
||||
MultiplierChangeAB: true,
|
||||
MultiplierChangeBA: true,
|
||||
TimeA: true,
|
||||
TimeB: true,
|
||||
}), `
|
||||
| Test name | Δ (A → B) | % (A → B) | % (B → A) | × (A → B) | × (B → A) | A | B |
|
||||
|--------------+-------------+-----------+-----------+-----------+-----------+-----------+-------------|
|
||||
| MyBenchmark3 | 85.981058ms | +5127.7% | -98.1% | +52.2766 | +0.0191 | 1.67681ms | 87.657868ms |
|
||||
| MyBenchmark4 | 2.299186ms | +4325.8% | -97.7% | +44.2584 | +0.0226 | 53.15µs | 2.352336ms |
|
||||
`)
|
||||
}
|
||||
|
||||
func expectEqual(t *testing.T, desc string, got, expect interface{}) {
|
||||
if !reflect.DeepEqual(got, expect) {
|
||||
t.Errorf("%v was not expected:\nGot:\n%+v\nExpected:\n%+v", desc, got, expect)
|
||||
}
|
||||
}
|
||||
|
|
@ -23,9 +23,6 @@ import (
|
|||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
|
||||
"dawn.googlesource.com/tint/tools/src/bench"
|
||||
|
|
@ -77,7 +74,18 @@ func run(pathA, pathB string) error {
|
|||
return err
|
||||
}
|
||||
|
||||
compare(benchA, benchB, fileName(pathA), fileName(pathB))
|
||||
cmp := bench.Compare(benchA.Benchmarks, benchB.Benchmarks, *minDiff, *minRelDiff)
|
||||
diff := cmp.Format(bench.DiffFormat{
|
||||
TestName: true,
|
||||
Delta: true,
|
||||
PercentChangeAB: true,
|
||||
TimeA: true,
|
||||
TimeB: true,
|
||||
})
|
||||
|
||||
fmt.Println("A:", pathA, " B:", pathB)
|
||||
fmt.Println()
|
||||
fmt.Println(diff)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -86,92 +94,3 @@ func fileName(path string) string {
|
|||
_, name := filepath.Split(path)
|
||||
return name
|
||||
}
|
||||
|
||||
func compare(benchA, benchB bench.Benchmark, nameA, nameB string) {
|
||||
type times struct {
|
||||
a time.Duration
|
||||
b time.Duration
|
||||
}
|
||||
byName := map[string]times{}
|
||||
for _, test := range benchA.Tests {
|
||||
byName[test.Name] = times{a: test.Duration}
|
||||
}
|
||||
for _, test := range benchB.Tests {
|
||||
t := byName[test.Name]
|
||||
t.b = test.Duration
|
||||
byName[test.Name] = t
|
||||
}
|
||||
|
||||
type delta struct {
|
||||
name string
|
||||
times times
|
||||
relDiff float64
|
||||
absRelDiff float64
|
||||
}
|
||||
deltas := []delta{}
|
||||
for name, times := range byName {
|
||||
if times.a == 0 || times.b == 0 {
|
||||
continue // Assuming test was missing from a or b
|
||||
}
|
||||
diff := times.b - times.a
|
||||
absDiff := diff
|
||||
if absDiff < 0 {
|
||||
absDiff = -absDiff
|
||||
}
|
||||
if absDiff < *minDiff {
|
||||
continue
|
||||
}
|
||||
|
||||
relDiff := float64(times.b) / float64(times.a)
|
||||
absRelDiff := relDiff
|
||||
if absRelDiff < 1 {
|
||||
absRelDiff = 1.0 / absRelDiff
|
||||
}
|
||||
if absRelDiff < (1.0 + *minRelDiff) {
|
||||
continue
|
||||
}
|
||||
|
||||
d := delta{
|
||||
name: name,
|
||||
times: times,
|
||||
relDiff: relDiff,
|
||||
absRelDiff: absRelDiff,
|
||||
}
|
||||
deltas = append(deltas, d)
|
||||
}
|
||||
|
||||
sort.Slice(deltas, func(i, j int) bool { return deltas[j].relDiff < deltas[i].relDiff })
|
||||
|
||||
fmt.Println("A:", nameA)
|
||||
fmt.Println("B:", nameB)
|
||||
fmt.Println()
|
||||
|
||||
buf := strings.Builder{}
|
||||
{
|
||||
w := tabwriter.NewWriter(&buf, 1, 1, 0, ' ', 0)
|
||||
fmt.Fprintln(w, "Test name\t | Δ (A → B)\t | % (A → B)\t | % (B → A)\t | × (A → B)\t | × (B → A)\t | A \t | B")
|
||||
fmt.Fprintln(w, "\t-+\t-+\t-+\t-+\t-+\t-+\t-+\t-")
|
||||
for _, delta := range deltas {
|
||||
a2b := delta.times.b - delta.times.a
|
||||
fmt.Fprintf(w, "%v \t | %v \t | %+2.1f%% \t | %+2.1f%% \t | %+.4f \t | %+.4f \t | %v \t | %v \t|\n",
|
||||
delta.name,
|
||||
a2b, // Δ (A → B)
|
||||
100*float64(a2b)/float64(delta.times.a), // % (A → B)
|
||||
100*float64(-a2b)/float64(delta.times.b), // % (B → A)
|
||||
float64(delta.times.b)/float64(delta.times.a), // × (A → B)
|
||||
float64(delta.times.a)/float64(delta.times.b), // × (B → A)
|
||||
delta.times.a, // A
|
||||
delta.times.b, // B
|
||||
)
|
||||
}
|
||||
w.Flush()
|
||||
}
|
||||
|
||||
// Split the table by line so we can add in a header line
|
||||
lines := strings.Split(buf.String(), "\n")
|
||||
fmt.Println(lines[0])
|
||||
fmt.Println(strings.ReplaceAll(lines[1], " ", "-"))
|
||||
for _, l := range lines[2:] {
|
||||
fmt.Println(l)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,904 @@
|
|||
// Copyright 2022 The Tint Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"dawn.googlesource.com/tint/tools/src/bench"
|
||||
"dawn.googlesource.com/tint/tools/src/git"
|
||||
"github.com/andygrunwald/go-gerrit"
|
||||
"github.com/go-git/go-git/v5/plumbing/transport"
|
||||
"github.com/go-git/go-git/v5/plumbing/transport/http"
|
||||
"github.com/shirou/gopsutil/cpu"
|
||||
)
|
||||
|
||||
// main entry point
|
||||
func main() {
|
||||
var cfgPath string
|
||||
flag.StringVar(&cfgPath, "c", "~/.config/perfmon/config.json", "the config file")
|
||||
flag.Parse()
|
||||
|
||||
if err := run(cfgPath); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// run starts the perfmon tool with the given config path
|
||||
func run(cfgPath string) error {
|
||||
cfgPath, err := expandHomeDir(cfgPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := findTools(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
g, err := git.New(tools.git)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cfg, err := loadConfig(cfgPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tintDir, resultsDir, err := makeWorkingDirs(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tintRepo, err := createOrOpenGitRepo(g, tintDir, cfg.Tint)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resultsRepo, err := createOrOpenGitRepo(g, resultsDir, cfg.Results)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
gerritClient, err := gerrit.NewClient(cfg.Gerrit.URL, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
gerritClient.Authentication.SetBasicAuth(cfg.Gerrit.Username, cfg.Gerrit.Password)
|
||||
|
||||
sysInfo, err := cpu.Info()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to obtain system info:\n %v", err)
|
||||
}
|
||||
|
||||
e := env{
|
||||
cfg: cfg,
|
||||
git: g,
|
||||
system: sysInfo,
|
||||
systemID: hash(sysInfo)[:8],
|
||||
tintDir: tintDir,
|
||||
buildDir: filepath.Join(tintDir, "out"),
|
||||
resultsDir: resultsDir,
|
||||
tintRepo: tintRepo,
|
||||
resultsRepo: resultsRepo,
|
||||
gerrit: gerritClient,
|
||||
|
||||
benchmarkCache: map[git.Hash]*bench.Run{},
|
||||
}
|
||||
|
||||
for true {
|
||||
{
|
||||
log.Println("scanning for review changes to benchmark...")
|
||||
change, err := e.findGerritChangeToBenchmark()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if change != nil {
|
||||
if err := e.benchmarkGerritChange(*change); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
log.Println("scanning for submitted changes to benchmark...")
|
||||
changesToBenchmark, err := e.changesToBenchmark()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(changesToBenchmark) > 0 {
|
||||
log.Printf("benchmarking %v changes...\n", len(changesToBenchmark))
|
||||
for i, c := range changesToBenchmark {
|
||||
log.Printf("benchmarking %v/%v....\n", i+1, len(changesToBenchmark))
|
||||
benchRes, err := e.benchmarkTintChange(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
commitRes, err := e.benchmarksToCommitResults(c, *benchRes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("pushing results...\n")
|
||||
if err := e.pushUpdatedResults(*commitRes); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
log.Println("nothing to do. Sleeping...")
|
||||
time.Sleep(time.Minute * 5)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Config holds the root configuration options for the perfmon tool
|
||||
type Config struct {
|
||||
WorkingDir string
|
||||
RootChange git.Hash
|
||||
Tint GitConfig
|
||||
Results GitConfig
|
||||
Gerrit GerritConfig
|
||||
Timeouts TimeoutsConfig
|
||||
ExternalAccounts []string
|
||||
BenchmarkRepetitions int
|
||||
}
|
||||
|
||||
// GitConfig holds the configuration options for accessing a git repo
|
||||
type GitConfig struct {
|
||||
URL string
|
||||
Branch string
|
||||
Auth git.Auth
|
||||
}
|
||||
|
||||
// GerritConfig holds the configuration options for accessing gerrit
|
||||
type GerritConfig struct {
|
||||
URL string
|
||||
Username string
|
||||
Email string
|
||||
Password string
|
||||
}
|
||||
|
||||
// TimeoutsConfig holds the configuration options for timeouts
|
||||
type TimeoutsConfig struct {
|
||||
Sync time.Duration
|
||||
Build time.Duration
|
||||
Benchmark time.Duration
|
||||
}
|
||||
|
||||
// HistoricResults contains the full set of historic benchmark results for a
|
||||
// given system
|
||||
type HistoricResults struct {
|
||||
System []cpu.InfoStat
|
||||
Commits []CommitResults
|
||||
}
|
||||
|
||||
// CommitResults holds the results of a single tint commit
|
||||
type CommitResults struct {
|
||||
Commit string
|
||||
CommitTime time.Time
|
||||
CommitDescription string
|
||||
Benchmarks []Benchmark
|
||||
}
|
||||
|
||||
// Benchmark holds the benchmark results for a single test
|
||||
type Benchmark struct {
|
||||
Name string
|
||||
Mean float64
|
||||
Median float64
|
||||
Stddev float64
|
||||
}
|
||||
|
||||
// setDefaults assigns default values to unassigned fields of cfg
|
||||
func (cfg *Config) setDefaults() {
|
||||
if cfg.RootChange.IsZero() {
|
||||
cfg.RootChange, _ = git.ParseHash("be2362b18c792364c6bf5744db6d3837fbc655a0")
|
||||
}
|
||||
cfg.Tint.setDefaults()
|
||||
cfg.Results.setDefaults()
|
||||
cfg.Timeouts.setDefaults()
|
||||
if cfg.BenchmarkRepetitions < 2 {
|
||||
cfg.BenchmarkRepetitions = 2
|
||||
}
|
||||
}
|
||||
|
||||
// setDefaults assigns default values to unassigned fields of cfg
|
||||
func (cfg *GitConfig) setDefaults() {
|
||||
if cfg.Branch == "" {
|
||||
cfg.Branch = "main"
|
||||
}
|
||||
}
|
||||
|
||||
// setDefaults assigns default values to unassigned fields of cfg
|
||||
func (cfg *TimeoutsConfig) setDefaults() {
|
||||
if cfg.Sync == 0 {
|
||||
cfg.Sync = time.Minute * 10
|
||||
}
|
||||
if cfg.Build == 0 {
|
||||
cfg.Build = time.Minute * 10
|
||||
}
|
||||
if cfg.Benchmark == 0 {
|
||||
cfg.Benchmark = time.Minute * 30
|
||||
}
|
||||
}
|
||||
|
||||
// AuthConfig holds the authentication options for accessing a git repo
|
||||
type AuthConfig struct {
|
||||
Username string
|
||||
Password string
|
||||
}
|
||||
|
||||
// authMethod returns a http.BasicAuth constructed from the AuthConfig
|
||||
func (cfg AuthConfig) authMethod() transport.AuthMethod {
|
||||
if cfg.Username != "" || cfg.Password != "" {
|
||||
return &http.BasicAuth{Username: cfg.Username, Password: cfg.Password}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// env holds the perfmon main environment state
|
||||
type env struct {
|
||||
cfg Config
|
||||
git *git.Git
|
||||
system []cpu.InfoStat
|
||||
systemID string
|
||||
tintDir string
|
||||
buildDir string
|
||||
resultsDir string
|
||||
tintRepo *git.Repository
|
||||
resultsRepo *git.Repository
|
||||
gerrit *gerrit.Client
|
||||
|
||||
benchmarkCache map[git.Hash]*bench.Run
|
||||
}
|
||||
|
||||
// changesToBenchmark fetches the list of changes that do not currently have
|
||||
// benchmark results, which should be benchmarked.
|
||||
func (e env) changesToBenchmark() ([]git.Hash, error) {
|
||||
log.Println("syncing tint repo...")
|
||||
latest, err := e.tintRepo.Fetch(e.cfg.Tint.Branch, &git.FetchOptions{Auth: e.cfg.Tint.Auth})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
allChanges, err := e.tintRepo.Log(&git.LogOptions{
|
||||
From: e.cfg.RootChange.String(),
|
||||
To: latest.String(),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to obtain tint log:\n %w", err)
|
||||
}
|
||||
changesWithBenchmarks, err := e.changesWithBenchmarks()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to gather changes with existing benchmarks:\n %w", err)
|
||||
}
|
||||
changesToBenchmark := make([]git.Hash, 0, len(allChanges))
|
||||
for _, c := range allChanges {
|
||||
if _, exists := changesWithBenchmarks[c.Hash]; !exists {
|
||||
changesToBenchmark = append(changesToBenchmark, c.Hash)
|
||||
}
|
||||
}
|
||||
// Reverse the order of changesToBenchmark, so that the oldest comes first.
|
||||
for i := len(changesToBenchmark)/2 - 1; i >= 0; i-- {
|
||||
j := len(changesToBenchmark) - 1 - i
|
||||
changesToBenchmark[i], changesToBenchmark[j] = changesToBenchmark[j], changesToBenchmark[i]
|
||||
}
|
||||
|
||||
return changesToBenchmark, nil
|
||||
}
|
||||
|
||||
// benchmarkTintChange checks out the given commit, fetches the tint third party
|
||||
// dependencies, builds tint, then runs the benchmarks, returning the results.
|
||||
func (e env) benchmarkTintChange(hash git.Hash) (*bench.Run, error) {
|
||||
if cached, ok := e.benchmarkCache[hash]; ok {
|
||||
log.Printf("reusing cached benchmark results of '%v'...\n", hash)
|
||||
return cached, nil
|
||||
}
|
||||
|
||||
log.Printf("checking out tint at '%v'...\n", hash)
|
||||
if err := checkout(hash, e.tintRepo); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log.Println("fetching tint dependencies...")
|
||||
if err := e.fetchTintDeps(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log.Println("building tint...")
|
||||
if err := e.buildTint(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log.Println("benchmarking tint...")
|
||||
run, err := e.benchmarkTint()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
e.benchmarkCache[hash] = run
|
||||
return run, nil
|
||||
}
|
||||
|
||||
func (e env) benchmarksToCommitResults(hash git.Hash, results bench.Run) (*CommitResults, error) {
|
||||
commits, err := e.tintRepo.Log(&git.LogOptions{
|
||||
From: hash.String(),
|
||||
Count: 1,
|
||||
})
|
||||
if err != nil || len(commits) != 1 {
|
||||
return nil, fmt.Errorf("failed to get commit object '%v' of tint repo:\n %w", hash, err)
|
||||
}
|
||||
commit := commits[0]
|
||||
|
||||
m := map[string]Benchmark{}
|
||||
for _, b := range results.Benchmarks {
|
||||
benchmark := m[b.Name]
|
||||
benchmark.Name = b.Name
|
||||
switch b.AggregateType {
|
||||
case bench.Mean:
|
||||
benchmark.Mean = float64(b.Duration) / float64(time.Second)
|
||||
case bench.Median:
|
||||
benchmark.Median = float64(b.Duration) / float64(time.Second)
|
||||
case bench.Stddev:
|
||||
benchmark.Stddev = float64(b.Duration) / float64(time.Second)
|
||||
}
|
||||
m[b.Name] = benchmark
|
||||
}
|
||||
|
||||
sorted := make([]Benchmark, 0, len(m))
|
||||
for _, b := range m {
|
||||
sorted = append(sorted, b)
|
||||
}
|
||||
sort.Slice(sorted, func(i, j int) bool { return sorted[i].Name < sorted[i].Name })
|
||||
|
||||
return &CommitResults{
|
||||
Commit: commit.Hash.String(),
|
||||
CommitDescription: commit.Subject,
|
||||
CommitTime: commit.Date,
|
||||
Benchmarks: sorted,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// changesWithBenchmarks returns a set of tint changes that we already have
|
||||
// benchmarks for.
|
||||
func (e env) changesWithBenchmarks() (map[git.Hash]struct{}, error) {
|
||||
log.Println("syncing results repo...")
|
||||
if err := fetchAndCheckoutLatest(e.resultsRepo, e.cfg.Results); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
_, absPath, err := e.resultsFilePaths()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
results, err := e.loadHistoricResults(absPath)
|
||||
if err != nil {
|
||||
log.Println(fmt.Errorf("WARNING: failed to open result file '%v':\n %w", absPath, err))
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
m := make(map[git.Hash]struct{}, len(results.Commits))
|
||||
for _, c := range results.Commits {
|
||||
hash, err := git.ParseHash(c.Commit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m[hash] = struct{}{}
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (e env) pushUpdatedResults(res CommitResults) error {
|
||||
log.Println("syncing results repo...")
|
||||
if err := fetchAndCheckoutLatest(e.resultsRepo, e.cfg.Results); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
relPath, absPath, err := e.resultsFilePaths()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
h, err := e.loadHistoricResults(absPath)
|
||||
if err != nil {
|
||||
log.Println(fmt.Errorf("failed to open result file '%v'. Creating new file\n %w", absPath, err))
|
||||
h = &HistoricResults{System: e.system}
|
||||
}
|
||||
|
||||
h.Commits = append(h.Commits, res)
|
||||
|
||||
// Sort the commits by timestamp
|
||||
sort.Slice(h.Commits, func(i, j int) bool { return h.Commits[i].CommitTime.Before(h.Commits[j].CommitTime) })
|
||||
|
||||
// Write the new results to the file
|
||||
f, err := os.Create(absPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create updated results file '%v':\n %w", absPath, err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
enc := json.NewEncoder(f)
|
||||
enc.SetIndent("", " ")
|
||||
if err := enc.Encode(h); err != nil {
|
||||
return fmt.Errorf("failed to encode updated results file '%v':\n %w", absPath, err)
|
||||
}
|
||||
|
||||
// Stage the file
|
||||
if err := e.resultsRepo.Add(relPath, nil); err != nil {
|
||||
return fmt.Errorf("failed to stage updated results file '%v':\n %w", relPath, err)
|
||||
}
|
||||
|
||||
// Commit the change
|
||||
msg := fmt.Sprintf("Add benchmark results for '%v'", res.Commit[:6])
|
||||
hash, err := e.resultsRepo.Commit(msg, &git.CommitOptions{
|
||||
AuthorName: "tint perfmon bot",
|
||||
AuthorEmail: "tint-perfmon-bot@gmail.com",
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to commit updated results file '%v':\n %w", absPath, err)
|
||||
}
|
||||
|
||||
// Push the change
|
||||
log.Println("pushing updated results to results repo...")
|
||||
if err := e.resultsRepo.Push(hash.String(), e.cfg.Results.Branch, &git.PushOptions{Auth: e.cfg.Results.Auth}); err != nil {
|
||||
return fmt.Errorf("failed to push updated results file '%v':\n %w", absPath, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// resultsFilePaths returns the paths to the results.json file, holding the
|
||||
// benchmarks for the given system.
|
||||
func (e env) resultsFilePaths() (relPath string, absPath string, err error) {
|
||||
dir := filepath.Join(e.resultsDir, "results")
|
||||
if err = os.MkdirAll(dir, 0777); err != nil {
|
||||
err = fmt.Errorf("failed to create results directory '%v':\n %w", dir, err)
|
||||
return
|
||||
}
|
||||
relPath = filepath.Join("results", e.systemID+".json")
|
||||
absPath = filepath.Join(dir, e.systemID+".json")
|
||||
return
|
||||
}
|
||||
|
||||
// loadHistoricResults loads and returns the results.json file for the given
|
||||
// system.
|
||||
func (e env) loadHistoricResults(path string) (*HistoricResults, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open result file '%v':\n %w", path, err)
|
||||
}
|
||||
defer file.Close()
|
||||
res := &HistoricResults{}
|
||||
if err := json.NewDecoder(file).Decode(res); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse result file '%v':\n %w", path, err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(res.System, e.system) {
|
||||
log.Printf(`WARNING: results file '%v' has different system information!
|
||||
File: %+v
|
||||
System: %+v
|
||||
`, path, res.System, e.system)
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// fetchTintDeps fetches the third party tint dependencies using gclient.
|
||||
func (e env) fetchTintDeps() error {
|
||||
gclientConfig := filepath.Join(e.tintDir, ".gclient")
|
||||
if _, err := os.Stat(gclientConfig); errors.Is(err, os.ErrNotExist) {
|
||||
standalone := filepath.Join(e.tintDir, "standalone.gclient")
|
||||
if err := copyFile(gclientConfig, standalone); err != nil {
|
||||
return fmt.Errorf("failed to copy '%v' to '%v':\n %w", standalone, gclientConfig, err)
|
||||
}
|
||||
}
|
||||
if _, err := call(tools.gclient, e.tintDir, e.cfg.Timeouts.Sync,
|
||||
"sync",
|
||||
"--force",
|
||||
); err != nil {
|
||||
return fmt.Errorf("failed to fetch tint dependencies:\n %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildTint builds the tint benchmarks.
|
||||
func (e env) buildTint() error {
|
||||
if err := os.MkdirAll(e.buildDir, 0777); err != nil {
|
||||
return fmt.Errorf("failed to create build directory at '%v':\n %w", e.buildDir, err)
|
||||
}
|
||||
if _, err := call(tools.cmake, e.buildDir, e.cfg.Timeouts.Build,
|
||||
e.tintDir,
|
||||
"-GNinja",
|
||||
"-DCMAKE_CXX_COMPILER_LAUNCHER=ccache",
|
||||
"-DCMAKE_BUILD_TYPE=Release",
|
||||
"-DTINT_BUILD_SPV_READER=1",
|
||||
"-DTINT_BUILD_WGSL_READER=1",
|
||||
"-DTINT_BUILD_GLSL_WRITER=1",
|
||||
"-DTINT_BUILD_HLSL_WRITER=1",
|
||||
"-DTINT_BUILD_MSL_WRITER=1",
|
||||
"-DTINT_BUILD_SPV_WRITER=1",
|
||||
"-DTINT_BUILD_WGSL_WRITER=1",
|
||||
"-DTINT_BUILD_BENCHMARKS=1",
|
||||
); err != nil {
|
||||
return fmt.Errorf("failed to generate tint build config:\n %w", err)
|
||||
}
|
||||
if _, err := call(tools.ninja, e.buildDir, e.cfg.Timeouts.Build); err != nil {
|
||||
return fmt.Errorf("failed to build tint:\n %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// benchmarkTint runs the tint benchmarks, returning the results.
|
||||
func (e env) benchmarkTint() (*bench.Run, error) {
|
||||
exe := filepath.Join(e.buildDir, "tint-benchmark")
|
||||
out, err := call(exe, e.buildDir, e.cfg.Timeouts.Benchmark,
|
||||
"--benchmark_format=json",
|
||||
fmt.Sprintf("--benchmark_repetitions=%v", e.cfg.BenchmarkRepetitions),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to benchmark tint:\n %w", err)
|
||||
}
|
||||
|
||||
results, err := bench.Parse(out)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse benchmark results:\n %w", err)
|
||||
}
|
||||
return &results, nil
|
||||
}
|
||||
|
||||
// findGerritChangeToBenchmark queries gerrit for a change to benchmark.
|
||||
func (e env) findGerritChangeToBenchmark() (*gerrit.ChangeInfo, error) {
|
||||
log.Println("querying gerrit for changes...")
|
||||
results, _, err := e.gerrit.Changes.QueryChanges(&gerrit.QueryChangeOptions{
|
||||
QueryOptions: gerrit.QueryOptions{
|
||||
Query: []string{"project:tint status:open+-age:3d"},
|
||||
Limit: 100,
|
||||
},
|
||||
ChangeOptions: gerrit.ChangeOptions{
|
||||
AdditionalFields: []string{"CURRENT_REVISION", "CURRENT_COMMIT", "MESSAGES", "LABELS", "DETAILED_ACCOUNTS"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get list of changes:\n %w", err)
|
||||
}
|
||||
|
||||
type candidate struct {
|
||||
change gerrit.ChangeInfo
|
||||
priority int
|
||||
}
|
||||
|
||||
candidates := make([]candidate, 0, len(*results))
|
||||
|
||||
for _, change := range *results {
|
||||
kokoroApproved := change.Labels["Kokoro"].Approved.AccountID != 0
|
||||
codeReviewScore := change.Labels["Code-Review"].Value
|
||||
codeReviewApproved := change.Labels["Code-Review"].Approved.AccountID != 0
|
||||
presubmitReady := change.Labels["Presubmit-Ready"].Approved.AccountID != 0
|
||||
verifiedScore := change.Labels["Verified"].Value
|
||||
|
||||
current, ok := change.Revisions[change.CurrentRevision]
|
||||
if !ok {
|
||||
log.Printf("WARNING: couldn't find current revision for change '%s'", change.ChangeID)
|
||||
}
|
||||
|
||||
canBenchmark := func() bool {
|
||||
// Is the change from a Googler, reviewed by a Googler or is from a allow-listed external developer?
|
||||
if !(strings.HasSuffix(current.Commit.Committer.Email, "@google.com") ||
|
||||
strings.HasSuffix(change.Labels["Code-Review"].Approved.Email, "@google.com") ||
|
||||
strings.HasSuffix(change.Labels["Code-Review"].Recommended.Email, "@google.com") ||
|
||||
strings.HasSuffix(change.Labels["Presubmit-Ready"].Approved.Email, "@google.com")) {
|
||||
permitted := false
|
||||
for _, email := range e.cfg.ExternalAccounts {
|
||||
if strings.ToLower(current.Commit.Committer.Email) == strings.ToLower(email) {
|
||||
permitted = true
|
||||
break
|
||||
}
|
||||
|
||||
}
|
||||
if !permitted {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Don't benchmark if the change has negative scores.
|
||||
if codeReviewScore < 0 || verifiedScore < 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
// Has the latest patchset already been benchmarked?
|
||||
for _, msg := range change.Messages {
|
||||
if msg.RevisionNumber == current.Number &&
|
||||
msg.Author.Email == e.cfg.Gerrit.Email {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}()
|
||||
if !canBenchmark {
|
||||
continue
|
||||
}
|
||||
|
||||
priority := 0
|
||||
if presubmitReady {
|
||||
priority += 10
|
||||
}
|
||||
priority += codeReviewScore
|
||||
if codeReviewApproved {
|
||||
priority += 2
|
||||
}
|
||||
if kokoroApproved {
|
||||
priority++
|
||||
}
|
||||
|
||||
candidates = append(candidates, candidate{change, priority})
|
||||
}
|
||||
|
||||
// Sort the candidates
|
||||
sort.Slice(candidates, func(i, j int) bool {
|
||||
return candidates[i].priority > candidates[j].priority
|
||||
})
|
||||
|
||||
if len(candidates) > 0 {
|
||||
log.Printf("%d gerrit changes to benchmark\n", len(candidates))
|
||||
return &candidates[0].change, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// benchmarks the gerrit change, posting the findings to the change
|
||||
func (e env) benchmarkGerritChange(change gerrit.ChangeInfo) error {
|
||||
current := change.Revisions[change.CurrentRevision]
|
||||
log.Printf("fetching '%v'...\n", current.Ref)
|
||||
currentHash, err := e.tintRepo.Fetch(current.Ref, &git.FetchOptions{Auth: e.cfg.Tint.Auth})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
parent := current.Commit.Parents[0].Commit
|
||||
parentHash, err := git.ParseHash(parent)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to parse parent hash '%v':\n %v", parent, err)
|
||||
}
|
||||
newRun, err := e.benchmarkTintChange(currentHash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := e.tintRepo.Fetch(parent, &git.FetchOptions{Auth: e.cfg.Tint.Auth}); err != nil {
|
||||
return err
|
||||
}
|
||||
parentRun, err := e.benchmarkTintChange(parentHash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// filters the benchmark results to only the mean aggregate values
|
||||
meanBenchmarkResults := func(in []bench.Benchmark) []bench.Benchmark {
|
||||
out := make([]bench.Benchmark, 0, len(in))
|
||||
for _, b := range in {
|
||||
if b.AggregateType == bench.Mean {
|
||||
out = append(out, b)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
newResults := meanBenchmarkResults(newRun.Benchmarks)
|
||||
parentResults := meanBenchmarkResults(parentRun.Benchmarks)
|
||||
|
||||
const minDiff = time.Microsecond * 50 // Ignore time diffs less than this duration
|
||||
const minRelDiff = 0.01 // Ignore absolute relative diffs between [1, 1+x]
|
||||
diff := bench.Compare(parentResults, newResults, minDiff, minRelDiff)
|
||||
diffFmt := bench.DiffFormat{
|
||||
TestName: true,
|
||||
Delta: true,
|
||||
PercentChangeAB: true,
|
||||
TimeA: true,
|
||||
TimeB: true,
|
||||
}
|
||||
|
||||
msg := &strings.Builder{}
|
||||
fmt.Fprintf(msg, "Tint perfmon analysis:\n")
|
||||
fmt.Fprintf(msg, " \n")
|
||||
fmt.Fprintf(msg, " A: parent change (%v) -> B: patchset %v\n", parent[:7], current.Number)
|
||||
fmt.Fprintf(msg, " \n")
|
||||
for _, line := range strings.Split(diff.Format(diffFmt), "\n") {
|
||||
fmt.Fprintf(msg, " %v\n", line)
|
||||
}
|
||||
|
||||
notify := "OWNER"
|
||||
if len(diff) > 0 {
|
||||
notify = "OWNER_REVIEWERS"
|
||||
}
|
||||
_, _, err = e.gerrit.Changes.SetReview(change.ChangeID, currentHash.String(), &gerrit.ReviewInput{
|
||||
Message: msg.String(),
|
||||
Tag: "autogenerated:perfmon",
|
||||
Notify: notify,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to post benchmark results to gerrit change:\n %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// createOrOpenGitRepo creates a new local repo by cloning cfg.URL into
|
||||
// filepath, or opens the existing repo at filepath.
|
||||
func createOrOpenGitRepo(g *git.Git, filepath string, cfg GitConfig) (*git.Repository, error) {
|
||||
repo, err := g.Open(filepath)
|
||||
if errors.Is(err, git.ErrRepositoryDoesNotExist) {
|
||||
log.Printf("cloning '%v' branch '%v' to '%v'...", cfg.URL, cfg.Branch, filepath)
|
||||
repo, err = g.Clone(filepath, cfg.URL, &git.CloneOptions{
|
||||
Branch: cfg.Branch,
|
||||
Auth: cfg.Auth,
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open git repository '%v':\n %w", filepath, err)
|
||||
}
|
||||
return repo, err
|
||||
}
|
||||
|
||||
// loadConfig loads the perfmon config file.
|
||||
func loadConfig(path string) (Config, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return Config{}, fmt.Errorf("failed to open config file at '%v':\n %w", path, err)
|
||||
}
|
||||
cfg := Config{}
|
||||
if err := json.NewDecoder(f).Decode(&cfg); err != nil {
|
||||
return Config{}, fmt.Errorf("failed to load config file at '%v':\n %w", path, err)
|
||||
}
|
||||
cfg.setDefaults()
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// makeWorkingDirs builds the tint repo and results repo directories.
|
||||
func makeWorkingDirs(cfg Config) (tintDir, resultsDir string, err error) {
|
||||
wd, err := expandHomeDir(cfg.WorkingDir)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if err := os.MkdirAll(wd, 0777); err != nil {
|
||||
return "", "", fmt.Errorf("failed to create working directory '%v':\n %w", wd, err)
|
||||
}
|
||||
tintDir = filepath.Join(wd, "tint")
|
||||
if err := os.MkdirAll(tintDir, 0777); err != nil {
|
||||
return "", "", fmt.Errorf("failed to create working tint directory '%v':\n %w", tintDir, err)
|
||||
}
|
||||
resultsDir = filepath.Join(wd, "results")
|
||||
if err := os.MkdirAll(resultsDir, 0777); err != nil {
|
||||
return "", "", fmt.Errorf("failed to create working results directory '%v':\n %w", resultsDir, err)
|
||||
}
|
||||
return tintDir, resultsDir, nil
|
||||
}
|
||||
|
||||
// fetchAndCheckoutLatest calls fetch(cfg.Branch) followed by checkoutLatest().
|
||||
func fetchAndCheckoutLatest(repo *git.Repository, cfg GitConfig) error {
|
||||
hash, err := repo.Fetch(cfg.Branch, &git.FetchOptions{Auth: cfg.Auth})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := repo.Checkout(hash.String(), nil); err != nil {
|
||||
return err
|
||||
}
|
||||
return checkout(hash, repo)
|
||||
}
|
||||
|
||||
// checkout checks out the change with the given hash.
|
||||
// Note: call fetch() to ensure that this is the latest change on the
|
||||
// branch.
|
||||
func checkout(hash git.Hash, repo *git.Repository) error {
|
||||
if err := repo.Checkout(hash.String(), nil); err != nil {
|
||||
return fmt.Errorf("failed to checkout '%v':\n %w", hash, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// expandHomeDir returns path with all occurrences of '~' replaced with the user
|
||||
// home directory.
|
||||
func expandHomeDir(path string) (string, error) {
|
||||
if strings.ContainsRune(path, '~') {
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to expand home dir:\n %w", err)
|
||||
}
|
||||
path = strings.ReplaceAll(path, "~", home)
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
|
||||
// tools holds the file paths to the executables used by this tool
|
||||
var tools struct {
|
||||
ccache string
|
||||
cmake string
|
||||
gclient string
|
||||
git string
|
||||
ninja string
|
||||
}
|
||||
|
||||
// findTools looks for the file paths for executables used by this tool,
|
||||
// returning an error if any could not be found.
|
||||
func findTools() error {
|
||||
for _, tool := range []struct {
|
||||
name string
|
||||
path *string
|
||||
}{
|
||||
{"ccache", &tools.ccache},
|
||||
{"cmake", &tools.cmake},
|
||||
{"gclient", &tools.gclient},
|
||||
{"git", &tools.git},
|
||||
{"ninja", &tools.ninja},
|
||||
} {
|
||||
path, err := exec.LookPath(tool.name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to find path to '%v':\n %w", tool.name, err)
|
||||
}
|
||||
*tool.path = path
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// copyFile copies the file at srcPath to dstPath.
|
||||
func copyFile(dstPath, srcPath string) error {
|
||||
src, err := os.Open(srcPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open file '%v':\n %w", srcPath, err)
|
||||
}
|
||||
defer src.Close()
|
||||
dst, err := os.Create(dstPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create file '%v':\n %w", dstPath, err)
|
||||
}
|
||||
defer dst.Close()
|
||||
_, err = io.Copy(dst, src)
|
||||
return err
|
||||
}
|
||||
|
||||
// call invokes the executable exe in the current working directory wd, with
|
||||
// the provided arguments.
|
||||
// If the executable does not complete within the timeout duration, then an
|
||||
// error is returned.
|
||||
func call(exe, wd string, timeout time.Duration, args ...string) (string, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, exe, args...)
|
||||
cmd.Dir = wd
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return string(out), fmt.Errorf("'%v %v' failed:\n %w\n%v", exe, args, err, string(out))
|
||||
}
|
||||
return string(out), nil
|
||||
}
|
||||
|
||||
// hash returns a hash of the string representation of 'o'.
|
||||
func hash(o interface{}) string {
|
||||
str := fmt.Sprintf("%+v", o)
|
||||
hash := sha256.New()
|
||||
hash.Write([]byte(str))
|
||||
return hex.EncodeToString(hash.Sum(nil))[:8]
|
||||
}
|
||||
|
|
@ -90,7 +90,7 @@ func run() error {
|
|||
|
||||
// This tool uses a mix of 'go-git' and the command line git.
|
||||
// go-git has the benefit of keeping the git information entirely in-memory,
|
||||
// but has issues working with chromiums tools and gerrit.
|
||||
// but has issues working with chromium's tools and gerrit.
|
||||
// To create new release branches in Tint, we use 'go-git', so we need to
|
||||
// dig out the username and password.
|
||||
var auth transport.AuthMethod
|
||||
|
|
|
|||
|
|
@ -0,0 +1,381 @@
|
|||
// Copyright 2022 The Tint Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package git
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Hash is a 20 byte, git object hash.
|
||||
type Hash [20]byte
|
||||
|
||||
func (h Hash) String() string { return hex.EncodeToString(h[:]) }
|
||||
|
||||
// IsZero returns true if the hash h is all zeros
|
||||
func (h Hash) IsZero() bool {
|
||||
zero := Hash{}
|
||||
return h == zero
|
||||
}
|
||||
|
||||
// ParseHash returns a Hash from a hexadecimal string.
|
||||
func ParseHash(s string) (Hash, error) {
|
||||
b, err := hex.DecodeString(s)
|
||||
if err != nil {
|
||||
return Hash{}, fmt.Errorf("failed to parse hash '%v':\n %w", s, err)
|
||||
}
|
||||
h := Hash{}
|
||||
copy(h[:], b)
|
||||
return h, nil
|
||||
}
|
||||
|
||||
// The timeout for git operations if no other timeout is specified
|
||||
var DefaultTimeout = time.Minute
|
||||
|
||||
// Git wraps the 'git' executable
|
||||
type Git struct {
|
||||
// Path to the git executable
|
||||
exe string
|
||||
// Debug flag to print all command to the `git` executable
|
||||
LogAllActions bool
|
||||
}
|
||||
|
||||
// New returns a new Git instance
|
||||
func New(exe string) (*Git, error) {
|
||||
if _, err := os.Stat(exe); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Git{exe: exe}, nil
|
||||
}
|
||||
|
||||
// Auth holds git authentication credentials
|
||||
type Auth struct {
|
||||
Username string
|
||||
Password string
|
||||
}
|
||||
|
||||
// Empty return true if there's no username or password for authentication
|
||||
func (a Auth) Empty() bool {
|
||||
return a.Username == "" && a.Password == ""
|
||||
}
|
||||
|
||||
// ErrRepositoryDoesNotExist indicates that a repository does not exist
|
||||
var ErrRepositoryDoesNotExist = errors.New("repository does not exist")
|
||||
|
||||
// Open opens an existing git repo at path. If the repository does not exist at
|
||||
// path then ErrRepositoryDoesNotExist is returned.
|
||||
func (g Git) Open(path string) (*Repository, error) {
|
||||
info, err := os.Stat(filepath.Join(path, ".git"))
|
||||
if err != nil || !info.IsDir() {
|
||||
return nil, ErrRepositoryDoesNotExist
|
||||
}
|
||||
return &Repository{g, path}, nil
|
||||
}
|
||||
|
||||
// Optional settings for Git.Clone
|
||||
type CloneOptions struct {
|
||||
// If specified then the given branch will be cloned instead of the default
|
||||
Branch string
|
||||
// Timeout for the operation
|
||||
Timeout time.Duration
|
||||
// Authentication for the clone
|
||||
Auth Auth
|
||||
}
|
||||
|
||||
// Clone performs a clone of the repository at url to path.
|
||||
func (g Git) Clone(path, url string, opt *CloneOptions) (*Repository, error) {
|
||||
if err := os.MkdirAll(path, 0777); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if opt == nil {
|
||||
opt = &CloneOptions{}
|
||||
}
|
||||
url, err := opt.Auth.addToURL(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r := &Repository{g, path}
|
||||
args := []string{"clone", url, "."}
|
||||
if opt.Branch != "" {
|
||||
args = append(args, "--branch", opt.Branch)
|
||||
}
|
||||
if _, err := r.run(opt.Timeout, args...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// Repository points to a git repository
|
||||
type Repository struct {
|
||||
// Path to the 'git' executable
|
||||
Git Git
|
||||
// Repo directory
|
||||
Path string
|
||||
}
|
||||
|
||||
// Optional settings for Repository.Fetch
|
||||
type FetchOptions struct {
|
||||
// The remote name. Defaults to 'origin'
|
||||
Remote string
|
||||
// Timeout for the operation
|
||||
Timeout time.Duration
|
||||
// Git authentication for the remote
|
||||
Auth Auth
|
||||
}
|
||||
|
||||
// Fetch performs a fetch of a reference from the remote, returning the Hash of
|
||||
// the fetched reference.
|
||||
func (r Repository) Fetch(ref string, opt *FetchOptions) (Hash, error) {
|
||||
if opt == nil {
|
||||
opt = &FetchOptions{}
|
||||
}
|
||||
if opt.Remote == "" {
|
||||
opt.Remote = "origin"
|
||||
}
|
||||
if _, err := r.run(opt.Timeout, "fetch", opt.Remote, ref); err != nil {
|
||||
return Hash{}, err
|
||||
}
|
||||
out, err := r.run(0, "rev-parse", "FETCH_HEAD")
|
||||
if err != nil {
|
||||
return Hash{}, err
|
||||
}
|
||||
return ParseHash(out)
|
||||
}
|
||||
|
||||
// Optional settings for Repository.Push
|
||||
type PushOptions struct {
|
||||
// The remote name. Defaults to 'origin'
|
||||
Remote string
|
||||
// Timeout for the operation
|
||||
Timeout time.Duration
|
||||
// Git authentication for the remote
|
||||
Auth Auth
|
||||
}
|
||||
|
||||
// Push performs a push of the local reference to the remote reference.
|
||||
func (r Repository) Push(localRef, remoteRef string, opt *PushOptions) error {
|
||||
if opt == nil {
|
||||
opt = &PushOptions{}
|
||||
}
|
||||
if opt.Remote == "" {
|
||||
opt.Remote = "origin"
|
||||
}
|
||||
url, err := r.run(opt.Timeout, "remote", "get-url", opt.Remote)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
url, err = opt.Auth.addToURL(url)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := r.run(opt.Timeout, "push", url, localRef+":"+remoteRef); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Optional settings for Repository.Add
|
||||
type AddOptions struct {
|
||||
// Timeout for the operation
|
||||
Timeout time.Duration
|
||||
// Git authentication for the remote
|
||||
Auth Auth
|
||||
}
|
||||
|
||||
// Add stages the listed files
|
||||
func (r Repository) Add(path string, opt *AddOptions) error {
|
||||
if opt == nil {
|
||||
opt = &AddOptions{}
|
||||
}
|
||||
if _, err := r.run(opt.Timeout, "add", path); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Optional settings for Repository.Commit
|
||||
type CommitOptions struct {
|
||||
// Timeout for the operation
|
||||
Timeout time.Duration
|
||||
// Author name
|
||||
AuthorName string
|
||||
// Author email address
|
||||
AuthorEmail string
|
||||
}
|
||||
|
||||
// Commit commits the staged files with the given message, returning the hash of
|
||||
// commit
|
||||
func (r Repository) Commit(msg string, opt *CommitOptions) (Hash, error) {
|
||||
if opt == nil {
|
||||
opt = &CommitOptions{}
|
||||
}
|
||||
args := []string{"commit", "-m", msg}
|
||||
if opt.AuthorName != "" || opt.AuthorEmail != "" {
|
||||
args = append(args, "--author", fmt.Sprintf("%v <%v>", opt.AuthorName, opt.AuthorEmail))
|
||||
}
|
||||
if _, err := r.run(opt.Timeout, args...); err != nil {
|
||||
return Hash{}, err
|
||||
}
|
||||
out, err := r.run(0, "rev-parse", "HEAD")
|
||||
if err != nil {
|
||||
return Hash{}, err
|
||||
}
|
||||
return ParseHash(out)
|
||||
}
|
||||
|
||||
// Optional settings for Repository.Checkout
|
||||
type CheckoutOptions struct {
|
||||
// Timeout for the operation
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// Checkout performs a checkout of a reference.
|
||||
func (r Repository) Checkout(ref string, opt *CheckoutOptions) error {
|
||||
if opt == nil {
|
||||
opt = &CheckoutOptions{}
|
||||
}
|
||||
if _, err := r.run(opt.Timeout, "checkout", ref); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Optional settings for Repository.Log
|
||||
type LogOptions struct {
|
||||
// The git reference to the oldest commit in the range to query.
|
||||
From string
|
||||
// The git reference to the newest commit in the range to query.
|
||||
To string
|
||||
// The maximum number of entries to return.
|
||||
Count int
|
||||
// Timeout for the operation
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// CommitInfo describes a single git commit
|
||||
type CommitInfo struct {
|
||||
Hash Hash
|
||||
Date time.Time
|
||||
Author string
|
||||
Subject string
|
||||
Description string
|
||||
}
|
||||
|
||||
// Log returns the list of commits between two references (inclusive).
|
||||
// The first returned commit is the most recent.
|
||||
func (r Repository) Log(opt *LogOptions) ([]CommitInfo, error) {
|
||||
if opt == nil {
|
||||
opt = &LogOptions{}
|
||||
}
|
||||
args := []string{"log"}
|
||||
rng := "HEAD"
|
||||
if opt.To != "" {
|
||||
rng = opt.To
|
||||
}
|
||||
if opt.From != "" {
|
||||
rng = opt.From + "^.." + rng
|
||||
}
|
||||
args = append(args, rng, "--pretty=format:ǁ%Hǀ%cIǀ%an <%ae>ǀ%sǀ%b")
|
||||
if opt.Count != 0 {
|
||||
args = append(args, fmt.Sprintf("-%d", opt.Count))
|
||||
}
|
||||
out, err := r.run(opt.Timeout, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return parseLog(out)
|
||||
}
|
||||
|
||||
func (r Repository) run(timeout time.Duration, args ...string) (string, error) {
|
||||
return r.Git.run(r.Path, timeout, args...)
|
||||
}
|
||||
|
||||
func (r Repository) runAll(timeout time.Duration, args ...[]string) error {
|
||||
for _, a := range args {
|
||||
if _, err := r.run(timeout, a...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g Git) run(dir string, timeout time.Duration, args ...string) (string, error) {
|
||||
if timeout == 0 {
|
||||
timeout = DefaultTimeout
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, g.exe, args...)
|
||||
cmd.Dir = dir
|
||||
if g.LogAllActions {
|
||||
fmt.Printf("%v> %v %v\n", dir, g.exe, strings.Join(args, " "))
|
||||
}
|
||||
out, err := cmd.CombinedOutput()
|
||||
if g.LogAllActions {
|
||||
fmt.Println(string(out))
|
||||
}
|
||||
if err != nil {
|
||||
return string(out), fmt.Errorf("%v> %v %v failed:\n %w\n%v",
|
||||
dir, g.exe, strings.Join(args, " "), err, string(out))
|
||||
}
|
||||
return strings.TrimSpace(string(out)), nil
|
||||
}
|
||||
|
||||
func (a Auth) addToURL(u string) (string, error) {
|
||||
if !a.Empty() {
|
||||
modified, err := url.Parse(u)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to parse url '%v': %v", u, err)
|
||||
}
|
||||
modified.User = url.UserPassword(a.Username, a.Password)
|
||||
u = modified.String()
|
||||
}
|
||||
return u, nil
|
||||
}
|
||||
|
||||
func parseLog(str string) ([]CommitInfo, error) {
|
||||
msgs := strings.Split(str, "ǁ")
|
||||
cls := make([]CommitInfo, 0, len(msgs))
|
||||
for _, s := range msgs {
|
||||
if parts := strings.Split(s, "ǀ"); len(parts) == 5 {
|
||||
hash, err := ParseHash(parts[0])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
date, err := time.Parse(time.RFC3339, parts[1])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cl := CommitInfo{
|
||||
Hash: hash,
|
||||
Date: date,
|
||||
Author: strings.TrimSpace(parts[2]),
|
||||
Subject: strings.TrimSpace(parts[3]),
|
||||
Description: strings.TrimSpace(parts[4]),
|
||||
}
|
||||
|
||||
cls = append(cls, cl)
|
||||
}
|
||||
}
|
||||
return cls, nil
|
||||
}
|
||||
|
|
@ -7,5 +7,8 @@ require (
|
|||
github.com/fatih/color v1.10.0
|
||||
github.com/go-git/go-git/v5 v5.4.2
|
||||
github.com/sergi/go-diff v1.2.0
|
||||
github.com/shirou/gopsutil v3.21.11+incompatible
|
||||
github.com/tklauser/go-sysconf v0.3.9 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.2 // indirect
|
||||
golang.org/x/net v0.0.0-20210614182718-04defd469f4e
|
||||
)
|
||||
|
|
|
|||
|
|
@ -31,6 +31,8 @@ github.com/go-git/go-git-fixtures/v4 v4.2.1 h1:n9gGL1Ct/yIw+nfsfr8s4+sbhT+Ncu2Su
|
|||
github.com/go-git/go-git-fixtures/v4 v4.2.1/go.mod h1:K8zd3kDUAykwTdDCr+I0per6Y6vMiRR/nnVTBtavnB0=
|
||||
github.com/go-git/go-git/v5 v5.4.2 h1:BXyZu9t0VkbiHtqrsvdq39UDhGJTl1h55VW6CSC4aY4=
|
||||
github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti4ihgckDc=
|
||||
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
|
||||
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
|
|
@ -68,6 +70,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
|
|||
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
|
||||
github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
|
||||
github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
|
||||
github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
|
||||
github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
|
||||
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
|
|
@ -75,8 +79,14 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
|
|||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo=
|
||||
github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs=
|
||||
github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ=
|
||||
github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8=
|
||||
github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI=
|
||||
github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0=
|
||||
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
|
||||
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
|
||||
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b h1:7mWr3k41Qtv8XlltBkDkl8LoP3mpSgBW8BUoxtEdbXg=
|
||||
|
|
@ -95,8 +105,9 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||
golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79 h1:RX8C8PRZc2hTIod4ds8ij+/4RQX3AqhYj3uOHmyaz4E=
|
||||
golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71 h1:ikCpsnYR+Ew0vu99XlDp55lGgDJdIMx3f4a18jfse/s=
|
||||
golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
|
|
|
|||
Loading…
Reference in New Issue