From d9f2a61c6f9e9436240fd0f0115d38d529ea48f1 Mon Sep 17 00:00:00 2001 From: Ben Clayton Date: Mon, 2 May 2022 20:44:21 +0000 Subject: [PATCH] tools: Add the 'cts export' sub-command Updates a Google spreadsheet with latest CTS statistics. Bug: dawn:1342 Change-Id: I01a9c99032c755d6a78af674675d8c9cbfc79019 Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/88443 Reviewed-by: Dan Sinclair Commit-Queue: Ben Clayton Kokoro: Kokoro --- go.mod | 7 +- go.sum | 3 + tools/src/cmd/cts/common/deps.go | 71 +++++++ tools/src/cmd/cts/export/export.go | 307 +++++++++++++++++++++++++++++ tools/src/cmd/cts/main.go | 1 + 5 files changed, 387 insertions(+), 2 deletions(-) create mode 100644 tools/src/cmd/cts/common/deps.go create mode 100644 tools/src/cmd/cts/export/export.go diff --git a/go.mod b/go.mod index b6d7bb1670..21d9656bfe 100644 --- a/go.mod +++ b/go.mod @@ -14,15 +14,19 @@ require ( github.com/tidwall/jsonc v0.3.2 go.chromium.org/luci v0.0.0-20220412023008-ab2409fe739a golang.org/x/net v0.0.0-20220403103023-749bd193bc2b + golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 + google.golang.org/api v0.63.0 google.golang.org/protobuf v1.28.0 ) require ( cloud.google.com/go v0.99.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/mock v1.6.0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/go-querystring v1.1.0 // indirect + github.com/googleapis/gax-go/v2 v2.1.1 // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect github.com/klauspost/compress v1.13.5 // indirect github.com/kr/pretty v0.3.0 // indirect @@ -34,11 +38,10 @@ require ( github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/tklauser/numcpus v0.4.0 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect - golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect + go.opencensus.io v0.23.0 // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27 // indirect golang.org/x/text v0.3.7 // indirect - google.golang.org/api v0.63.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20220216160803-4663080d8bc8 // indirect google.golang.org/grpc v1.44.0 // indirect diff --git a/go.sum b/go.sum index bdd66d6da7..0b540d3577 100644 --- a/go.sum +++ b/go.sum @@ -174,6 +174,7 @@ github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -257,6 +258,7 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1 h1:dp3bWCh+PPO1zjRRiCSczJav13sBvG4UhNyVTa1KqdU= github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20210901121439-eee08aaf2717 h1:V1j4G8AXIJeyzT3ng2Oh4IRo/VEgRWYAsyYwhOz5rko= @@ -490,6 +492,7 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.22.6/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.starlark.net v0.0.0-20210223155950-e043a3d3c984/go.mod h1:t3mmBBPzAVvK0L0n1drDmrQsJ8FoIx4INCqVMTr/Zo0= diff --git a/tools/src/cmd/cts/common/deps.go b/tools/src/cmd/cts/common/deps.go new file mode 100644 index 0000000000..27b13d3074 --- /dev/null +++ b/tools/src/cmd/cts/common/deps.go @@ -0,0 +1,71 @@ +// Copyright 2022 The Dawn Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "fmt" + "regexp" + "strings" + + "dawn.googlesource.com/dawn/tools/src/container" +) + +// The regular expression used to search for the CTS hash +var reCTSHash = regexp.MustCompile(reEscape(ctsHashPrefix) + `[0-9a-fA-F]+`) + +const ( + // The string prefix for the CTS hash in the DEPs file, used for identifying + // and updating the DEPS file. + ctsHashPrefix = `{chromium_git}/external/github.com/gpuweb/cts@` +) + +func reEscape(s string) string { + return strings.ReplaceAll(strings.ReplaceAll(s, `/`, `\/`), `.`, `\.`) +} + +// UpdateCTSHashInDeps replaces the CTS hashes in 'deps' with 'newCTSHash'. +// Returns: +// newDEPS - the new DEPS content +// oldCTSHash - the old CTS hash in the 'deps' +func UpdateCTSHashInDeps(deps, newCTSHash string) (newDEPS, oldCTSHash string, err error) { + // Collect old CTS hashes, and replace these with newCTSHash + b := strings.Builder{} + oldCTSHashes := []string{} + matches := reCTSHash.FindAllStringIndex(deps, -1) + if len(matches) == 0 { + return "", "", fmt.Errorf("failed to find a CTS hash in DEPS file") + } + end := 0 + for _, match := range matches { + oldCTSHashes = append(oldCTSHashes, deps[match[0]+len(ctsHashPrefix):match[1]]) + b.WriteString(deps[end:match[0]]) + b.WriteString(ctsHashPrefix + newCTSHash) + end = match[1] + } + b.WriteString(deps[end:]) + + newDEPS = b.String() + if deps == newDEPS { + fmt.Println("CTS is already up to date") + return "", "", nil + } + + if s := container.NewSet(oldCTSHashes...); len(s) > 1 { + fmt.Println("DEPS contained multiple hashes for CTS, using first for logs") + } + oldCTSHash = oldCTSHashes[0] + + return newDEPS, oldCTSHash, nil +} diff --git a/tools/src/cmd/cts/export/export.go b/tools/src/cmd/cts/export/export.go new file mode 100644 index 0000000000..2e5826ec15 --- /dev/null +++ b/tools/src/cmd/cts/export/export.go @@ -0,0 +1,307 @@ +// Copyright 2022 The Dawn Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package export + +import ( + "context" + "flag" + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "dawn.googlesource.com/dawn/tools/src/buildbucket" + "dawn.googlesource.com/dawn/tools/src/cmd/cts/common" + "dawn.googlesource.com/dawn/tools/src/cts/result" + "dawn.googlesource.com/dawn/tools/src/gerrit" + "dawn.googlesource.com/dawn/tools/src/git" + "dawn.googlesource.com/dawn/tools/src/gitiles" + "dawn.googlesource.com/dawn/tools/src/resultsdb" + "dawn.googlesource.com/dawn/tools/src/utils" + "go.chromium.org/luci/auth/client/authcli" + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + "google.golang.org/api/sheets/v4" +) + +func init() { + common.Register(&cmd{}) +} + +type cmd struct { + flags struct { + auth authcli.Flags + cacheDir string + } +} + +func (cmd) Name() string { + return "export" +} + +func (cmd) Desc() string { + return "exports the latest CTS results to Google sheets" +} + +func (c *cmd) RegisterFlags(ctx context.Context, cfg common.Config) ([]string, error) { + c.flags.auth.Register(flag.CommandLine, common.DefaultAuthOptions()) + flag.StringVar(&c.flags.cacheDir, "cache", common.DefaultCacheDir, "path to the results cache") + return nil, nil +} + +func (c *cmd) Run(ctx context.Context, cfg common.Config) error { + // Validate command line arguments + auth, err := c.flags.auth.Options() + if err != nil { + return fmt.Errorf("failed to obtain authentication options: %w", err) + } + + // Load the credentials used for accessing the sheets document + authdir := utils.ExpandHome(os.ExpandEnv(auth.SecretsDir)) + credentialsPath := filepath.Join(authdir, "credentials.json") + b, err := ioutil.ReadFile(credentialsPath) + if err != nil { + return fmt.Errorf("unable to read credentials file '%v'\n"+ + "Obtain this file from: https://console.developers.google.com/apis/credentials\n%w", + credentialsPath, err) + } + credentials, err := google.CredentialsFromJSON(ctx, b, "https://www.googleapis.com/auth/spreadsheets") + if err != nil { + return fmt.Errorf("unable to parse client secret file to config: %w", err) + } + + // Create the sheets service client + s, err := sheets.New(oauth2.NewClient(ctx, credentials.TokenSource)) + if err != nil { + return fmt.Errorf("unable to create sheets client: %w", err) + } + + // Get the CTS statistics spreadsheet + spreadsheet, err := s.Spreadsheets.Get(cfg.Sheets.ID).Do() + if err != nil { + return fmt.Errorf("failed to get spreadsheet: %w", err) + } + + // Scan the sheets of the spreadsheet (tabs at the bottom) for the 'data' + // sheet. + var dataSheet *sheets.Sheet + for _, sheet := range spreadsheet.Sheets { + if strings.ToLower(sheet.Properties.Title) == "data" { + dataSheet = sheet + break + } + } + if dataSheet == nil { + return fmt.Errorf("failed to find data sheet") + } + + // Fetch the table column names + columns, err := fetchRow[string](s, spreadsheet, dataSheet, 0) + + // Create a gerrit client, and find the latest CTS roll + gerrit, err := gerrit.New(cfg.Gerrit.Host, gerrit.Credentials{}) + if err != nil { + return err + } + latestRoll, err := common.LatestCTSRoll(gerrit) + if err != nil { + return err + } + + // Grab the results from the latest CTS roll + bb, err := buildbucket.New(ctx, auth) + if err != nil { + return err + } + rdb, err := resultsdb.New(ctx, auth) + if err != nil { + return err + } + results, ps, err := common.MostRecentResultsForChange(ctx, cfg, c.flags.cacheDir, gerrit, bb, rdb, latestRoll.Number) + if err != nil { + return err + } + if len(results) == 0 { + return fmt.Errorf("no results found") + } + + // Find the CTS revision from the latest CTS roll + dawn, err := gitiles.New(ctx, cfg.Git.Dawn.Host, cfg.Git.Dawn.Project) + if err != nil { + return fmt.Errorf("failed to open dawn host: %w", err) + } + deps, err := dawn.DownloadFile(ctx, ps.RefsChanges(), "DEPS") + if err != nil { + return fmt.Errorf("failed to download DEPS from %v: %w", ps.RefsChanges(), err) + } + _, ctsHash, err := common.UpdateCTSHashInDeps(deps, "") + if err != nil { + return fmt.Errorf("failed to find CTS hash in deps: %w", err) + } + + // Grab the CTS revision to count the number of unimplemented tests + numUnimplemented, err := countUnimplementedTests(cfg, ctsHash) + if err != nil { + return fmt.Errorf("failed to obtain number of unimplemented tests: %w", err) + } + + // Generate a new set of counts of test by status + log.Printf("exporting results from cl %v ps %v...", ps.Change, ps.Patchset) + counts := map[result.Status]int{} + for _, r := range results { + counts[r.Status] = counts[r.Status] + 1 + } + + // Generate new cell data based on the table column names + data := []any{} + for _, column := range columns { + switch strings.ToLower(column) { + case "date": + data = append(data, time.Now().UTC().Format("2006-01-02")) + case "change": + data = append(data, ps.Change) + case "unimplemented": + data = append(data, numUnimplemented) + default: + count, ok := counts[result.Status(column)] + if !ok { + log.Println("no results with status", column) + } + data = append(data, count) + } + } + + // Add a new row to the spreadsheet + _, err = s.Spreadsheets.Values.Append(spreadsheet.SpreadsheetId, "Data", &sheets.ValueRange{ + Values: [][]any{data}, + }).ValueInputOption("RAW").Do() + if err != nil { + return fmt.Errorf("failed to update spreadsheet: %v", err) + } + + return nil +} + +// rowRange returns a sheets range ("name!Ai:i") for the entire row with the +// given index. +func rowRange(index int, sheet *sheets.Sheet) string { + return fmt.Sprintf("%v!A%v:%v", sheet.Properties.Title, index+1, index+1) +} + +// columnRange returns a sheets range ("name!i1:i") for the entire column with +// the given index. +func columnRange(index int, sheet *sheets.Sheet) string { + col := 'A' + index + if index > 25 { + panic("UNIMPLEMENTED") + } + return fmt.Sprintf("%v!%c1:%c", sheet.Properties.Title, col, col) +} + +// fetchRow returns all the values in the given sheet's row. +func fetchRow[T any](srv *sheets.Service, spreadsheet *sheets.Spreadsheet, sheet *sheets.Sheet, row int) ([]T, error) { + rng := rowRange(row, sheet) + data, err := srv.Spreadsheets.Values.Get(spreadsheet.SpreadsheetId, rng).Do() + if err != nil { + return nil, fmt.Errorf("Couldn't fetch %v: %w", rng, err) + } + out := make([]T, len(data.Values[0])) + for column, v := range data.Values[0] { + val, ok := v.(T) + if !ok { + return nil, fmt.Errorf("cell at %v:%v was type %T, but expected type %T", row, column, v, val) + } + out[column] = val + } + return out, nil +} + +// countUnimplementedTests checks out the WebGPU CTS at ctsHash, builds the node +// command line tool, and runs it with '--list-unimplemented webgpu:*' to count +// the total number of unimplemented tests, which is returned. +func countUnimplementedTests(cfg common.Config, ctsHash string) (int, error) { + tmpDir, err := os.MkdirTemp("", "dawn-cts-export") + if err != nil { + return 0, err + } + defer os.RemoveAll(tmpDir) + + dir := filepath.Join(tmpDir, "cts") + + gitExe, err := exec.LookPath("git") + if err != nil { + return 0, fmt.Errorf("failed to find git on PATH: %w", err) + } + + git, err := git.New(gitExe) + if err != nil { + return 0, err + } + + log.Printf("cloning cts to '%v'...", dir) + repo, err := git.Clone(dir, cfg.Git.CTS.HttpsURL(), nil) + if err != nil { + return 0, fmt.Errorf("failed to clone cts: %v", err) + } + + log.Printf("checking out cts @ '%v'...", ctsHash) + if _, err := repo.Fetch(ctsHash, nil); err != nil { + return 0, fmt.Errorf("failed to fetch cts: %v", err) + } + if err := repo.Checkout(ctsHash, nil); err != nil { + return 0, fmt.Errorf("failed to clone cts: %v", err) + } + + { + npm, err := exec.LookPath("npm") + if err != nil { + return 0, fmt.Errorf("failed to find npm on PATH: %w", err) + } + cmd := exec.Command(npm, "ci") + cmd.Dir = dir + if out, err := cmd.CombinedOutput(); err != nil { + return 0, fmt.Errorf("failed to run npm ci: %w\n%v", err, string(out)) + } + } + { + npx, err := exec.LookPath("npx") + if err != nil { + return 0, fmt.Errorf("failed to find npx on PATH: %w", err) + } + cmd := exec.Command(npx, "grunt", "run:build-out-node") + cmd.Dir = dir + if out, err := cmd.CombinedOutput(); err != nil { + return 0, fmt.Errorf("failed to build CTS typescript: %w\n%v", err, string(out)) + } + } + { + sh, err := exec.LookPath("node") + if err != nil { + return 0, fmt.Errorf("failed to find sh on PATH: %w", err) + } + cmd := exec.Command(sh, "./tools/run_node", "--list-unimplemented", "webgpu:*") + cmd.Dir = dir + out, err := cmd.CombinedOutput() + if err != nil { + return 0, fmt.Errorf("failed to gather unimplemented tests: %w", err) + } + lines := strings.Split(string(out), "\n") + return len(lines), nil + } +} diff --git a/tools/src/cmd/cts/main.go b/tools/src/cmd/cts/main.go index 0aefd2929a..a44b15dcde 100644 --- a/tools/src/cmd/cts/main.go +++ b/tools/src/cmd/cts/main.go @@ -28,6 +28,7 @@ import ( "dawn.googlesource.com/dawn/tools/src/utils" // Register sub-commands + _ "dawn.googlesource.com/dawn/tools/src/cmd/cts/export" _ "dawn.googlesource.com/dawn/tools/src/cmd/cts/format" _ "dawn.googlesource.com/dawn/tools/src/cmd/cts/merge" _ "dawn.googlesource.com/dawn/tools/src/cmd/cts/results"