tools: Add the 'cts export' sub-command

Updates a Google spreadsheet with latest CTS statistics.

Bug: dawn:1342
Change-Id: I01a9c99032c755d6a78af674675d8c9cbfc79019
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/88443
Reviewed-by: Dan Sinclair <dsinclair@chromium.org>
Commit-Queue: Ben Clayton <bclayton@google.com>
Kokoro: Kokoro <noreply+kokoro@google.com>
This commit is contained in:
Ben Clayton 2022-05-02 20:44:21 +00:00
parent cf4a85e249
commit d9f2a61c6f
5 changed files with 387 additions and 2 deletions

7
go.mod
View File

@ -14,15 +14,19 @@ require (
github.com/tidwall/jsonc v0.3.2
go.chromium.org/luci v0.0.0-20220412023008-ab2409fe739a
golang.org/x/net v0.0.0-20220403103023-749bd193bc2b
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8
google.golang.org/api v0.63.0
google.golang.org/protobuf v1.28.0
)
require (
cloud.google.com/go v0.99.0 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/mock v1.6.0 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/googleapis/gax-go/v2 v2.1.1 // indirect
github.com/julienschmidt/httprouter v1.3.0 // indirect
github.com/klauspost/compress v1.13.5 // indirect
github.com/kr/pretty v0.3.0 // indirect
@ -34,11 +38,10 @@ require (
github.com/tklauser/go-sysconf v0.3.10 // indirect
github.com/tklauser/numcpus v0.4.0 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
go.opencensus.io v0.23.0 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27 // indirect
golang.org/x/text v0.3.7 // indirect
google.golang.org/api v0.63.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220216160803-4663080d8bc8 // indirect
google.golang.org/grpc v1.44.0 // indirect

3
go.sum
View File

@ -174,6 +174,7 @@ github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4er
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@ -257,6 +258,7 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
github.com/googleapis/gax-go/v2 v2.1.1 h1:dp3bWCh+PPO1zjRRiCSczJav13sBvG4UhNyVTa1KqdU=
github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gopherjs/gopherjs v0.0.0-20210901121439-eee08aaf2717 h1:V1j4G8AXIJeyzT3ng2Oh4IRo/VEgRWYAsyYwhOz5rko=
@ -490,6 +492,7 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
go.opencensus.io v0.22.6/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M=
go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
go.starlark.net v0.0.0-20210223155950-e043a3d3c984/go.mod h1:t3mmBBPzAVvK0L0n1drDmrQsJ8FoIx4INCqVMTr/Zo0=

View File

@ -0,0 +1,71 @@
// Copyright 2022 The Dawn Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package common
import (
"fmt"
"regexp"
"strings"
"dawn.googlesource.com/dawn/tools/src/container"
)
// The regular expression used to search for the CTS hash
var reCTSHash = regexp.MustCompile(reEscape(ctsHashPrefix) + `[0-9a-fA-F]+`)
const (
// The string prefix for the CTS hash in the DEPs file, used for identifying
// and updating the DEPS file.
ctsHashPrefix = `{chromium_git}/external/github.com/gpuweb/cts@`
)
func reEscape(s string) string {
return strings.ReplaceAll(strings.ReplaceAll(s, `/`, `\/`), `.`, `\.`)
}
// UpdateCTSHashInDeps replaces the CTS hashes in 'deps' with 'newCTSHash'.
// Returns:
// newDEPS - the new DEPS content
// oldCTSHash - the old CTS hash in the 'deps'
func UpdateCTSHashInDeps(deps, newCTSHash string) (newDEPS, oldCTSHash string, err error) {
// Collect old CTS hashes, and replace these with newCTSHash
b := strings.Builder{}
oldCTSHashes := []string{}
matches := reCTSHash.FindAllStringIndex(deps, -1)
if len(matches) == 0 {
return "", "", fmt.Errorf("failed to find a CTS hash in DEPS file")
}
end := 0
for _, match := range matches {
oldCTSHashes = append(oldCTSHashes, deps[match[0]+len(ctsHashPrefix):match[1]])
b.WriteString(deps[end:match[0]])
b.WriteString(ctsHashPrefix + newCTSHash)
end = match[1]
}
b.WriteString(deps[end:])
newDEPS = b.String()
if deps == newDEPS {
fmt.Println("CTS is already up to date")
return "", "", nil
}
if s := container.NewSet(oldCTSHashes...); len(s) > 1 {
fmt.Println("DEPS contained multiple hashes for CTS, using first for logs")
}
oldCTSHash = oldCTSHashes[0]
return newDEPS, oldCTSHash, nil
}

View File

@ -0,0 +1,307 @@
// Copyright 2022 The Dawn Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package export
import (
"context"
"flag"
"fmt"
"io/ioutil"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"dawn.googlesource.com/dawn/tools/src/buildbucket"
"dawn.googlesource.com/dawn/tools/src/cmd/cts/common"
"dawn.googlesource.com/dawn/tools/src/cts/result"
"dawn.googlesource.com/dawn/tools/src/gerrit"
"dawn.googlesource.com/dawn/tools/src/git"
"dawn.googlesource.com/dawn/tools/src/gitiles"
"dawn.googlesource.com/dawn/tools/src/resultsdb"
"dawn.googlesource.com/dawn/tools/src/utils"
"go.chromium.org/luci/auth/client/authcli"
"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
"google.golang.org/api/sheets/v4"
)
func init() {
common.Register(&cmd{})
}
type cmd struct {
flags struct {
auth authcli.Flags
cacheDir string
}
}
func (cmd) Name() string {
return "export"
}
func (cmd) Desc() string {
return "exports the latest CTS results to Google sheets"
}
func (c *cmd) RegisterFlags(ctx context.Context, cfg common.Config) ([]string, error) {
c.flags.auth.Register(flag.CommandLine, common.DefaultAuthOptions())
flag.StringVar(&c.flags.cacheDir, "cache", common.DefaultCacheDir, "path to the results cache")
return nil, nil
}
func (c *cmd) Run(ctx context.Context, cfg common.Config) error {
// Validate command line arguments
auth, err := c.flags.auth.Options()
if err != nil {
return fmt.Errorf("failed to obtain authentication options: %w", err)
}
// Load the credentials used for accessing the sheets document
authdir := utils.ExpandHome(os.ExpandEnv(auth.SecretsDir))
credentialsPath := filepath.Join(authdir, "credentials.json")
b, err := ioutil.ReadFile(credentialsPath)
if err != nil {
return fmt.Errorf("unable to read credentials file '%v'\n"+
"Obtain this file from: https://console.developers.google.com/apis/credentials\n%w",
credentialsPath, err)
}
credentials, err := google.CredentialsFromJSON(ctx, b, "https://www.googleapis.com/auth/spreadsheets")
if err != nil {
return fmt.Errorf("unable to parse client secret file to config: %w", err)
}
// Create the sheets service client
s, err := sheets.New(oauth2.NewClient(ctx, credentials.TokenSource))
if err != nil {
return fmt.Errorf("unable to create sheets client: %w", err)
}
// Get the CTS statistics spreadsheet
spreadsheet, err := s.Spreadsheets.Get(cfg.Sheets.ID).Do()
if err != nil {
return fmt.Errorf("failed to get spreadsheet: %w", err)
}
// Scan the sheets of the spreadsheet (tabs at the bottom) for the 'data'
// sheet.
var dataSheet *sheets.Sheet
for _, sheet := range spreadsheet.Sheets {
if strings.ToLower(sheet.Properties.Title) == "data" {
dataSheet = sheet
break
}
}
if dataSheet == nil {
return fmt.Errorf("failed to find data sheet")
}
// Fetch the table column names
columns, err := fetchRow[string](s, spreadsheet, dataSheet, 0)
// Create a gerrit client, and find the latest CTS roll
gerrit, err := gerrit.New(cfg.Gerrit.Host, gerrit.Credentials{})
if err != nil {
return err
}
latestRoll, err := common.LatestCTSRoll(gerrit)
if err != nil {
return err
}
// Grab the results from the latest CTS roll
bb, err := buildbucket.New(ctx, auth)
if err != nil {
return err
}
rdb, err := resultsdb.New(ctx, auth)
if err != nil {
return err
}
results, ps, err := common.MostRecentResultsForChange(ctx, cfg, c.flags.cacheDir, gerrit, bb, rdb, latestRoll.Number)
if err != nil {
return err
}
if len(results) == 0 {
return fmt.Errorf("no results found")
}
// Find the CTS revision from the latest CTS roll
dawn, err := gitiles.New(ctx, cfg.Git.Dawn.Host, cfg.Git.Dawn.Project)
if err != nil {
return fmt.Errorf("failed to open dawn host: %w", err)
}
deps, err := dawn.DownloadFile(ctx, ps.RefsChanges(), "DEPS")
if err != nil {
return fmt.Errorf("failed to download DEPS from %v: %w", ps.RefsChanges(), err)
}
_, ctsHash, err := common.UpdateCTSHashInDeps(deps, "<unused>")
if err != nil {
return fmt.Errorf("failed to find CTS hash in deps: %w", err)
}
// Grab the CTS revision to count the number of unimplemented tests
numUnimplemented, err := countUnimplementedTests(cfg, ctsHash)
if err != nil {
return fmt.Errorf("failed to obtain number of unimplemented tests: %w", err)
}
// Generate a new set of counts of test by status
log.Printf("exporting results from cl %v ps %v...", ps.Change, ps.Patchset)
counts := map[result.Status]int{}
for _, r := range results {
counts[r.Status] = counts[r.Status] + 1
}
// Generate new cell data based on the table column names
data := []any{}
for _, column := range columns {
switch strings.ToLower(column) {
case "date":
data = append(data, time.Now().UTC().Format("2006-01-02"))
case "change":
data = append(data, ps.Change)
case "unimplemented":
data = append(data, numUnimplemented)
default:
count, ok := counts[result.Status(column)]
if !ok {
log.Println("no results with status", column)
}
data = append(data, count)
}
}
// Add a new row to the spreadsheet
_, err = s.Spreadsheets.Values.Append(spreadsheet.SpreadsheetId, "Data", &sheets.ValueRange{
Values: [][]any{data},
}).ValueInputOption("RAW").Do()
if err != nil {
return fmt.Errorf("failed to update spreadsheet: %v", err)
}
return nil
}
// rowRange returns a sheets range ("name!Ai:i") for the entire row with the
// given index.
func rowRange(index int, sheet *sheets.Sheet) string {
return fmt.Sprintf("%v!A%v:%v", sheet.Properties.Title, index+1, index+1)
}
// columnRange returns a sheets range ("name!i1:i") for the entire column with
// the given index.
func columnRange(index int, sheet *sheets.Sheet) string {
col := 'A' + index
if index > 25 {
panic("UNIMPLEMENTED")
}
return fmt.Sprintf("%v!%c1:%c", sheet.Properties.Title, col, col)
}
// fetchRow returns all the values in the given sheet's row.
func fetchRow[T any](srv *sheets.Service, spreadsheet *sheets.Spreadsheet, sheet *sheets.Sheet, row int) ([]T, error) {
rng := rowRange(row, sheet)
data, err := srv.Spreadsheets.Values.Get(spreadsheet.SpreadsheetId, rng).Do()
if err != nil {
return nil, fmt.Errorf("Couldn't fetch %v: %w", rng, err)
}
out := make([]T, len(data.Values[0]))
for column, v := range data.Values[0] {
val, ok := v.(T)
if !ok {
return nil, fmt.Errorf("cell at %v:%v was type %T, but expected type %T", row, column, v, val)
}
out[column] = val
}
return out, nil
}
// countUnimplementedTests checks out the WebGPU CTS at ctsHash, builds the node
// command line tool, and runs it with '--list-unimplemented webgpu:*' to count
// the total number of unimplemented tests, which is returned.
func countUnimplementedTests(cfg common.Config, ctsHash string) (int, error) {
tmpDir, err := os.MkdirTemp("", "dawn-cts-export")
if err != nil {
return 0, err
}
defer os.RemoveAll(tmpDir)
dir := filepath.Join(tmpDir, "cts")
gitExe, err := exec.LookPath("git")
if err != nil {
return 0, fmt.Errorf("failed to find git on PATH: %w", err)
}
git, err := git.New(gitExe)
if err != nil {
return 0, err
}
log.Printf("cloning cts to '%v'...", dir)
repo, err := git.Clone(dir, cfg.Git.CTS.HttpsURL(), nil)
if err != nil {
return 0, fmt.Errorf("failed to clone cts: %v", err)
}
log.Printf("checking out cts @ '%v'...", ctsHash)
if _, err := repo.Fetch(ctsHash, nil); err != nil {
return 0, fmt.Errorf("failed to fetch cts: %v", err)
}
if err := repo.Checkout(ctsHash, nil); err != nil {
return 0, fmt.Errorf("failed to clone cts: %v", err)
}
{
npm, err := exec.LookPath("npm")
if err != nil {
return 0, fmt.Errorf("failed to find npm on PATH: %w", err)
}
cmd := exec.Command(npm, "ci")
cmd.Dir = dir
if out, err := cmd.CombinedOutput(); err != nil {
return 0, fmt.Errorf("failed to run npm ci: %w\n%v", err, string(out))
}
}
{
npx, err := exec.LookPath("npx")
if err != nil {
return 0, fmt.Errorf("failed to find npx on PATH: %w", err)
}
cmd := exec.Command(npx, "grunt", "run:build-out-node")
cmd.Dir = dir
if out, err := cmd.CombinedOutput(); err != nil {
return 0, fmt.Errorf("failed to build CTS typescript: %w\n%v", err, string(out))
}
}
{
sh, err := exec.LookPath("node")
if err != nil {
return 0, fmt.Errorf("failed to find sh on PATH: %w", err)
}
cmd := exec.Command(sh, "./tools/run_node", "--list-unimplemented", "webgpu:*")
cmd.Dir = dir
out, err := cmd.CombinedOutput()
if err != nil {
return 0, fmt.Errorf("failed to gather unimplemented tests: %w", err)
}
lines := strings.Split(string(out), "\n")
return len(lines), nil
}
}

View File

@ -28,6 +28,7 @@ import (
"dawn.googlesource.com/dawn/tools/src/utils"
// Register sub-commands
_ "dawn.googlesource.com/dawn/tools/src/cmd/cts/export"
_ "dawn.googlesource.com/dawn/tools/src/cmd/cts/format"
_ "dawn.googlesource.com/dawn/tools/src/cmd/cts/merge"
_ "dawn.googlesource.com/dawn/tools/src/cmd/cts/results"