tools: Add the 'cts export' sub-command

Updates a Google spreadsheet with latest CTS statistics.

Bug: dawn:1342
Change-Id: I01a9c99032c755d6a78af674675d8c9cbfc79019
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/88443
Reviewed-by: Dan Sinclair <dsinclair@chromium.org>
Commit-Queue: Ben Clayton <bclayton@google.com>
Kokoro: Kokoro <noreply+kokoro@google.com>

go.mod

@@ -14,15 +14,19 @@ require (
 	github.com/tidwall/jsonc v0.3.2
 	go.chromium.org/luci v0.0.0-20220412023008-ab2409fe739a
 	golang.org/x/net v0.0.0-20220403103023-749bd193bc2b
+	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8
+	google.golang.org/api v0.63.0
 	google.golang.org/protobuf v1.28.0
 )
 
 require (
 	cloud.google.com/go v0.99.0 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/mock v1.6.0 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.1.1 // indirect
 	github.com/julienschmidt/httprouter v1.3.0 // indirect
 	github.com/klauspost/compress v1.13.5 // indirect
 	github.com/kr/pretty v0.3.0 // indirect
@@ -34,11 +38,10 @@ require (
 	github.com/tklauser/go-sysconf v0.3.10 // indirect
 	github.com/tklauser/numcpus v0.4.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
-	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
+	go.opencensus.io v0.23.0 // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
 	golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27 // indirect
 	golang.org/x/text v0.3.7 // indirect
-	google.golang.org/api v0.63.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20220216160803-4663080d8bc8 // indirect
 	google.golang.org/grpc v1.44.0 // indirect

go.sum

@@ -174,6 +174,7 @@ github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4er
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -257,6 +258,7 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
+github.com/googleapis/gax-go/v2 v2.1.1 h1:dp3bWCh+PPO1zjRRiCSczJav13sBvG4UhNyVTa1KqdU=
 github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gopherjs/gopherjs v0.0.0-20210901121439-eee08aaf2717 h1:V1j4G8AXIJeyzT3ng2Oh4IRo/VEgRWYAsyYwhOz5rko=
@@ -490,6 +492,7 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 go.opencensus.io v0.22.6/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.starlark.net v0.0.0-20210223155950-e043a3d3c984/go.mod h1:t3mmBBPzAVvK0L0n1drDmrQsJ8FoIx4INCqVMTr/Zo0=

tools/src/cmd/cts/common/deps.go

@@ -0,0 +1,71 @@
+// Copyright 2022 The Dawn Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+
+	"dawn.googlesource.com/dawn/tools/src/container"
+)
+
+// The regular expression used to search for the CTS hash
+var reCTSHash = regexp.MustCompile(reEscape(ctsHashPrefix) + `[0-9a-fA-F]+`)
+
+const (
+	// The string prefix for the CTS hash in the DEPs file, used for identifying
+	// and updating the DEPS file.
+	ctsHashPrefix = `{chromium_git}/external/github.com/gpuweb/cts@`
+)
+
+func reEscape(s string) string {
+	return strings.ReplaceAll(strings.ReplaceAll(s, `/`, `\/`), `.`, `\.`)
+}
+
+// UpdateCTSHashInDeps replaces the CTS hashes in 'deps' with 'newCTSHash'.
+// Returns:
+//  newDEPS    - the new DEPS content
+//  oldCTSHash - the old CTS hash in the 'deps'
+func UpdateCTSHashInDeps(deps, newCTSHash string) (newDEPS, oldCTSHash string, err error) {
+	// Collect old CTS hashes, and replace these with newCTSHash
+	b := strings.Builder{}
+	oldCTSHashes := []string{}
+	matches := reCTSHash.FindAllStringIndex(deps, -1)
+	if len(matches) == 0 {
+		return "", "", fmt.Errorf("failed to find a CTS hash in DEPS file")
+	}
+	end := 0
+	for _, match := range matches {
+		oldCTSHashes = append(oldCTSHashes, deps[match[0]+len(ctsHashPrefix):match[1]])
+		b.WriteString(deps[end:match[0]])
+		b.WriteString(ctsHashPrefix + newCTSHash)
+		end = match[1]
+	}
+	b.WriteString(deps[end:])
+
+	newDEPS = b.String()
+	if deps == newDEPS {
+		fmt.Println("CTS is already up to date")
+		return "", "", nil
+	}
+
+	if s := container.NewSet(oldCTSHashes...); len(s) > 1 {
+		fmt.Println("DEPS contained multiple hashes for CTS, using first for logs")
+	}
+	oldCTSHash = oldCTSHashes[0]
+
+	return newDEPS, oldCTSHash, nil
+}

tools/src/cmd/cts/export/export.go

@@ -0,0 +1,307 @@
+// Copyright 2022 The Dawn Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package export
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"dawn.googlesource.com/dawn/tools/src/buildbucket"
+	"dawn.googlesource.com/dawn/tools/src/cmd/cts/common"
+	"dawn.googlesource.com/dawn/tools/src/cts/result"
+	"dawn.googlesource.com/dawn/tools/src/gerrit"
+	"dawn.googlesource.com/dawn/tools/src/git"
+	"dawn.googlesource.com/dawn/tools/src/gitiles"
+	"dawn.googlesource.com/dawn/tools/src/resultsdb"
+	"dawn.googlesource.com/dawn/tools/src/utils"
+	"go.chromium.org/luci/auth/client/authcli"
+	"golang.org/x/oauth2"
+	"golang.org/x/oauth2/google"
+	"google.golang.org/api/sheets/v4"
+)
+
+func init() {
+	common.Register(&cmd{})
+}
+
+type cmd struct {
+	flags struct {
+		auth     authcli.Flags
+		cacheDir string
+	}
+}
+
+func (cmd) Name() string {
+	return "export"
+}
+
+func (cmd) Desc() string {
+	return "exports the latest CTS results to Google sheets"
+}
+
+func (c *cmd) RegisterFlags(ctx context.Context, cfg common.Config) ([]string, error) {
+	c.flags.auth.Register(flag.CommandLine, common.DefaultAuthOptions())
+	flag.StringVar(&c.flags.cacheDir, "cache", common.DefaultCacheDir, "path to the results cache")
+	return nil, nil
+}
+
+func (c *cmd) Run(ctx context.Context, cfg common.Config) error {
+	// Validate command line arguments
+	auth, err := c.flags.auth.Options()
+	if err != nil {
+		return fmt.Errorf("failed to obtain authentication options: %w", err)
+	}
+
+	// Load the credentials used for accessing the sheets document
+	authdir := utils.ExpandHome(os.ExpandEnv(auth.SecretsDir))
+	credentialsPath := filepath.Join(authdir, "credentials.json")
+	b, err := ioutil.ReadFile(credentialsPath)
+	if err != nil {
+		return fmt.Errorf("unable to read credentials file '%v'\n"+
+			"Obtain this file from: https://console.developers.google.com/apis/credentials\n%w",
+			credentialsPath, err)
+	}
+	credentials, err := google.CredentialsFromJSON(ctx, b, "https://www.googleapis.com/auth/spreadsheets")
+	if err != nil {
+		return fmt.Errorf("unable to parse client secret file to config: %w", err)
+	}
+
+	// Create the sheets service client
+	s, err := sheets.New(oauth2.NewClient(ctx, credentials.TokenSource))
+	if err != nil {
+		return fmt.Errorf("unable to create sheets client: %w", err)
+	}
+
+	// Get the CTS statistics spreadsheet
+	spreadsheet, err := s.Spreadsheets.Get(cfg.Sheets.ID).Do()
+	if err != nil {
+		return fmt.Errorf("failed to get spreadsheet: %w", err)
+	}
+
+	// Scan the sheets of the spreadsheet (tabs at the bottom) for the 'data'
+	// sheet.
+	var dataSheet *sheets.Sheet
+	for _, sheet := range spreadsheet.Sheets {
+		if strings.ToLower(sheet.Properties.Title) == "data" {
+			dataSheet = sheet
+			break
+		}
+	}
+	if dataSheet == nil {
+		return fmt.Errorf("failed to find data sheet")
+	}
+
+	// Fetch the table column names
+	columns, err := fetchRow[string](s, spreadsheet, dataSheet, 0)
+
+	// Create a gerrit client, and find the latest CTS roll
+	gerrit, err := gerrit.New(cfg.Gerrit.Host, gerrit.Credentials{})
+	if err != nil {
+		return err
+	}
+	latestRoll, err := common.LatestCTSRoll(gerrit)
+	if err != nil {
+		return err
+	}
+
+	// Grab the results from the latest CTS roll
+	bb, err := buildbucket.New(ctx, auth)
+	if err != nil {
+		return err
+	}
+	rdb, err := resultsdb.New(ctx, auth)
+	if err != nil {
+		return err
+	}
+	results, ps, err := common.MostRecentResultsForChange(ctx, cfg, c.flags.cacheDir, gerrit, bb, rdb, latestRoll.Number)
+	if err != nil {
+		return err
+	}
+	if len(results) == 0 {
+		return fmt.Errorf("no results found")
+	}
+
+	// Find the CTS revision from the latest CTS roll
+	dawn, err := gitiles.New(ctx, cfg.Git.Dawn.Host, cfg.Git.Dawn.Project)
+	if err != nil {
+		return fmt.Errorf("failed to open dawn host: %w", err)
+	}
+	deps, err := dawn.DownloadFile(ctx, ps.RefsChanges(), "DEPS")
+	if err != nil {
+		return fmt.Errorf("failed to download DEPS from %v: %w", ps.RefsChanges(), err)
+	}
+	_, ctsHash, err := common.UpdateCTSHashInDeps(deps, "<unused>")
+	if err != nil {
+		return fmt.Errorf("failed to find CTS hash in deps: %w", err)
+	}
+
+	// Grab the CTS revision to count the number of unimplemented tests
+	numUnimplemented, err := countUnimplementedTests(cfg, ctsHash)
+	if err != nil {
+		return fmt.Errorf("failed to obtain number of unimplemented tests: %w", err)
+	}
+
+	// Generate a new set of counts of test by status
+	log.Printf("exporting results from cl %v ps %v...", ps.Change, ps.Patchset)
+	counts := map[result.Status]int{}
+	for _, r := range results {
+		counts[r.Status] = counts[r.Status] + 1
+	}
+
+	// Generate new cell data based on the table column names
+	data := []any{}
+	for _, column := range columns {
+		switch strings.ToLower(column) {
+		case "date":
+			data = append(data, time.Now().UTC().Format("2006-01-02"))
+		case "change":
+			data = append(data, ps.Change)
+		case "unimplemented":
+			data = append(data, numUnimplemented)
+		default:
+			count, ok := counts[result.Status(column)]
+			if !ok {
+				log.Println("no results with status", column)
+			}
+			data = append(data, count)
+		}
+	}
+
+	// Add a new row to the spreadsheet
+	_, err = s.Spreadsheets.Values.Append(spreadsheet.SpreadsheetId, "Data", &sheets.ValueRange{
+		Values: [][]any{data},
+	}).ValueInputOption("RAW").Do()
+	if err != nil {
+		return fmt.Errorf("failed to update spreadsheet: %v", err)
+	}
+
+	return nil
+}
+
+// rowRange returns a sheets range ("name!Ai:i") for the entire row with the
+// given index.
+func rowRange(index int, sheet *sheets.Sheet) string {
+	return fmt.Sprintf("%v!A%v:%v", sheet.Properties.Title, index+1, index+1)
+}
+
+// columnRange returns a sheets range ("name!i1:i") for the entire column with
+// the given index.
+func columnRange(index int, sheet *sheets.Sheet) string {
+	col := 'A' + index
+	if index > 25 {
+		panic("UNIMPLEMENTED")
+	}
+	return fmt.Sprintf("%v!%c1:%c", sheet.Properties.Title, col, col)
+}
+
+// fetchRow returns all the values in the given sheet's row.
+func fetchRow[T any](srv *sheets.Service, spreadsheet *sheets.Spreadsheet, sheet *sheets.Sheet, row int) ([]T, error) {
+	rng := rowRange(row, sheet)
+	data, err := srv.Spreadsheets.Values.Get(spreadsheet.SpreadsheetId, rng).Do()
+	if err != nil {
+		return nil, fmt.Errorf("Couldn't fetch %v: %w", rng, err)
+	}
+	out := make([]T, len(data.Values[0]))
+	for column, v := range data.Values[0] {
+		val, ok := v.(T)
+		if !ok {
+			return nil, fmt.Errorf("cell at %v:%v was type %T, but expected type %T", row, column, v, val)
+		}
+		out[column] = val
+	}
+	return out, nil
+}
+
+// countUnimplementedTests checks out the WebGPU CTS at ctsHash, builds the node
+// command line tool, and runs it with '--list-unimplemented webgpu:*' to count
+// the total number of unimplemented tests, which is returned.
+func countUnimplementedTests(cfg common.Config, ctsHash string) (int, error) {
+	tmpDir, err := os.MkdirTemp("", "dawn-cts-export")
+	if err != nil {
+		return 0, err
+	}
+	defer os.RemoveAll(tmpDir)
+
+	dir := filepath.Join(tmpDir, "cts")
+
+	gitExe, err := exec.LookPath("git")
+	if err != nil {
+		return 0, fmt.Errorf("failed to find git on PATH: %w", err)
+	}
+
+	git, err := git.New(gitExe)
+	if err != nil {
+		return 0, err
+	}
+
+	log.Printf("cloning cts to '%v'...", dir)
+	repo, err := git.Clone(dir, cfg.Git.CTS.HttpsURL(), nil)
+	if err != nil {
+		return 0, fmt.Errorf("failed to clone cts: %v", err)
+	}
+
+	log.Printf("checking out cts @ '%v'...", ctsHash)
+	if _, err := repo.Fetch(ctsHash, nil); err != nil {
+		return 0, fmt.Errorf("failed to fetch cts: %v", err)
+	}
+	if err := repo.Checkout(ctsHash, nil); err != nil {
+		return 0, fmt.Errorf("failed to clone cts: %v", err)
+	}
+
+	{
+		npm, err := exec.LookPath("npm")
+		if err != nil {
+			return 0, fmt.Errorf("failed to find npm on PATH: %w", err)
+		}
+		cmd := exec.Command(npm, "ci")
+		cmd.Dir = dir
+		if out, err := cmd.CombinedOutput(); err != nil {
+			return 0, fmt.Errorf("failed to run npm ci: %w\n%v", err, string(out))
+		}
+	}
+	{
+		npx, err := exec.LookPath("npx")
+		if err != nil {
+			return 0, fmt.Errorf("failed to find npx on PATH: %w", err)
+		}
+		cmd := exec.Command(npx, "grunt", "run:build-out-node")
+		cmd.Dir = dir
+		if out, err := cmd.CombinedOutput(); err != nil {
+			return 0, fmt.Errorf("failed to build CTS typescript: %w\n%v", err, string(out))
+		}
+	}
+	{
+		sh, err := exec.LookPath("node")
+		if err != nil {
+			return 0, fmt.Errorf("failed to find sh on PATH: %w", err)
+		}
+		cmd := exec.Command(sh, "./tools/run_node", "--list-unimplemented", "webgpu:*")
+		cmd.Dir = dir
+		out, err := cmd.CombinedOutput()
+		if err != nil {
+			return 0, fmt.Errorf("failed to gather unimplemented tests: %w", err)
+		}
+		lines := strings.Split(string(out), "\n")
+		return len(lines), nil
+	}
+}

tools/src/cmd/cts/main.go

@@ -28,6 +28,7 @@ import (
 	"dawn.googlesource.com/dawn/tools/src/utils"
 
 	// Register sub-commands
+	_ "dawn.googlesource.com/dawn/tools/src/cmd/cts/export"
 	_ "dawn.googlesource.com/dawn/tools/src/cmd/cts/format"
 	_ "dawn.googlesource.com/dawn/tools/src/cmd/cts/merge"
 	_ "dawn.googlesource.com/dawn/tools/src/cmd/cts/results"