tools: Add the 'cts export' sub-command

Updates a Google spreadsheet with latest CTS statistics.

Bug: dawn:1342
Change-Id: I01a9c99032c755d6a78af674675d8c9cbfc79019
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/88443
Reviewed-by: Dan Sinclair <dsinclair@chromium.org>
Commit-Queue: Ben Clayton <bclayton@google.com>
Kokoro: Kokoro <noreply+kokoro@google.com>

tools/src/cmd/cts/common/deps.go

@@ -0,0 +1,71 @@
+// Copyright 2022 The Dawn Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+
+	"dawn.googlesource.com/dawn/tools/src/container"
+)
+
+// The regular expression used to search for the CTS hash
+var reCTSHash = regexp.MustCompile(reEscape(ctsHashPrefix) + `[0-9a-fA-F]+`)
+
+const (
+	// The string prefix for the CTS hash in the DEPs file, used for identifying
+	// and updating the DEPS file.
+	ctsHashPrefix = `{chromium_git}/external/github.com/gpuweb/cts@`
+)
+
+func reEscape(s string) string {
+	return strings.ReplaceAll(strings.ReplaceAll(s, `/`, `\/`), `.`, `\.`)
+}
+
+// UpdateCTSHashInDeps replaces the CTS hashes in 'deps' with 'newCTSHash'.
+// Returns:
+//  newDEPS    - the new DEPS content
+//  oldCTSHash - the old CTS hash in the 'deps'
+func UpdateCTSHashInDeps(deps, newCTSHash string) (newDEPS, oldCTSHash string, err error) {
+	// Collect old CTS hashes, and replace these with newCTSHash
+	b := strings.Builder{}
+	oldCTSHashes := []string{}
+	matches := reCTSHash.FindAllStringIndex(deps, -1)
+	if len(matches) == 0 {
+		return "", "", fmt.Errorf("failed to find a CTS hash in DEPS file")
+	}
+	end := 0
+	for _, match := range matches {
+		oldCTSHashes = append(oldCTSHashes, deps[match[0]+len(ctsHashPrefix):match[1]])
+		b.WriteString(deps[end:match[0]])
+		b.WriteString(ctsHashPrefix + newCTSHash)
+		end = match[1]
+	}
+	b.WriteString(deps[end:])
+
+	newDEPS = b.String()
+	if deps == newDEPS {
+		fmt.Println("CTS is already up to date")
+		return "", "", nil
+	}
+
+	if s := container.NewSet(oldCTSHashes...); len(s) > 1 {
+		fmt.Println("DEPS contained multiple hashes for CTS, using first for logs")
+	}
+	oldCTSHash = oldCTSHashes[0]
+
+	return newDEPS, oldCTSHash, nil
+}

tools/src/cmd/cts/export/export.go

@@ -0,0 +1,307 @@
+// Copyright 2022 The Dawn Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package export
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"dawn.googlesource.com/dawn/tools/src/buildbucket"
+	"dawn.googlesource.com/dawn/tools/src/cmd/cts/common"
+	"dawn.googlesource.com/dawn/tools/src/cts/result"
+	"dawn.googlesource.com/dawn/tools/src/gerrit"
+	"dawn.googlesource.com/dawn/tools/src/git"
+	"dawn.googlesource.com/dawn/tools/src/gitiles"
+	"dawn.googlesource.com/dawn/tools/src/resultsdb"
+	"dawn.googlesource.com/dawn/tools/src/utils"
+	"go.chromium.org/luci/auth/client/authcli"
+	"golang.org/x/oauth2"
+	"golang.org/x/oauth2/google"
+	"google.golang.org/api/sheets/v4"
+)
+
+func init() {
+	common.Register(&cmd{})
+}
+
+type cmd struct {
+	flags struct {
+		auth     authcli.Flags
+		cacheDir string
+	}
+}
+
+func (cmd) Name() string {
+	return "export"
+}
+
+func (cmd) Desc() string {
+	return "exports the latest CTS results to Google sheets"
+}
+
+func (c *cmd) RegisterFlags(ctx context.Context, cfg common.Config) ([]string, error) {
+	c.flags.auth.Register(flag.CommandLine, common.DefaultAuthOptions())
+	flag.StringVar(&c.flags.cacheDir, "cache", common.DefaultCacheDir, "path to the results cache")
+	return nil, nil
+}
+
+func (c *cmd) Run(ctx context.Context, cfg common.Config) error {
+	// Validate command line arguments
+	auth, err := c.flags.auth.Options()
+	if err != nil {
+		return fmt.Errorf("failed to obtain authentication options: %w", err)
+	}
+
+	// Load the credentials used for accessing the sheets document
+	authdir := utils.ExpandHome(os.ExpandEnv(auth.SecretsDir))
+	credentialsPath := filepath.Join(authdir, "credentials.json")
+	b, err := ioutil.ReadFile(credentialsPath)
+	if err != nil {
+		return fmt.Errorf("unable to read credentials file '%v'\n"+
+			"Obtain this file from: https://console.developers.google.com/apis/credentials\n%w",
+			credentialsPath, err)
+	}
+	credentials, err := google.CredentialsFromJSON(ctx, b, "https://www.googleapis.com/auth/spreadsheets")
+	if err != nil {
+		return fmt.Errorf("unable to parse client secret file to config: %w", err)
+	}
+
+	// Create the sheets service client
+	s, err := sheets.New(oauth2.NewClient(ctx, credentials.TokenSource))
+	if err != nil {
+		return fmt.Errorf("unable to create sheets client: %w", err)
+	}
+
+	// Get the CTS statistics spreadsheet
+	spreadsheet, err := s.Spreadsheets.Get(cfg.Sheets.ID).Do()
+	if err != nil {
+		return fmt.Errorf("failed to get spreadsheet: %w", err)
+	}
+
+	// Scan the sheets of the spreadsheet (tabs at the bottom) for the 'data'
+	// sheet.
+	var dataSheet *sheets.Sheet
+	for _, sheet := range spreadsheet.Sheets {
+		if strings.ToLower(sheet.Properties.Title) == "data" {
+			dataSheet = sheet
+			break
+		}
+	}
+	if dataSheet == nil {
+		return fmt.Errorf("failed to find data sheet")
+	}
+
+	// Fetch the table column names
+	columns, err := fetchRow[string](s, spreadsheet, dataSheet, 0)
+
+	// Create a gerrit client, and find the latest CTS roll
+	gerrit, err := gerrit.New(cfg.Gerrit.Host, gerrit.Credentials{})
+	if err != nil {
+		return err
+	}
+	latestRoll, err := common.LatestCTSRoll(gerrit)
+	if err != nil {
+		return err
+	}
+
+	// Grab the results from the latest CTS roll
+	bb, err := buildbucket.New(ctx, auth)
+	if err != nil {
+		return err
+	}
+	rdb, err := resultsdb.New(ctx, auth)
+	if err != nil {
+		return err
+	}
+	results, ps, err := common.MostRecentResultsForChange(ctx, cfg, c.flags.cacheDir, gerrit, bb, rdb, latestRoll.Number)
+	if err != nil {
+		return err
+	}
+	if len(results) == 0 {
+		return fmt.Errorf("no results found")
+	}
+
+	// Find the CTS revision from the latest CTS roll
+	dawn, err := gitiles.New(ctx, cfg.Git.Dawn.Host, cfg.Git.Dawn.Project)
+	if err != nil {
+		return fmt.Errorf("failed to open dawn host: %w", err)
+	}
+	deps, err := dawn.DownloadFile(ctx, ps.RefsChanges(), "DEPS")
+	if err != nil {
+		return fmt.Errorf("failed to download DEPS from %v: %w", ps.RefsChanges(), err)
+	}
+	_, ctsHash, err := common.UpdateCTSHashInDeps(deps, "<unused>")
+	if err != nil {
+		return fmt.Errorf("failed to find CTS hash in deps: %w", err)
+	}
+
+	// Grab the CTS revision to count the number of unimplemented tests
+	numUnimplemented, err := countUnimplementedTests(cfg, ctsHash)
+	if err != nil {
+		return fmt.Errorf("failed to obtain number of unimplemented tests: %w", err)
+	}
+
+	// Generate a new set of counts of test by status
+	log.Printf("exporting results from cl %v ps %v...", ps.Change, ps.Patchset)
+	counts := map[result.Status]int{}
+	for _, r := range results {
+		counts[r.Status] = counts[r.Status] + 1
+	}
+
+	// Generate new cell data based on the table column names
+	data := []any{}
+	for _, column := range columns {
+		switch strings.ToLower(column) {
+		case "date":
+			data = append(data, time.Now().UTC().Format("2006-01-02"))
+		case "change":
+			data = append(data, ps.Change)
+		case "unimplemented":
+			data = append(data, numUnimplemented)
+		default:
+			count, ok := counts[result.Status(column)]
+			if !ok {
+				log.Println("no results with status", column)
+			}
+			data = append(data, count)
+		}
+	}
+
+	// Add a new row to the spreadsheet
+	_, err = s.Spreadsheets.Values.Append(spreadsheet.SpreadsheetId, "Data", &sheets.ValueRange{
+		Values: [][]any{data},
+	}).ValueInputOption("RAW").Do()
+	if err != nil {
+		return fmt.Errorf("failed to update spreadsheet: %v", err)
+	}
+
+	return nil
+}
+
+// rowRange returns a sheets range ("name!Ai:i") for the entire row with the
+// given index.
+func rowRange(index int, sheet *sheets.Sheet) string {
+	return fmt.Sprintf("%v!A%v:%v", sheet.Properties.Title, index+1, index+1)
+}
+
+// columnRange returns a sheets range ("name!i1:i") for the entire column with
+// the given index.
+func columnRange(index int, sheet *sheets.Sheet) string {
+	col := 'A' + index
+	if index > 25 {
+		panic("UNIMPLEMENTED")
+	}
+	return fmt.Sprintf("%v!%c1:%c", sheet.Properties.Title, col, col)
+}
+
+// fetchRow returns all the values in the given sheet's row.
+func fetchRow[T any](srv *sheets.Service, spreadsheet *sheets.Spreadsheet, sheet *sheets.Sheet, row int) ([]T, error) {
+	rng := rowRange(row, sheet)
+	data, err := srv.Spreadsheets.Values.Get(spreadsheet.SpreadsheetId, rng).Do()
+	if err != nil {
+		return nil, fmt.Errorf("Couldn't fetch %v: %w", rng, err)
+	}
+	out := make([]T, len(data.Values[0]))
+	for column, v := range data.Values[0] {
+		val, ok := v.(T)
+		if !ok {
+			return nil, fmt.Errorf("cell at %v:%v was type %T, but expected type %T", row, column, v, val)
+		}
+		out[column] = val
+	}
+	return out, nil
+}
+
+// countUnimplementedTests checks out the WebGPU CTS at ctsHash, builds the node
+// command line tool, and runs it with '--list-unimplemented webgpu:*' to count
+// the total number of unimplemented tests, which is returned.
+func countUnimplementedTests(cfg common.Config, ctsHash string) (int, error) {
+	tmpDir, err := os.MkdirTemp("", "dawn-cts-export")
+	if err != nil {
+		return 0, err
+	}
+	defer os.RemoveAll(tmpDir)
+
+	dir := filepath.Join(tmpDir, "cts")
+
+	gitExe, err := exec.LookPath("git")
+	if err != nil {
+		return 0, fmt.Errorf("failed to find git on PATH: %w", err)
+	}
+
+	git, err := git.New(gitExe)
+	if err != nil {
+		return 0, err
+	}
+
+	log.Printf("cloning cts to '%v'...", dir)
+	repo, err := git.Clone(dir, cfg.Git.CTS.HttpsURL(), nil)
+	if err != nil {
+		return 0, fmt.Errorf("failed to clone cts: %v", err)
+	}
+
+	log.Printf("checking out cts @ '%v'...", ctsHash)
+	if _, err := repo.Fetch(ctsHash, nil); err != nil {
+		return 0, fmt.Errorf("failed to fetch cts: %v", err)
+	}
+	if err := repo.Checkout(ctsHash, nil); err != nil {
+		return 0, fmt.Errorf("failed to clone cts: %v", err)
+	}
+
+	{
+		npm, err := exec.LookPath("npm")
+		if err != nil {
+			return 0, fmt.Errorf("failed to find npm on PATH: %w", err)
+		}
+		cmd := exec.Command(npm, "ci")
+		cmd.Dir = dir
+		if out, err := cmd.CombinedOutput(); err != nil {
+			return 0, fmt.Errorf("failed to run npm ci: %w\n%v", err, string(out))
+		}
+	}
+	{
+		npx, err := exec.LookPath("npx")
+		if err != nil {
+			return 0, fmt.Errorf("failed to find npx on PATH: %w", err)
+		}
+		cmd := exec.Command(npx, "grunt", "run:build-out-node")
+		cmd.Dir = dir
+		if out, err := cmd.CombinedOutput(); err != nil {
+			return 0, fmt.Errorf("failed to build CTS typescript: %w\n%v", err, string(out))
+		}
+	}
+	{
+		sh, err := exec.LookPath("node")
+		if err != nil {
+			return 0, fmt.Errorf("failed to find sh on PATH: %w", err)
+		}
+		cmd := exec.Command(sh, "./tools/run_node", "--list-unimplemented", "webgpu:*")
+		cmd.Dir = dir
+		out, err := cmd.CombinedOutput()
+		if err != nil {
+			return 0, fmt.Errorf("failed to gather unimplemented tests: %w", err)
+		}
+		lines := strings.Split(string(out), "\n")
+		return len(lines), nil
+	}
+}

tools/src/cmd/cts/main.go

@@ -28,6 +28,7 @@ import (
 	"dawn.googlesource.com/dawn/tools/src/utils"
 
 	// Register sub-commands
+	_ "dawn.googlesource.com/dawn/tools/src/cmd/cts/export"
 	_ "dawn.googlesource.com/dawn/tools/src/cmd/cts/format"
 	_ "dawn.googlesource.com/dawn/tools/src/cmd/cts/merge"
 	_ "dawn.googlesource.com/dawn/tools/src/cmd/cts/results"