tools/cts: Add cts validate, improvements & fixes

• Add `cts validate` command used to check for expectation collisions.
  Can be used as a presubmit check.
  This is more tightly checked than the previous logic, as this works on just
  the expectations, instead of results.

• Fix an issue where the test result reduction could introduce collisions with
  'Skip' expectations.
  To fix this, the update process first adds 'consumed' results for the skipped
  tests, preventing test tree reduction for that part of the tree.

• Fix a bug in the generation of 'New failures' and 'New flakes' which produced
  more expectations than was necessary.
  The issue here was that the tree roots could contain overlaps, and roots could
  be processed before sub-trees, resulting in inefficient expectations.

• Fix collisions in the expectations file, and update with results from
  the most recent roll.

Change-Id: I7b64553408998fb4416458ce564fc49c8f6d4d07
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/101860
Commit-Queue: Ben Clayton <bclayton@google.com>
Kokoro: Kokoro <noreply+kokoro@google.com>
Reviewed-by: Austin Eng <enga@chromium.org>

tools/src/cmd/cts/common/paths.go

@@ -25,6 +25,10 @@ const (
 	// RelativeExpectationsPath is the dawn-root relative path to the
 	// expectations.txt file.
 	RelativeExpectationsPath = "webgpu-cts/expectations.txt"
+
+	// RelativeTestListPath is the dawn-root relative path to the
+	// test_list.txt file.
+	RelativeTestListPath = "third_party/gn/webgpu-cts/test_list.txt"
 )
 
 // DefaultExpectationsPath returns the default path to the expectations.txt
@@ -36,3 +40,13 @@ func DefaultExpectationsPath() string {
 	}
 	return path
 }
+
+// DefaultTestListPath returns the default path to the test_list.txt
+// file. Returns an empty string if the file cannot be found.
+func DefaultTestListPath() string {
+	path := filepath.Join(utils.DawnRoot(), RelativeTestListPath)
+	if _, err := os.Stat(path); err != nil {
+		return ""
+	}
+	return path
+}

tools/src/cmd/cts/format/format.go

@@ -37,7 +37,7 @@ func (cmd) Name() string {
 }
 
 func (cmd) Desc() string {
-	return "formats a WebGPUExpectation file"
+	return "formats a WebGPU expectations.txt file"
 }
 
 func (c *cmd) RegisterFlags(ctx context.Context, cfg common.Config) ([]string, error) {

tools/src/cmd/cts/main.go

@@ -35,6 +35,7 @@ import (
 	_ "dawn.googlesource.com/dawn/tools/src/cmd/cts/roll"
 	_ "dawn.googlesource.com/dawn/tools/src/cmd/cts/time"
 	_ "dawn.googlesource.com/dawn/tools/src/cmd/cts/update"
+	_ "dawn.googlesource.com/dawn/tools/src/cmd/cts/validate"
 )
 
 func main() {

tools/src/cmd/cts/roll/roll.go

@@ -34,6 +34,7 @@ import (
 	"dawn.googlesource.com/dawn/tools/src/cmd/cts/common"
 	"dawn.googlesource.com/dawn/tools/src/container"
 	"dawn.googlesource.com/dawn/tools/src/cts/expectations"
+	"dawn.googlesource.com/dawn/tools/src/cts/query"
 	"dawn.googlesource.com/dawn/tools/src/cts/result"
 	"dawn.googlesource.com/dawn/tools/src/fileutils"
 	"dawn.googlesource.com/dawn/tools/src/gerrit"
@@ -246,6 +247,16 @@ func (r *roller) roll(ctx context.Context) error {
 		return err
 	}
 
+	// Pull out the test list from the generated files
+	testlist := func() []query.Query {
+		lines := strings.Split(generatedFiles[testListRelPath], "\n")
+		list := make([]query.Query, len(lines))
+		for i, line := range lines {
+			list[i] = query.Parse(line)
+		}
+		return list
+	}()
+
 	deletedFiles := []string{}
 	if currentWebTestFiles, err := r.dawn.ListFiles(ctx, dawnHash, webTestsPath); err != nil {
 		// If there's an error, allow NotFound. It means the directory did not exist, so no files
@@ -348,7 +359,7 @@ func (r *roller) roll(ctx context.Context) error {
 		// Note: The new expectations are not used if the last attempt didn't
 		// fail, but we always want to post the diagnostics
 		newExpectations := ex.Clone()
-		diags, err := newExpectations.Update(results)
+		diags, err := newExpectations.Update(results, testlist)
 		if err != nil {
 			return err
 		}

tools/src/cmd/cts/update/update.go

@@ -18,9 +18,13 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"io/ioutil"
+	"os"
+	"strings"
 
 	"dawn.googlesource.com/dawn/tools/src/cmd/cts/common"
 	"dawn.googlesource.com/dawn/tools/src/cts/expectations"
+	"dawn.googlesource.com/dawn/tools/src/cts/query"
 	"dawn.googlesource.com/dawn/tools/src/cts/result"
 	"go.chromium.org/luci/auth/client/authcli"
 )
@@ -53,6 +57,19 @@ func (c *cmd) RegisterFlags(ctx context.Context, cfg common.Config) ([]string, e
 	return nil, nil
 }
 
+func loadTestList(path string) ([]query.Query, error) {
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load test list: %w", err)
+	}
+	lines := strings.Split(string(data), "\n")
+	out := make([]query.Query, len(lines))
+	for i, l := range lines {
+		out[i] = query.Parse(l)
+	}
+	return out, nil
+}
+
 func (c *cmd) Run(ctx context.Context, cfg common.Config) error {
 	// Validate command line arguments
 	auth, err := c.flags.auth.Options()
@@ -75,16 +92,24 @@ func (c *cmd) Run(ctx context.Context, cfg common.Config) error {
 		return err
 	}
 
+	testlist, err := loadTestList(common.DefaultTestListPath())
+	if err != nil {
+		return err
+	}
+
+	if diag := ex.Validate(); diag.NumErrors() > 0 {
+		diag.Print(os.Stdout, c.flags.expectations)
+		return fmt.Errorf("validation failed")
+	}
+
 	// Update the expectations file with the results
-	msgs, err := ex.Update(results)
+	diag, err := ex.Update(results, testlist)
 	if err != nil {
 		return err
 	}
 
 	// Print any diagnostics
-	for _, msg := range msgs {
-		fmt.Printf("%v:%v %v\n", c.flags.expectations, msg.Line, msg.Message)
-	}
+	diag.Print(os.Stdout, c.flags.expectations)
 
 	// Save the updated expectations file
 	return ex.Save(c.flags.expectations)

tools/src/cmd/cts/validate/validate.go

@@ -0,0 +1,66 @@
+// Copyright 2022 The Dawn Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package validate
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+
+	"dawn.googlesource.com/dawn/tools/src/cmd/cts/common"
+	"dawn.googlesource.com/dawn/tools/src/cts/expectations"
+)
+
+func init() {
+	common.Register(&cmd{})
+}
+
+type cmd struct {
+	flags struct {
+		expectations string // expectations file path
+	}
+}
+
+func (cmd) Name() string {
+	return "validate"
+}
+
+func (cmd) Desc() string {
+	return "validates a WebGPU expectations.txt file"
+}
+
+func (c *cmd) RegisterFlags(ctx context.Context, cfg common.Config) ([]string, error) {
+	defaultExpectations := common.DefaultExpectationsPath()
+	flag.StringVar(&c.flags.expectations, "expectations", defaultExpectations, "path to CTS expectations file to update")
+	return nil, nil
+}
+
+func (c *cmd) Run(ctx context.Context, cfg common.Config) error {
+	ex, err := expectations.Load(c.flags.expectations)
+	if err != nil {
+		return err
+	}
+	diags := ex.Validate()
+
+	// Print any diagnostics
+	diags.Print(os.Stdout, c.flags.expectations)
+	if numErrs := diags.NumErrors(); numErrs > 0 {
+		return fmt.Errorf("%v errors found", numErrs)
+	}
+
+	fmt.Println("no issues found")
+	return nil
+}