tools/cts: Add cts validate, improvements & fixes

• Add `cts validate` command used to check for expectation collisions.
  Can be used as a presubmit check.
  This is more tightly checked than the previous logic, as this works on just
  the expectations, instead of results.

• Fix an issue where the test result reduction could introduce collisions with
  'Skip' expectations.
  To fix this, the update process first adds 'consumed' results for the skipped
  tests, preventing test tree reduction for that part of the tree.

• Fix a bug in the generation of 'New failures' and 'New flakes' which produced
  more expectations than was necessary.
  The issue here was that the tree roots could contain overlaps, and roots could
  be processed before sub-trees, resulting in inefficient expectations.

• Fix collisions in the expectations file, and update with results from
  the most recent roll.

Change-Id: I7b64553408998fb4416458ce564fc49c8f6d4d07
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/101860
Commit-Queue: Ben Clayton <bclayton@google.com>
Kokoro: Kokoro <noreply+kokoro@google.com>
Reviewed-by: Austin Eng <enga@chromium.org>

tools/src/cmd/cts/common/paths.go

@@ -25,6 +25,10 @@ const (
 	// RelativeExpectationsPath is the dawn-root relative path to the
 	// expectations.txt file.
 	RelativeExpectationsPath = "webgpu-cts/expectations.txt"
+
+	// RelativeTestListPath is the dawn-root relative path to the
+	// test_list.txt file.
+	RelativeTestListPath = "third_party/gn/webgpu-cts/test_list.txt"
 )
 
 // DefaultExpectationsPath returns the default path to the expectations.txt
@@ -36,3 +40,13 @@ func DefaultExpectationsPath() string {
 	}
 	return path
 }
+
+// DefaultTestListPath returns the default path to the test_list.txt
+// file. Returns an empty string if the file cannot be found.
+func DefaultTestListPath() string {
+	path := filepath.Join(utils.DawnRoot(), RelativeTestListPath)
+	if _, err := os.Stat(path); err != nil {
+		return ""
+	}
+	return path
+}

tools/src/cmd/cts/format/format.go

@@ -37,7 +37,7 @@ func (cmd) Name() string {
 }
 
 func (cmd) Desc() string {
-	return "formats a WebGPUExpectation file"
+	return "formats a WebGPU expectations.txt file"
 }
 
 func (c *cmd) RegisterFlags(ctx context.Context, cfg common.Config) ([]string, error) {

tools/src/cmd/cts/main.go

@@ -35,6 +35,7 @@ import (
 	_ "dawn.googlesource.com/dawn/tools/src/cmd/cts/roll"
 	_ "dawn.googlesource.com/dawn/tools/src/cmd/cts/time"
 	_ "dawn.googlesource.com/dawn/tools/src/cmd/cts/update"
+	_ "dawn.googlesource.com/dawn/tools/src/cmd/cts/validate"
 )
 
 func main() {

tools/src/cmd/cts/roll/roll.go

@@ -34,6 +34,7 @@ import (
 	"dawn.googlesource.com/dawn/tools/src/cmd/cts/common"
 	"dawn.googlesource.com/dawn/tools/src/container"
 	"dawn.googlesource.com/dawn/tools/src/cts/expectations"
+	"dawn.googlesource.com/dawn/tools/src/cts/query"
 	"dawn.googlesource.com/dawn/tools/src/cts/result"
 	"dawn.googlesource.com/dawn/tools/src/fileutils"
 	"dawn.googlesource.com/dawn/tools/src/gerrit"
@@ -246,6 +247,16 @@ func (r *roller) roll(ctx context.Context) error {
 		return err
 	}
 
+	// Pull out the test list from the generated files
+	testlist := func() []query.Query {
+		lines := strings.Split(generatedFiles[testListRelPath], "\n")
+		list := make([]query.Query, len(lines))
+		for i, line := range lines {
+			list[i] = query.Parse(line)
+		}
+		return list
+	}()
+
 	deletedFiles := []string{}
 	if currentWebTestFiles, err := r.dawn.ListFiles(ctx, dawnHash, webTestsPath); err != nil {
 		// If there's an error, allow NotFound. It means the directory did not exist, so no files
@@ -348,7 +359,7 @@ func (r *roller) roll(ctx context.Context) error {
 		// Note: The new expectations are not used if the last attempt didn't
 		// fail, but we always want to post the diagnostics
 		newExpectations := ex.Clone()
-		diags, err := newExpectations.Update(results)
+		diags, err := newExpectations.Update(results, testlist)
 		if err != nil {
 			return err
 		}

tools/src/cmd/cts/update/update.go

@@ -18,9 +18,13 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"io/ioutil"
+	"os"
+	"strings"
 
 	"dawn.googlesource.com/dawn/tools/src/cmd/cts/common"
 	"dawn.googlesource.com/dawn/tools/src/cts/expectations"
+	"dawn.googlesource.com/dawn/tools/src/cts/query"
 	"dawn.googlesource.com/dawn/tools/src/cts/result"
 	"go.chromium.org/luci/auth/client/authcli"
 )
@@ -53,6 +57,19 @@ func (c *cmd) RegisterFlags(ctx context.Context, cfg common.Config) ([]string, e
 	return nil, nil
 }
 
+func loadTestList(path string) ([]query.Query, error) {
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load test list: %w", err)
+	}
+	lines := strings.Split(string(data), "\n")
+	out := make([]query.Query, len(lines))
+	for i, l := range lines {
+		out[i] = query.Parse(l)
+	}
+	return out, nil
+}
+
 func (c *cmd) Run(ctx context.Context, cfg common.Config) error {
 	// Validate command line arguments
 	auth, err := c.flags.auth.Options()
@@ -75,16 +92,24 @@ func (c *cmd) Run(ctx context.Context, cfg common.Config) error {
 		return err
 	}
 
+	testlist, err := loadTestList(common.DefaultTestListPath())
+	if err != nil {
+		return err
+	}
+
+	if diag := ex.Validate(); diag.NumErrors() > 0 {
+		diag.Print(os.Stdout, c.flags.expectations)
+		return fmt.Errorf("validation failed")
+	}
+
 	// Update the expectations file with the results
-	msgs, err := ex.Update(results)
+	diag, err := ex.Update(results, testlist)
 	if err != nil {
 		return err
 	}
 
 	// Print any diagnostics
-	for _, msg := range msgs {
+	diag.Print(os.Stdout, c.flags.expectations)
-		fmt.Printf("%v:%v %v\n", c.flags.expectations, msg.Line, msg.Message)
-	}
 
 	// Save the updated expectations file
 	return ex.Save(c.flags.expectations)

tools/src/cmd/cts/validate/validate.go

@@ -0,0 +1,66 @@
+// Copyright 2022 The Dawn Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package validate
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+
+	"dawn.googlesource.com/dawn/tools/src/cmd/cts/common"
+	"dawn.googlesource.com/dawn/tools/src/cts/expectations"
+)
+
+func init() {
+	common.Register(&cmd{})
+}
+
+type cmd struct {
+	flags struct {
+		expectations string // expectations file path
+	}
+}
+
+func (cmd) Name() string {
+	return "validate"
+}
+
+func (cmd) Desc() string {
+	return "validates a WebGPU expectations.txt file"
+}
+
+func (c *cmd) RegisterFlags(ctx context.Context, cfg common.Config) ([]string, error) {
+	defaultExpectations := common.DefaultExpectationsPath()
+	flag.StringVar(&c.flags.expectations, "expectations", defaultExpectations, "path to CTS expectations file to update")
+	return nil, nil
+}
+
+func (c *cmd) Run(ctx context.Context, cfg common.Config) error {
+	ex, err := expectations.Load(c.flags.expectations)
+	if err != nil {
+		return err
+	}
+	diags := ex.Validate()
+
+	// Print any diagnostics
+	diags.Print(os.Stdout, c.flags.expectations)
+	if numErrs := diags.NumErrors(); numErrs > 0 {
+		return fmt.Errorf("%v errors found", numErrs)
+	}
+
+	fmt.Println("no issues found")
+	return nil
+}

tools/src/cts/expectations/diagnostic.go

@@ -16,6 +16,7 @@ package expectations
 
 import (
 	"fmt"
+	"io"
 	"strings"
 )
 
@@ -54,3 +55,24 @@ func (e Diagnostic) String() string {
 
 // Error implements the 'error' interface.
 func (e Diagnostic) Error() string { return e.String() }
+
+// Diagnostics is a list of diagnostic
+type Diagnostics []Diagnostic
+
+// NumErrors returns number of errors in the diagnostics
+func (l Diagnostics) NumErrors() int {
+	count := 0
+	for _, d := range l {
+		if d.Severity == Error {
+			count++
+		}
+	}
+	return count
+}
+
+// Print prints the list of diagnostics to 'w'
+func (l Diagnostics) Print(w io.Writer, path string) {
+	for _, d := range l {
+		fmt.Fprintf(w, "%v:%v %v\n", path, d.Line, d.Message)
+	}
+}

tools/src/cts/expectations/update.go

@@ -37,7 +37,11 @@ import (
 //   existing expectation lines.
 //
 // Update returns a list of diagnostics for things that should be addressed.
-func (c *Content) Update(results result.List) ([]Diagnostic, error) {
+//
+// Note: Validate() should be called before attempting to update the
+// expectations. If Validate() returns errors, then Update() behaviour is
+// undefined.
+func (c *Content) Update(results result.List, testlist []query.Query) (Diagnostics, error) {
 	// Make a copy of the results. This code mutates the list.
 	results = append(result.List{}, results...)
 
@@ -53,13 +57,23 @@ func (c *Content) Update(results result.List) ([]Diagnostic, error) {
 		tagSets[len(tagSets)-i-1] = s.Tags
 	}
 
-	// Update those expectations!
+	// Scan the full result list to obtain all the test variants
+	// (unique tag combinations).
+	variants := results.Variants()
+
+	// Add 'consumed' results for tests that were skipped.
+	// This ensures that skipped results are not included in reduced trees.
+	results = c.appendConsumedResultsForSkippedTests(results, testlist, variants)
+
 	u := updater{
 		in:       *c,
 		out:      Content{},
 		qt:       newQueryTree(results),
+		variants: variants,
 		tagSets:  tagSets,
 	}
+
+	// Update those expectations!
 	if err := u.build(); err != nil {
 		return nil, fmt.Errorf("while updating expectations: %w", err)
 	}
@@ -73,10 +87,44 @@ type updater struct {
 	in       Content   // the original expectations Content
 	out      Content   // newly built expectations Content
 	qt       queryTree // the query tree
+	variants []container.Set[string]
 	diags    []Diagnostic  // diagnostics raised during update
 	tagSets  []result.Tags // reverse-ordered tag-sets of 'in'
 }
 
+// Returns 'results' with additional 'consumed' results for tests that have
+// 'Skip' expectations. This fills in gaps for results, preventing tree 
+// reductions from marking skipped results as failure, which could result in 
+// expectation collisions.
+func (c *Content) appendConsumedResultsForSkippedTests(results result.List,
+	testlist []query.Query,
+	variants []container.Set[string]) result.List {
+	tree := query.Tree[struct{}]{}
+	for _, q := range testlist {
+		tree.Add(q, struct{}{})
+	}
+	for _, c := range c.Chunks {
+		for _, ex := range c.Expectations {
+			if container.NewSet(ex.Status...).Contains(string(result.Skip)) {
+				for _, variant := range variants {
+					if !variant.ContainsAll(ex.Tags) {
+						continue
+					}
+					glob, _ := tree.Glob(query.Parse(ex.Query))
+					for _, qd := range glob {
+						results = append(results, result.Result{
+							Query:  qd.Query,
+							Tags:   variant,
+							Status: consumed,
+						})
+					}
+				}
+			}
+		}
+	}
+	return results
+}
+
 // simplifyStatuses replaces all result statuses that are not one of
 // 'Pass', 'RetryOnFailure', 'Slow', 'Skip' with 'Failure', and also replaces
 // 'Skip' results with 'Pass'.
@@ -160,14 +208,9 @@ func (qt *queryTree) glob(q query.Query) (result.List, error) {
 	return out, nil
 }
 
-// globAndCheckForCollisions returns the list of results matching the given tags
+// globTags returns the list of results matching the given tags under (or with)
-// under (or with) the given query.
+// the given query.
-// globAndCheckForCollisions will return an error if any of the results are
+func (qt *queryTree) globTags(q query.Query, t result.Tags) (result.List, error) {
-// already consumed by a non-zero line. The non-zero line distinguishes between
-// results consumed by expectations declared in the input (non-zero line), vs
-// those that were introduced by the update (zero line). We only want to error
-// if there's a collision in user declared expectations.
-func (qt *queryTree) globAndCheckForCollisions(q query.Query, t result.Tags) (result.List, error) {
 	glob, err := qt.tree.Glob(q)
 	if err != nil {
 		return nil, err
@@ -177,12 +220,6 @@ func (qt *queryTree) globAndCheckForCollisions(q query.Query, t result.Tags) (re
 	for _, indices := range glob {
 		for _, idx := range indices.Data {
 			if r := qt.results[idx]; r.Tags.ContainsAll(t) {
-				if at := qt.consumedAt[idx]; at > 0 {
-					if len(t) > 0 {
-						return nil, fmt.Errorf("%v %v collides with expectation at line %v", t, q, at)
-					}
-					return nil, fmt.Errorf("%v collides with expectation at line %v", q, at)
-				}
 				out = append(out, r)
 			}
 		}
@@ -194,7 +231,7 @@ func (qt *queryTree) globAndCheckForCollisions(q query.Query, t result.Tags) (re
 // under (or with) the given query, as consumed.
 // line is used to record the line at which the results were consumed. If the
 // results were consumed as part of generating new expectations then line should
-// be 0. See queryTree.globAndCheckForCollisions().
+// be 0.
 func (qt *queryTree) markAsConsumed(q query.Query, t result.Tags, line int) {
 	if glob, err := qt.tree.Glob(q); err == nil {
 		for _, indices := range glob {
@@ -274,21 +311,12 @@ func (u *updater) chunk(in Chunk) Chunk {
 	return out
 }
 
-// chunk returns a new list of Expectations, based on the Expectation 'in',
+// expectation returns a new list of Expectations, based on the Expectation 'in',
 // using the new result data.
 func (u *updater) expectation(in Expectation, keep bool) []Expectation {
 	// noResults is a helper for returning when the expectation has no test
 	// results.
-	// If the expectation has an expected 'Skip' result, then we're likely
-	// to be missing results (as the test was not run). In this situation
-	// the expectation is preserved, and no diagnostics are raised.
-	// If the expectation did not have a 'Skip' result, then a diagnostic will
-	// be raised and the expectation will be removed.
 	noResults := func() []Expectation {
-		if container.NewSet(in.Status...).Contains(string(result.Skip)) {
-			return []Expectation{in}
-		}
-		// Expectation does not have a 'Skip' result.
 		if len(in.Tags) > 0 {
 			u.diag(Warning, in.Line, "no results found for '%v' with tags %v", in.Query, in.Tags)
 		} else {
@@ -298,12 +326,11 @@ func (u *updater) expectation(in Expectation, keep bool) []Expectation {
 		return []Expectation{}
 	}
 
-	// Grab all the results that match the expectation's query
 	q := query.Parse(in.Query)
 
 	// Glob the results for the expectation's query + tag combination.
 	// Ensure that none of these are already consumed.
-	results, err := u.qt.globAndCheckForCollisions(q, in.Tags)
+	results, err := u.qt.globTags(q, in.Tags)
 	// If we can't find any results for this query + tag combination, then bail.
 	switch {
 	case errors.As(err, &query.ErrNoDataForQuery{}):
@@ -357,18 +384,14 @@ func (u *updater) expectation(in Expectation, keep bool) []Expectation {
 // addNewExpectations (potentially) appends to 'u.out' chunks for new flaky and
 // failing tests.
 func (u *updater) addNewExpectations() error {
-	// Scan the full result list to obtain all the test variants
-	// (unique tag combinations).
-	allVariants := u.qt.results.Variants()
-
 	// For each variant:
 	// • Build a query tree using the results filtered to the variant, and then
 	//   reduce the tree.
 	// • Take all the reduced-tree leaf nodes, and add these to 'roots'.
 	// Once we've collected all the roots, we'll use these to build the
 	// expectations across the reduced set of tags.
-	roots := container.NewMap[string, query.Query]()
+	roots := query.Tree[bool]{}
-	for _, variant := range allVariants {
+	for _, variant := range u.variants {
 		// Build a tree from the results matching the given variant.
 		tree, err := u.qt.results.FilterByVariant(variant).StatusTree()
 		if err != nil {
@@ -378,15 +401,16 @@ func (u *updater) addNewExpectations() error {
 		tree.Reduce(treeReducer)
 		// Add all the reduced leaf nodes to 'roots'.
 		for _, qd := range tree.List() {
-			roots.Add(qd.Query.String(), qd.Query)
+			// Use Split() to ensure that only the leaves have data (true) in the tree
+			roots.Split(qd.Query, true)
 		}
 	}
 
 	// Build all the expectations for each of the roots.
 	expectations := []Expectation{}
-	for _, root := range roots.Values() {
+	for _, root := range roots.List() {
 		expectations = append(expectations, u.expectationsForRoot(
-			root,                  // Root query
+			root.Query,            // Root query
 			0,                     // Line number
 			"crbug.com/dawn/0000", // Bug
 			"",                    // Comment

tools/src/cts/expectations/update_test.go

@@ -18,6 +18,7 @@ import (
 	"strings"
 	"testing"
 
+	"dawn.googlesource.com/dawn/tools/src/container"
 	"dawn.googlesource.com/dawn/tools/src/cts/expectations"
 	"dawn.googlesource.com/dawn/tools/src/cts/query"
 	"dawn.googlesource.com/dawn/tools/src/cts/result"
@@ -41,7 +42,7 @@ func TestUpdate(t *testing.T) {
 		expectations string
 		results      result.List
 		updated      string
-		diagnostics  []expectations.Diagnostic
+		diagnostics  expectations.Diagnostics
 		err          string
 	}
 	for _, test := range []Test{
@@ -73,7 +74,7 @@ some:other,test:* [ Failure ]
 			updated: `
 some:other,test:* [ Failure ]
 `,
-			diagnostics: []expectations.Diagnostic{
+			diagnostics: expectations.Diagnostics{
 				{
 					Severity: expectations.Warning,
 					Line:     headerLines + 2,
@@ -109,7 +110,7 @@ some:other,test:* [ Failure ]
 			updated: `
 some:other,test:* [ Failure ]
 `,
-			diagnostics: []expectations.Diagnostic{
+			diagnostics: expectations.Diagnostics{
 				{
 					Severity: expectations.Warning,
 					Line:     headerLines + 3,
@@ -117,55 +118,6 @@ some:other,test:* [ Failure ]
 				},
 			},
 		},
-		{ //////////////////////////////////////////////////////////////////////
-			name: "no results found Skip",
-			expectations: `
-crbug.com/a/123 a:missing,test,result:* [ Skip ]
-
-some:other,test:* [ Failure ]
-`,
-			results: result.List{
-				result.Result{
-					Query:  Q("some:other,test:*"),
-					Tags:   result.NewTags("os-a", "gpu-a"),
-					Status: result.Failure,
-				},
-				result.Result{
-					Query:  Q("some:other,test:*"),
-					Tags:   result.NewTags("os-b", "gpu-b"),
-					Status: result.Failure,
-				},
-			},
-			updated: `
-crbug.com/a/123 a:missing,test,result:* [ Skip ]
-
-some:other,test:* [ Failure ]
-`,
-		},
-		{ //////////////////////////////////////////////////////////////////////
-			name: "simple expectation collision",
-			expectations: `
-a:b,c:* [ Failure ]
-a:b,c:* [ Failure ]
-`,
-			results: result.List{
-				result.Result{
-					Query:  Q("a:b,c:d"),
-					Tags:   result.NewTags("os-a", "os-c", "gpu-b"),
-					Status: result.Failure,
-				},
-			},
-			updated: `
-a:b,c:* [ Failure ]
-`,
-			diagnostics: []expectations.Diagnostic{
-				{
-					Severity: expectations.Error,
-					Line:     headerLines + 3,
-					Message:  "a:b,c:* collides with expectation at line 8",
-				},
-			},
-		},
 		{ //////////////////////////////////////////////////////////////////////
 			name: "simple expectation with tags",
 			expectations: `
@@ -182,100 +134,6 @@ a:b,c:* [ Failure ]
 			updated: `
 a:b,c:* [ Failure ]
 `,
-			diagnostics: []expectations.Diagnostic{
-				{
-					Severity: expectations.Error,
-					Line:     headerLines + 3,
-					Message:  "[gpu-b] a:b,c:* collides with expectation at line 8",
-				},
-			},
-		},
-		{ //////////////////////////////////////////////////////////////////////
-			name: "simple expectation collision KEEP",
-			expectations: `
-# KEEP
-a:b,c:* [ Failure ]
-a:b,c:* [ Failure ]
-`,
-			results: result.List{
-				result.Result{
-					Query:  Q("a:b,c:d"),
-					Tags:   result.NewTags("os-a", "os-c", "gpu-b"),
-					Status: result.Failure,
-				},
-			},
-			updated: `
-# KEEP
-a:b,c:* [ Failure ]
-`,
-			diagnostics: []expectations.Diagnostic{
-				{
-					Severity: expectations.Error,
-					Line:     headerLines + 4,
-					Message:  "a:b,c:* collides with expectation at line 9",
-				},
-			},
-		},
-		{ //////////////////////////////////////////////////////////////////////
-			name: "collision with child-expectation",
-			expectations: `
-a:b:x:* [ Failure ]
-a:b:* [ Failure ]
-`,
-			results: result.List{
-				result.Result{
-					Query:  Q("a:b:x:*"),
-					Tags:   result.NewTags("os-a", "os-c", "gpu-b"),
-					Status: result.Failure,
-				},
-				result.Result{
-					Query:  Q("a:b:y:*"),
-					Tags:   result.NewTags("os-a", "os-c", "gpu-b"),
-					Status: result.Failure,
-				},
-			},
-			updated: `
-a:b:x:* [ Failure ]
-
-# New failures. Please triage:
-crbug.com/dawn/0000 a:b:y:* [ Failure ]
-`,
-			diagnostics: []expectations.Diagnostic{
-				{
-					Severity: expectations.Error,
-					Line:     headerLines + 3,
-					Message:  "a:b:* collides with expectation at line 8",
-				},
-			},
-		},
-		{ //////////////////////////////////////////////////////////////////////
-			name: "collision with parent-expectation",
-			expectations: `
-a:b:* [ Failure ]
-a:b:x:* [ Failure ]
-`,
-			results: result.List{
-				result.Result{
-					Query:  Q("a:b:x:*"),
-					Tags:   result.NewTags("os-a", "os-c", "gpu-b"),
-					Status: result.Failure,
-				},
-				result.Result{
-					Query:  Q("a:b:y:*"),
-					Tags:   result.NewTags("os-a", "os-c", "gpu-b"),
-					Status: result.Failure,
-				},
-			},
-			updated: `
-a:b:* [ Failure ]
-`,
-			diagnostics: []expectations.Diagnostic{
-				{
-					Severity: expectations.Error,
-					Line:     headerLines + 3,
-					Message:  "a:b:x:* collides with expectation at line 8",
-				},
-			},
 		},
 		{ //////////////////////////////////////////////////////////////////////
 			name: "expectation test now passes",
@@ -345,7 +203,7 @@ crbug.com/a/123 [ gpu-b os-b ] a:b,c:d [ Failure ]
 crbug.com/a/123 [ gpu-a os-a ] a:b,c:d [ Failure ]
 crbug.com/a/123 [ gpu-b os-b ] a:b,c:d [ Failure ]
 `,
-			diagnostics: []expectations.Diagnostic{
+			diagnostics: expectations.Diagnostics{
 				{
 					Severity: expectations.Note,
 					Line:     headerLines + 3,
@@ -369,7 +227,7 @@ crbug.com/a/123 a:b,c:d:* [ Failure ]
 # KEEP
 crbug.com/a/123 a:b,c:d:* [ Failure ]
 `,
-			diagnostics: []expectations.Diagnostic{
+			diagnostics: expectations.Diagnostics{
 				{
 					Severity: expectations.Note,
 					Line:     headerLines + 3,
@@ -391,6 +249,11 @@ crbug.com/a/123 a:b,c:d:* [ Failure ]
 					Tags:   result.NewTags("os-a", "gpu-b"),
 					Status: result.Abort,
 				},
+				result.Result{
+					Query:  Q("suite:dir_a,dir_b:test_c:case=4;*"),
+					Tags:   result.NewTags("os-a", "gpu-a"),
+					Status: result.Crash,
+				},
 				result.Result{
 					Query:  Q("suite:dir_a,dir_b:test_c:case=4;*"),
 					Tags:   result.NewTags("os-b", "gpu-b"),
@@ -402,22 +265,22 @@ crbug.com/a/123 a:b,c:d:* [ Failure ]
 					Status: result.RetryOnFailure,
 				},
 				result.Result{
-					Query:  Q("suite:dir_a,dir_b:test_b;case=5;*"),
+					Query:  Q("suite:dir_a,dir_b:test_b:case=5;*"),
 					Tags:   result.NewTags("os-b", "gpu-b"),
 					Status: result.Pass,
 				},
 				result.Result{
-					Query:  Q("suite:dir_a,dir_b:test_b:*"),
+					Query:  Q("suite:dir_a,dir_b:test_b:case=6;*"),
 					Tags:   result.NewTags("os-a", "gpu-a"),
 					Status: result.Slow,
 				},
 				result.Result{
-					Query:  Q("suite:dir_a,dir_b:test_b:*"),
+					Query:  Q("suite:dir_a,dir_b:test_b:case=6;*"),
 					Tags:   result.NewTags("os-b", "gpu-a"),
 					Status: result.Pass,
 				},
 				result.Result{
-					Query:  Q("suite:dir_a,dir_b:test_c:*"),
+					Query:  Q("suite:dir_a,dir_b:test_c:case=6;*"),
 					Tags:   result.NewTags("os-a", "gpu-a"),
 					Status: result.RetryOnFailure,
 				},
@@ -425,14 +288,13 @@ crbug.com/a/123 a:b,c:d:* [ Failure ]
 			updated: `# A comment
 
 # New flakes. Please triage:
-crbug.com/dawn/0000 [ gpu-a os-a ] suite:dir_a,dir_b:test_c:* [ RetryOnFailure ]
+crbug.com/dawn/0000 suite:dir_a,dir_b:test_c:case=5;* [ RetryOnFailure ]
-crbug.com/dawn/0000 [ gpu-b os-b ] suite:dir_a,dir_b:test_c:case=5;* [ RetryOnFailure ]
+crbug.com/dawn/0000 suite:dir_a,dir_b:test_c:case=6;* [ RetryOnFailure ]
 
 # New failures. Please triage:
-crbug.com/dawn/0000 [ gpu-b os-a ] suite:* [ Failure ]
+crbug.com/dawn/0000 suite:dir_a,dir_b:test_a:* [ Failure ]
-crbug.com/dawn/0000 [ gpu-a os-a ] suite:dir_a,dir_b:test_a:* [ Failure ]
 crbug.com/dawn/0000 [ gpu-a os-a ] suite:dir_a,dir_b:test_b:* [ Slow ]
-crbug.com/dawn/0000 [ gpu-b os-b ] suite:dir_a,dir_b:test_c:case=4;* [ Failure ]
+crbug.com/dawn/0000 suite:dir_a,dir_b:test_c:case=4;* [ Failure ]
 `,
 		},
 		{ //////////////////////////////////////////////////////////////////////
@@ -650,8 +512,13 @@ crbug.com/dawn/0000 a:b,c:t29:* [ Failure ]
 			t.Fatalf("'%v': expectations.Parse():\n%v", test.name, err)
 		}
 
+		testList := container.NewMap[string, query.Query]()
+		for _, r := range test.results {
+			testList.Add(r.Query.String(), r.Query)
+		}
+
 		errMsg := ""
-		diagnostics, err := ex.Update(test.results)
+		diagnostics, err := ex.Update(test.results, testList.Values())
 		if err != nil {
 			errMsg = err.Error()
 		}

tools/src/cts/expectations/validate.go

@@ -0,0 +1,77 @@
+// Copyright 2022 The Dawn Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package expectations provides types and helpers for parsing, updating and
+// writing WebGPU expectations files.
+//
+// See <dawn>/webgpu-cts/expectations.txt for more information.
+package expectations
+
+import (
+	"fmt"
+
+	"dawn.googlesource.com/dawn/tools/src/container"
+	"dawn.googlesource.com/dawn/tools/src/cts/query"
+)
+
+func (c Content) tagsCollide(a, b container.Set[string]) bool {
+	for _, set := range c.Tags.Sets {
+		aSet := a.Intersection(set.Tags)
+		bSet := b.Intersection(set.Tags)
+		if len(aSet) != 0 && len(bSet) != 0 && len(aSet.Intersection(bSet)) == 0 {
+			return false
+		}
+	}
+	return true
+}
+
+// Validate checks that the expectations do not contain errors
+func (c Content) Validate() Diagnostics {
+	tree, _ := query.NewTree[Expectations]()
+	for _, chunk := range c.Chunks {
+		for _, ex := range chunk.Expectations {
+			node := tree.GetOrCreate(query.Parse(ex.Query), func() Expectations {
+				return Expectations{}
+			})
+			*node = append(*node, ex)
+		}
+	}
+	var out Diagnostics
+	for _, chunk := range c.Chunks {
+		for _, ex := range chunk.Expectations {
+			glob, err := tree.Glob(query.Parse(ex.Query))
+			if err != nil {
+				out = append(out, Diagnostic{
+					Severity: Error,
+					Line:     ex.Line,
+					Message:  err.Error(),
+				})
+				continue
+			}
+			for _, qd := range glob {
+				expectations := qd.Data
+				for _, other := range expectations {
+					if other.Line != ex.Line && c.tagsCollide(ex.Tags, other.Tags) {
+						out = append(out, Diagnostic{
+							Severity: Error,
+							Line:     ex.Line,
+							Message:  fmt.Sprintf("expectation collides with expectation on line %v", other.Line),
+						})
+					}
+				}
+			}
+		}
+	}
+	return out
+}

tools/src/cts/expectations/validate_test.go

@@ -0,0 +1,130 @@
+// Copyright 2022 The Dawn Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package expectations_test
+
+import (
+	"testing"
+
+	"dawn.googlesource.com/dawn/tools/src/cts/expectations"
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestValidate(t *testing.T) {
+	header := `# BEGIN TAG HEADER
+# OS
+# tags: [ os-a os-b os-c ]
+# GPU
+# tags: [ gpu-a gpu-b gpu-c ]
+# END TAG HEADER
+`
+
+	type Test struct {
+		name         string
+		expectations string
+		diagnostics  expectations.Diagnostics
+	}
+	for _, test := range []Test{
+		{ //////////////////////////////////////////////////////////////////////
+			name:         "empty",
+			expectations: ``,
+		},
+		{ //////////////////////////////////////////////////////////////////////
+			name: "simple",
+			expectations: `
+crbug.com/a/123 a:b,c:d,* [ Failure ]
+`,
+		},
+		{ //////////////////////////////////////////////////////////////////////
+			name: "no-tag collision",
+			expectations: `
+crbug.com/a/123 a:b,c:d,* [ Failure ]
+crbug.com/a/123 a:x,x:d,* [ Failure ]
+crbug.com/a/123 a:b,c:d,* [ Failure ]
+`,
+			diagnostics: expectations.Diagnostics{
+				{
+					Line:     8,
+					Severity: expectations.Error,
+					Message:  "expectation collides with expectation on line 10",
+				},
+				{
+					Line:     10,
+					Severity: expectations.Error,
+					Message:  "expectation collides with expectation on line 8",
+				},
+			},
+		},
+		{ //////////////////////////////////////////////////////////////////////
+			name: "tag collision",
+			expectations: `
+crbug.com/a/123 [ os-a ] a:b,c:d,* [ Failure ]
+crbug.com/a/123 a:x,x:d,* [ Failure ]
+crbug.com/a/123 [ os-a ] a:b,c:d,* [ Failure ]
+`,
+			diagnostics: expectations.Diagnostics{
+				{
+					Line:     8,
+					Severity: expectations.Error,
+					Message:  "expectation collides with expectation on line 10",
+				},
+				{
+					Line:     10,
+					Severity: expectations.Error,
+					Message:  "expectation collides with expectation on line 8",
+				},
+			},
+		},
+		{ //////////////////////////////////////////////////////////////////////
+			name: "nested no-tag collision",
+			expectations: `
+crbug.com/a/123 a:b,c:d,e:* [ Failure ]
+crbug.com/a/123 a:x,x:d,* [ Failure ]
+crbug.com/a/123 a:b,c:d,* [ Failure ]
+`,
+			diagnostics: expectations.Diagnostics{
+				{
+					Line:     10,
+					Severity: expectations.Error,
+					Message:  "expectation collides with expectation on line 8",
+				},
+			},
+		},
+		{ //////////////////////////////////////////////////////////////////////
+			name: "tag collision",
+			expectations: `
+crbug.com/a/123 [ os-a ] a:b,c:d,e:* [ Failure ]
+crbug.com/a/123 a:x,x:d,* [ Failure ]
+crbug.com/a/123 [ os-a ] a:b,c:d,* [ Failure ]
+`,
+			diagnostics: expectations.Diagnostics{
+				{
+					Line:     10,
+					Severity: expectations.Error,
+					Message:  "expectation collides with expectation on line 8",
+				},
+			},
+		},
+	} {
+		ex, err := expectations.Parse(header + test.expectations)
+		if err != nil {
+			t.Fatalf("'%v': expectations.Parse():\n%v", test.name, err)
+		}
+
+		diagnostics := ex.Validate()
+		if diff := cmp.Diff(diagnostics, test.diagnostics); diff != "" {
+			t.Errorf("'%v': expectations.Update() error:\n%v", test.name, diff)
+		}
+	}
+}

tools/src/cts/query/tree.go

@@ -252,7 +252,7 @@ func NewTree[Data any](entries ...QueryData[Data]) (Tree[Data], error) {
 }
 
 // Add adds a new data to the tree.
-// Returns ErrDuplicateData if the tree already contains a data for the given
+// Returns ErrDuplicateData if the tree already contains a data for the given node at query
 func (t *Tree[Data]) Add(q Query, d Data) error {
 	node := &t.TreeNode
 	q.Walk(func(q Query, t Target, n string) error {
@@ -266,6 +266,37 @@ func (t *Tree[Data]) Add(q Query, d Data) error {
 	return nil
 }
 
+// Split adds a new data to the tree, clearing any ancestor node's data.
+// Returns ErrDuplicateData if the tree already contains a data for the given node at query
+func (t *Tree[Data]) Split(q Query, d Data) error {
+	node := &t.TreeNode
+	q.Walk(func(q Query, t Target, n string) error {
+		delete(node.Children, TreeNodeChildKey{Name: "*", Target: t})
+		node.Data = nil
+		node = node.getOrCreateChild(TreeNodeChildKey{n, t})
+		return nil
+	})
+	if node.Data != nil {
+		return ErrDuplicateData{node.Query}
+	}
+	node.Data = &d
+	return nil
+}
+
+// GetOrCreate returns existing, or adds a new data to the tree.
+func (t *Tree[Data]) GetOrCreate(q Query, create func() Data) *Data {
+	node := &t.TreeNode
+	q.Walk(func(q Query, t Target, n string) error {
+		node = node.getOrCreateChild(TreeNodeChildKey{n, t})
+		return nil
+	})
+	if node.Data == nil {
+		data := create()
+		node.Data = &data
+	}
+	return node.Data
+}
+
 // Reduce reduces the tree using the Merger function f.
 // If the Merger function returns a non-nil Data value, then this will be used
 // to replace the non-leaf node with a new leaf node holding the returned Data.

tools/src/cts/query/tree_test.go

@@ -238,7 +238,6 @@ func TestNewSingle(t *testing.T) {
 			t.Errorf("NewTree(%v) tree was not as expected:\n%v", test.in, diff)
 		}
 	}
-
 }
 
 func TestNewMultiple(t *testing.T) {
@@ -351,6 +350,223 @@ func TestNewWithCollision(t *testing.T) {
 	}
 }
 
+func TestSplit(t *testing.T) {
+	type Tree = query.Tree[string]
+	type Node = query.TreeNode[string]
+	type QueryData = query.QueryData[string]
+	type Children = query.TreeNodeChildren[string]
+
+	type Test struct {
+		in   QueryData
+		pre  Tree
+		post Tree
+	}
+	for _, test := range []Test{
+		{ /////////////////////////////////////////////////////////////////////
+			in: QueryData{
+				Query: Q(`suite:*`),
+				Data:  pass,
+			},
+			post: Tree{
+				TreeNode: Node{
+					Children: Children{
+						query.TreeNodeChildKey{`suite`, query.Suite}: {
+							Query: Q(`suite`),
+							Children: Children{
+								query.TreeNodeChildKey{`*`, query.Files}: {
+									Query: Q(`suite:*`),
+									Data:  &pass,
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{ /////////////////////////////////////////////////////////////////////
+			in: QueryData{
+				Query: Q(`suite:a,b:*`),
+				Data:  pass,
+			},
+			pre: Tree{
+				TreeNode: Node{
+					Children: Children{
+						query.TreeNodeChildKey{`suite`, query.Suite}: {
+							Query: Q(`suite`),
+							Children: Children{
+								query.TreeNodeChildKey{`a`, query.Files}: {
+									Query: Q(`suite:a`),
+									Children: Children{
+										query.TreeNodeChildKey{`*`, query.Files}: {
+											Query: Q(`suite:a,*`),
+											Data:  &pass,
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			post: Tree{
+				TreeNode: Node{
+					Children: Children{
+						query.TreeNodeChildKey{`suite`, query.Suite}: {
+							Query: Q(`suite`),
+							Children: Children{
+								query.TreeNodeChildKey{`a`, query.Files}: {
+									Query: Q(`suite:a`),
+									Children: Children{
+										query.TreeNodeChildKey{`b`, query.Files}: {
+											Query: Q(`suite:a,b`),
+											Children: Children{
+												query.TreeNodeChildKey{`*`, query.Tests}: {
+													Query: Q(`suite:a,b:*`),
+													Data:  &pass,
+												},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{ /////////////////////////////////////////////////////////////////////
+			in: QueryData{
+				Query: Q(`suite:a:*`),
+				Data:  pass,
+			},
+			pre: Tree{
+				TreeNode: Node{
+					Children: Children{
+						query.TreeNodeChildKey{`suite`, query.Suite}: {
+							Query: Q(`suite`),
+							Children: Children{
+								query.TreeNodeChildKey{`a`, query.Files}: {
+									Query: Q(`suite:a`),
+									Children: Children{
+										query.TreeNodeChildKey{`b`, query.Files}: {
+											Query: Q(`suite:a,b`),
+											Children: Children{
+												query.TreeNodeChildKey{`*`, query.Files}: {
+													Query: Q(`suite:a,b,*`),
+													Data:  &pass,
+												},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			post: Tree{
+				TreeNode: Node{
+					Children: Children{
+						query.TreeNodeChildKey{`suite`, query.Suite}: {
+							Query: Q(`suite`),
+							Children: Children{
+								query.TreeNodeChildKey{`a`, query.Files}: {
+									Query: Q(`suite:a`),
+									Children: Children{
+										query.TreeNodeChildKey{`*`, query.Tests}: {
+											Query: Q(`suite:a:*`),
+											Data:  &pass,
+										},
+										query.TreeNodeChildKey{`b`, query.Files}: {
+											Query: Q(`suite:a,b`),
+											Children: Children{
+												query.TreeNodeChildKey{`*`, query.Files}: {
+													Query: Q(`suite:a,b,*`),
+													Data:  &pass,
+												},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{ /////////////////////////////////////////////////////////////////////
+			in: QueryData{
+				Query: Q(`suite:a,b:c:*`),
+				Data:  pass,
+			},
+			pre: Tree{
+				TreeNode: Node{
+					Children: Children{
+						query.TreeNodeChildKey{`suite`, query.Suite}: {
+							Query: Q(`suite`),
+							Children: Children{
+								query.TreeNodeChildKey{`a`, query.Files}: {
+									Query: Q(`suite:a`),
+									Children: Children{
+										query.TreeNodeChildKey{`b`, query.Files}: {
+											Query: Q(`suite:a,b`),
+											Children: Children{
+												query.TreeNodeChildKey{`*`, query.Tests}: {
+													Query: Q(`suite:a,b:*`),
+													Data:  &pass,
+												},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			post: Tree{
+				TreeNode: Node{
+					Children: Children{
+						query.TreeNodeChildKey{`suite`, query.Suite}: {
+							Query: Q(`suite`),
+							Children: Children{
+								query.TreeNodeChildKey{`a`, query.Files}: {
+									Query: Q(`suite:a`),
+									Children: Children{
+										query.TreeNodeChildKey{`b`, query.Files}: {
+											Query: Q(`suite:a,b`),
+											Children: Children{
+												query.TreeNodeChildKey{`c`, query.Tests}: {
+													Query: Q(`suite:a,b:c`),
+													Children: Children{
+														query.TreeNodeChildKey{`*`, query.Cases}: {
+															Query: Q(`suite:a,b:c:*`),
+															Data:  &pass,
+														},
+													},
+												},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	} {
+		tree := test.pre
+		if err := tree.Split(test.in.Query, test.in.Data); err != nil {
+			t.Errorf("NewTree(%v): %v", test.in, err)
+			continue
+		}
+		if diff := cmp.Diff(tree, test.post); diff != "" {
+			t.Errorf("Split(%v) tree was not as expected:\n%v", test.in, diff)
+		}
+	}
+}
+
 func TestList(t *testing.T) {
 	type QueryData = query.QueryData[string]
 

webgpu-cts/expectations.txt

@@ -79,7 +79,8 @@ crbug.com/dawn/1487 [ intel-gen-9 win10 ] webgpu:api,operation,resource_init,tex
 # Flaky failures using SwiftShader (crbug.com/1344876).
 # KEEP
 ################################################################################
-crbug.com/1343969 [ win ] webgpu:web_platform,copyToTexture,ImageBitmap:* [ RetryOnFailure Slow ]
+crbug.com/1343969 [ win ] webgpu:web_platform,copyToTexture,ImageBitmap:from_ImageData:* [ RetryOnFailure Slow ]
+crbug.com/1343969 [ webgpu-adapter-default win ] webgpu:web_platform,copyToTexture,ImageBitmap:from_canvas:* [ RetryOnFailure Slow ]
 crbug.com/1344876 [ ubuntu webgpu-adapter-swiftshader ] webgpu:web_platform,copyToTexture,ImageBitmap:from_ImageData:* [ RetryOnFailure Slow ]
 crbug.com/1359106 [ ubuntu webgpu-adapter-default ] webgpu:web_platform,copyToTexture,ImageBitmap:from_ImageData:* [ RetryOnFailure ]
 
@@ -389,9 +390,6 @@ crbug.com/tint/1613 webgpu:shader,execution,expression,unary,f32_arithmetic:nega
 ################################################################################
 # shader execution failures
 ################################################################################
-crbug.com/dawn/0000 [ intel-gen-9 target-cpu-32 ] webgpu:shader,execution,expression,binary,f32_arithmetic:addition:inputSource="uniform";vectorize="_undef_" [ Failure ]
-crbug.com/dawn/0000 [ intel-gen-9 target-cpu-32 ] webgpu:shader,execution,expression,binary,f32_arithmetic:division:inputSource="uniform";vectorize="_undef_" [ Failure ]
-crbug.com/dawn/0000 [ intel-gen-9 target-cpu-32 ] webgpu:shader,execution,expression,binary,f32_arithmetic:multiplication:inputSource="uniform";vectorize="_undef_" [ Failure ]
 crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,binary,f32_arithmetic:remainder:inputSource="storage_r";vectorize="_undef_" [ Failure ]
 crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,binary,f32_arithmetic:remainder:inputSource="storage_r";vectorize=2 [ Failure ]
 crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,binary,f32_arithmetic:remainder:inputSource="storage_r";vectorize=3 [ Failure ]
@@ -404,7 +402,6 @@ crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,binary,f32_arit
 crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,binary,f32_arithmetic:remainder:inputSource="uniform";vectorize=2 [ Failure ]
 crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,binary,f32_arithmetic:remainder:inputSource="uniform";vectorize=3 [ Failure ]
 crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,binary,f32_arithmetic:remainder:inputSource="uniform";vectorize=4 [ Failure ]
-crbug.com/dawn/0000 [ intel-gen-9 win10 ] webgpu:shader,execution,expression,binary,f32_arithmetic:subtraction:inputSource="uniform";vectorize="_undef_" [ Failure ]
 crbug.com/dawn/0000 [ intel-gen-9 ubuntu ] webgpu:shader,execution,expression,call,builtin,countTrailingZeros:i32:inputSource="storage_r";vectorize="_undef_" [ Failure ]
 crbug.com/dawn/0000 [ intel-gen-9 ubuntu ] webgpu:shader,execution,expression,call,builtin,countTrailingZeros:i32:inputSource="storage_r";vectorize=2 [ Failure ]
 crbug.com/dawn/0000 [ intel-gen-9 ubuntu ] webgpu:shader,execution,expression,call,builtin,countTrailingZeros:i32:inputSource="storage_r";vectorize=3 [ Failure ]
@@ -429,11 +426,12 @@ crbug.com/dawn/0000 [ intel-gen-9 ubuntu ] webgpu:shader,execution,expression,ca
 crbug.com/dawn/0000 [ intel-gen-9 ubuntu ] webgpu:shader,execution,expression,call,builtin,countTrailingZeros:u32:inputSource="uniform";vectorize=2 [ Failure ]
 crbug.com/dawn/0000 [ intel-gen-9 ubuntu ] webgpu:shader,execution,expression,call,builtin,countTrailingZeros:u32:inputSource="uniform";vectorize=3 [ Failure ]
 crbug.com/dawn/0000 [ intel-gen-9 ubuntu ] webgpu:shader,execution,expression,call,builtin,countTrailingZeros:u32:inputSource="uniform";vectorize=4 [ Failure ]
-crbug.com/dawn/0000 [ dawn-backend-validation intel-gen-9 monterey target-cpu-64 ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec3:* [ Failure ]
+crbug.com/dawn/0000 [ monterey ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec3:inputSource="storage_r" [ Failure ]
-crbug.com/dawn/0000 [ dawn-no-backend-validation intel-gen-9 monterey target-cpu-64 ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec3:* [ Failure ]
+crbug.com/dawn/0000 [ monterey ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec3:inputSource="storage_rw" [ Failure ]
-crbug.com/dawn/0000 [ dawn-no-backend-validation nvidia-0x2184 target-cpu-32 win10 ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec3:inputSource="uniform" [ Failure ]
+crbug.com/dawn/0000 [ monterey ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec3:inputSource="uniform" [ Failure ]
-crbug.com/dawn/0000 [ intel-gen-9 monterey target-cpu-64 ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec4:* [ Failure ]
+crbug.com/dawn/0000 [ monterey ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec4:inputSource="storage_r" [ Failure ]
-crbug.com/dawn/0000 [ intel-gen-9 target-cpu-32 win10 ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec4:inputSource="uniform" [ Failure ]
+crbug.com/dawn/0000 [ monterey ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec4:inputSource="storage_rw" [ Failure ]
+crbug.com/dawn/0000 [ monterey ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec4:inputSource="uniform" [ Failure ]
 crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,extractBits:i32:inputSource="storage_r";width=1 [ Failure ]
 crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,extractBits:i32:inputSource="storage_r";width=2 [ Failure ]
 crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,extractBits:i32:inputSource="storage_r";width=3 [ Failure ]
@@ -506,22 +504,33 @@ crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,in
 crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,insertBits:integer:inputSource="uniform";signed=true;width=2 [ Failure ]
 crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,insertBits:integer:inputSource="uniform";signed=true;width=3 [ Failure ]
 crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,insertBits:integer:inputSource="uniform";signed=true;width=4 [ Failure ]
-crbug.com/dawn/0000 [ intel-gen-9 target-cpu-32 ] webgpu:shader,execution,expression,call,builtin,ldexp:f32:inputSource="uniform";vectorize="_undef_" [ Failure ]
+crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,length:f32_vec2:inputSource="const" [ Failure ]
-crbug.com/dawn/0000 [ intel-gen-9 target-cpu-32 ] webgpu:shader,execution,expression,call,builtin,max:f32:inputSource="uniform";vectorize="_undef_" [ Failure ]
+crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,length:f32_vec3:inputSource="const" [ Failure ]
-crbug.com/dawn/0000 [ intel-gen-9 target-cpu-32 ] webgpu:shader,execution,expression,call,builtin,min:f32:inputSource="uniform";vectorize="_undef_" [ Failure ]
+crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,length:f32_vec4:inputSource="const" [ Failure ]
-crbug.com/dawn/0000 [ intel-gen-9 target-cpu-32 ] webgpu:shader,execution,expression,call,builtin,pow:f32:inputSource="uniform";vectorize="_undef_" [ Failure ]
+crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,normalize:f32_vec2:inputSource="const" [ Failure ]
-crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="mixed";dispatch="indirect";groupSize={"x":1,"y":1,"z":1};numGroups={"x":1,"y":1,"z":1} [ Failure ]
+crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,normalize:f32_vec3:inputSource="const" [ Failure ]
-crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="mixed";dispatch="indirect";groupSize={"x":1,"y":1,"z":1};numGroups={"x":8,"y":4,"z":2} [ Failure ]
+crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,normalize:f32_vec4:inputSource="const" [ Failure ]
-crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="mixed";dispatch="indirect";groupSize={"x":3,"y":7,"z":5};numGroups={"x":13,"y":9,"z":11} [ Failure ]
+crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,quantizeToF16:* [ Failure ]
-crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="mixed";dispatch="indirect";groupSize={"x":8,"y":4,"z":2};numGroups={"x":1,"y":1,"z":1} [ Failure ]
+crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,saturate:f32:inputSource="const";vectorize="_undef_" [ Failure ]
-crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="param";dispatch="indirect";groupSize={"x":1,"y":1,"z":1};numGroups={"x":1,"y":1,"z":1} [ Failure ]
+crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,saturate:f32:inputSource="const";vectorize=2 [ Failure ]
-crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="param";dispatch="indirect";groupSize={"x":1,"y":1,"z":1};numGroups={"x":8,"y":4,"z":2} [ Failure ]
+crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,saturate:f32:inputSource="const";vectorize=3 [ Failure ]
-crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="param";dispatch="indirect";groupSize={"x":3,"y":7,"z":5};numGroups={"x":13,"y":9,"z":11} [ Failure ]
+crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,saturate:f32:inputSource="const";vectorize=4 [ Failure ]
-crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="param";dispatch="indirect";groupSize={"x":8,"y":4,"z":2};numGroups={"x":1,"y":1,"z":1} [ Failure ]
+crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="storage_r";vectorize="_undef_" [ Failure ]
-crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="struct";dispatch="indirect";groupSize={"x":1,"y":1,"z":1};numGroups={"x":1,"y":1,"z":1} [ Failure ]
+crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="storage_r";vectorize=2 [ Failure ]
-crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="struct";dispatch="indirect";groupSize={"x":1,"y":1,"z":1};numGroups={"x":8,"y":4,"z":2} [ Failure ]
+crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="storage_r";vectorize=3 [ Failure ]
-crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="struct";dispatch="indirect";groupSize={"x":3,"y":7,"z":5};numGroups={"x":13,"y":9,"z":11} [ Failure ]
+crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="storage_r";vectorize=4 [ Failure ]
-crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="struct";dispatch="indirect";groupSize={"x":8,"y":4,"z":2};numGroups={"x":1,"y":1,"z":1} [ Failure ]
+crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="storage_rw";vectorize="_undef_" [ Failure ]
+crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="storage_rw";vectorize=2 [ Failure ]
+crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="storage_rw";vectorize=3 [ Failure ]
+crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="storage_rw";vectorize=4 [ Failure ]
+crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="uniform";vectorize="_undef_" [ Failure ]
+crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="uniform";vectorize=2 [ Failure ]
+crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="uniform";vectorize=3 [ Failure ]
+crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="uniform";vectorize=4 [ Failure ]
+crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,smoothstep:f32:inputSource="const";vectorize="_undef_" [ Failure ]
+crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,smoothstep:f32:inputSource="const";vectorize=2 [ Failure ]
+crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,smoothstep:f32:inputSource="const";vectorize=3 [ Failure ]
+crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,smoothstep:f32:inputSource="const";vectorize=4 [ Failure ]
 
 ################################################################################
 # webgpu:shader,execution,memory_model
@@ -617,13 +626,8 @@ crbug.com/dawn/0000 [ win ] webgpu:api,operation,render_pipeline,pipeline_output
 crbug.com/dawn/0000 [ mac ] webgpu:web_platform,canvas,readbackFromWebGPUCanvas:* [ Failure ]
 crbug.com/dawn/0000 [ mac ] worker_webgpu:api,operation,render_pass,storeOp:render_pass_store_op,depth_stencil_attachment_only:depthStencilFormat="stencil8";* [ Failure ]
 
-################################################################################
-# slow tests
-# KEEP
-################################################################################
-crbug.com/dawn/0000 [ win ] webgpu:shader,execution,expression,binary,f32_logical:* [ Slow ]
-
 # New failures. Please triage:
+crbug.com/dawn/0000 [ monterey ] webgpu:api,operation,command_buffer,copyTextureToTexture:color_textures,non_compressed,non_array:srcFormat="rgb9e5ufloat";dstFormat="rgb9e5ufloat";dimension="1d" [ Failure ]
 crbug.com/dawn/0000 [ intel-gen-9 win10 ] webgpu:api,operation,command_buffer,image_copy:mip_levels:initMethod="WriteTexture";checkMethod="PartialCopyT2B";format="rgba32uint";dimension="2d" [ Failure ]
 crbug.com/dawn/0000 [ win10 ] webgpu:api,validation,createBindGroup:buffer,resource_binding_size:type="read-only-storage" [ Failure ]
 crbug.com/dawn/0000 [ win10 ] webgpu:api,validation,createBindGroup:buffer,resource_binding_size:type="storage" [ Failure ]
@@ -650,35 +654,12 @@ crbug.com/dawn/0000 webgpu:api,validation,render_pass,render_pass_descriptor:tim
 crbug.com/dawn/0000 webgpu:api,validation,render_pass,render_pass_descriptor:timestampWrites,query_set_type:queryTypeA="timestamp";queryTypeB="occlusion" [ Failure ]
 crbug.com/dawn/0000 webgpu:api,validation,render_pass,render_pass_descriptor:timestamp_writes_location:locationA="beginning";locationB="beginning" [ Failure ]
 crbug.com/dawn/0000 webgpu:api,validation,render_pass,render_pass_descriptor:timestamp_writes_location:locationA="end";locationB="end" [ Failure ]
-crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,length:f32_vec2:inputSource="const" [ Failure ]
+crbug.com/dawn/0000 [ monterey ] webgpu:web_platform,canvas,configure:alpha_mode:* [ Failure ]
-crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,length:f32_vec3:inputSource="const" [ Failure ]
+crbug.com/dawn/0000 [ monterey ] webgpu:web_platform,canvas,configure:defaults:* [ Failure ]
-crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,length:f32_vec4:inputSource="const" [ Failure ]
+crbug.com/dawn/0000 [ monterey ] webgpu:web_platform,canvas,configure:device:* [ Failure ]
-crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,normalize:f32_vec2:inputSource="const" [ Failure ]
+crbug.com/dawn/0000 [ monterey ] webgpu:web_platform,canvas,configure:format:canvasType="offscreen";format="rgba8unorm" [ Failure ]
-crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,normalize:f32_vec3:inputSource="const" [ Failure ]
+crbug.com/dawn/0000 [ monterey ] webgpu:web_platform,canvas,configure:format:canvasType="onscreen";format="rgba8unorm" [ Failure ]
-crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,normalize:f32_vec4:inputSource="const" [ Failure ]
-crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,quantizeToF16:* [ Failure ]
-crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,saturate:f32:inputSource="const";vectorize="_undef_" [ Failure ]
-crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,saturate:f32:inputSource="const";vectorize=2 [ Failure ]
-crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,saturate:f32:inputSource="const";vectorize=3 [ Failure ]
-crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,saturate:f32:inputSource="const";vectorize=4 [ Failure ]
-crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:* [ Failure ]
-crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,smoothstep:f32:inputSource="const";vectorize="_undef_" [ Failure ]
-crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,smoothstep:f32:inputSource="const";vectorize=2 [ Failure ]
-crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,smoothstep:f32:inputSource="const";vectorize=3 [ Failure ]
-crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,smoothstep:f32:inputSource="const";vectorize=4 [ Failure ]
-crbug.com/dawn/0000 [ intel-gen-9 target-cpu-32 ] webgpu:shader,execution,expression,call,builtin,step:f32:inputSource="uniform";vectorize="_undef_" [ Failure ]
-crbug.com/dawn/0000 [ monterey webgpu-adapter-default ] webgpu:web_platform,canvas,configure:alpha_mode:* [ Failure ]
-crbug.com/dawn/0000 [ monterey webgpu-adapter-swiftshader ] webgpu:web_platform,canvas,configure:alpha_mode:* [ Failure ]
-crbug.com/dawn/0000 [ monterey webgpu-adapter-default ] webgpu:web_platform,canvas,configure:defaults:* [ Failure ]
-crbug.com/dawn/0000 [ monterey webgpu-adapter-swiftshader ] webgpu:web_platform,canvas,configure:defaults:* [ Failure ]
-crbug.com/dawn/0000 [ monterey webgpu-adapter-default ] webgpu:web_platform,canvas,configure:device:* [ Failure ]
-crbug.com/dawn/0000 [ monterey webgpu-adapter-swiftshader ] webgpu:web_platform,canvas,configure:device:* [ Failure ]
-crbug.com/dawn/0000 [ monterey webgpu-adapter-default ] webgpu:web_platform,canvas,configure:format:canvasType="offscreen";format="rgba8unorm" [ Failure ]
-crbug.com/dawn/0000 [ monterey webgpu-adapter-swiftshader ] webgpu:web_platform,canvas,configure:format:canvasType="offscreen";format="rgba8unorm" [ Failure ]
-crbug.com/dawn/0000 [ monterey webgpu-adapter-default ] webgpu:web_platform,canvas,configure:format:canvasType="onscreen";format="rgba8unorm" [ Failure ]
-crbug.com/dawn/0000 [ monterey webgpu-adapter-swiftshader ] webgpu:web_platform,canvas,configure:format:canvasType="onscreen";format="rgba8unorm" [ Failure ]
 crbug.com/dawn/0000 [ monterey webgpu-adapter-default ] webgpu:web_platform,canvas,configure:usage:* [ Failure ]
 crbug.com/dawn/0000 [ monterey webgpu-adapter-swiftshader ] webgpu:web_platform,canvas,configure:usage:* [ Failure ]
 crbug.com/dawn/0000 [ ubuntu webgpu-adapter-default ] webgpu:web_platform,canvas,configure:usage:* [ Failure ]
-crbug.com/dawn/0000 [ monterey webgpu-adapter-default ] webgpu:web_platform,canvas,getCurrentTexture:* [ Failure ]
+crbug.com/dawn/0000 [ monterey ] webgpu:web_platform,canvas,getCurrentTexture:* [ Failure ]
-crbug.com/dawn/0000 [ monterey webgpu-adapter-swiftshader ] webgpu:web_platform,canvas,getCurrentTexture:* [ Failure ]