From aad2e9c0b57f6911d4e1ecd4d709b1f08656bf27 Mon Sep 17 00:00:00 2001 From: Ben Clayton Date: Sat, 17 Sep 2022 19:30:29 +0000 Subject: [PATCH] tools/cts: Add `cts validate`, improvements & fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit • Add `cts validate` command used to check for expectation collisions. Can be used as a presubmit check. This is more tightly checked than the previous logic, as this works on just the expectations, instead of results. • Fix an issue where the test result reduction could introduce collisions with 'Skip' expectations. To fix this, the update process first adds 'consumed' results for the skipped tests, preventing test tree reduction for that part of the tree. • Fix a bug in the generation of 'New failures' and 'New flakes' which produced more expectations than was necessary. The issue here was that the tree roots could contain overlaps, and roots could be processed before sub-trees, resulting in inefficient expectations. • Fix collisions in the expectations file, and update with results from the most recent roll. Change-Id: I7b64553408998fb4416458ce564fc49c8f6d4d07 Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/101860 Commit-Queue: Ben Clayton Kokoro: Kokoro Reviewed-by: Austin Eng --- tools/src/cmd/cts/common/paths.go | 14 ++ tools/src/cmd/cts/format/format.go | 2 +- tools/src/cmd/cts/main.go | 1 + tools/src/cmd/cts/roll/roll.go | 13 +- tools/src/cmd/cts/update/update.go | 33 ++- tools/src/cmd/cts/validate/validate.go | 66 ++++++ tools/src/cts/expectations/diagnostic.go | 22 ++ tools/src/cts/expectations/update.go | 118 ++++++----- tools/src/cts/expectations/update_test.go | 183 +++------------- tools/src/cts/expectations/validate.go | 77 +++++++ tools/src/cts/expectations/validate_test.go | 130 ++++++++++++ tools/src/cts/query/tree.go | 33 ++- tools/src/cts/query/tree_test.go | 218 +++++++++++++++++++- webgpu-cts/expectations.txt | 103 ++++----- 14 files changed, 739 insertions(+), 274 deletions(-) create mode 100644 tools/src/cmd/cts/validate/validate.go create mode 100644 tools/src/cts/expectations/validate.go create mode 100644 tools/src/cts/expectations/validate_test.go diff --git a/tools/src/cmd/cts/common/paths.go b/tools/src/cmd/cts/common/paths.go index fda05ed20d..9cd88e11ac 100644 --- a/tools/src/cmd/cts/common/paths.go +++ b/tools/src/cmd/cts/common/paths.go @@ -25,6 +25,10 @@ const ( // RelativeExpectationsPath is the dawn-root relative path to the // expectations.txt file. RelativeExpectationsPath = "webgpu-cts/expectations.txt" + + // RelativeTestListPath is the dawn-root relative path to the + // test_list.txt file. + RelativeTestListPath = "third_party/gn/webgpu-cts/test_list.txt" ) // DefaultExpectationsPath returns the default path to the expectations.txt @@ -36,3 +40,13 @@ func DefaultExpectationsPath() string { } return path } + +// DefaultTestListPath returns the default path to the test_list.txt +// file. Returns an empty string if the file cannot be found. +func DefaultTestListPath() string { + path := filepath.Join(utils.DawnRoot(), RelativeTestListPath) + if _, err := os.Stat(path); err != nil { + return "" + } + return path +} diff --git a/tools/src/cmd/cts/format/format.go b/tools/src/cmd/cts/format/format.go index bb2aa702de..b15b2f9d05 100644 --- a/tools/src/cmd/cts/format/format.go +++ b/tools/src/cmd/cts/format/format.go @@ -37,7 +37,7 @@ func (cmd) Name() string { } func (cmd) Desc() string { - return "formats a WebGPUExpectation file" + return "formats a WebGPU expectations.txt file" } func (c *cmd) RegisterFlags(ctx context.Context, cfg common.Config) ([]string, error) { diff --git a/tools/src/cmd/cts/main.go b/tools/src/cmd/cts/main.go index 4f90c2ac6b..871b70c620 100644 --- a/tools/src/cmd/cts/main.go +++ b/tools/src/cmd/cts/main.go @@ -35,6 +35,7 @@ import ( _ "dawn.googlesource.com/dawn/tools/src/cmd/cts/roll" _ "dawn.googlesource.com/dawn/tools/src/cmd/cts/time" _ "dawn.googlesource.com/dawn/tools/src/cmd/cts/update" + _ "dawn.googlesource.com/dawn/tools/src/cmd/cts/validate" ) func main() { diff --git a/tools/src/cmd/cts/roll/roll.go b/tools/src/cmd/cts/roll/roll.go index 69bebd1f6c..48322aa0cd 100644 --- a/tools/src/cmd/cts/roll/roll.go +++ b/tools/src/cmd/cts/roll/roll.go @@ -34,6 +34,7 @@ import ( "dawn.googlesource.com/dawn/tools/src/cmd/cts/common" "dawn.googlesource.com/dawn/tools/src/container" "dawn.googlesource.com/dawn/tools/src/cts/expectations" + "dawn.googlesource.com/dawn/tools/src/cts/query" "dawn.googlesource.com/dawn/tools/src/cts/result" "dawn.googlesource.com/dawn/tools/src/fileutils" "dawn.googlesource.com/dawn/tools/src/gerrit" @@ -246,6 +247,16 @@ func (r *roller) roll(ctx context.Context) error { return err } + // Pull out the test list from the generated files + testlist := func() []query.Query { + lines := strings.Split(generatedFiles[testListRelPath], "\n") + list := make([]query.Query, len(lines)) + for i, line := range lines { + list[i] = query.Parse(line) + } + return list + }() + deletedFiles := []string{} if currentWebTestFiles, err := r.dawn.ListFiles(ctx, dawnHash, webTestsPath); err != nil { // If there's an error, allow NotFound. It means the directory did not exist, so no files @@ -348,7 +359,7 @@ func (r *roller) roll(ctx context.Context) error { // Note: The new expectations are not used if the last attempt didn't // fail, but we always want to post the diagnostics newExpectations := ex.Clone() - diags, err := newExpectations.Update(results) + diags, err := newExpectations.Update(results, testlist) if err != nil { return err } diff --git a/tools/src/cmd/cts/update/update.go b/tools/src/cmd/cts/update/update.go index 159215603b..ae242c7e48 100644 --- a/tools/src/cmd/cts/update/update.go +++ b/tools/src/cmd/cts/update/update.go @@ -18,9 +18,13 @@ import ( "context" "flag" "fmt" + "io/ioutil" + "os" + "strings" "dawn.googlesource.com/dawn/tools/src/cmd/cts/common" "dawn.googlesource.com/dawn/tools/src/cts/expectations" + "dawn.googlesource.com/dawn/tools/src/cts/query" "dawn.googlesource.com/dawn/tools/src/cts/result" "go.chromium.org/luci/auth/client/authcli" ) @@ -53,6 +57,19 @@ func (c *cmd) RegisterFlags(ctx context.Context, cfg common.Config) ([]string, e return nil, nil } +func loadTestList(path string) ([]query.Query, error) { + data, err := ioutil.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to load test list: %w", err) + } + lines := strings.Split(string(data), "\n") + out := make([]query.Query, len(lines)) + for i, l := range lines { + out[i] = query.Parse(l) + } + return out, nil +} + func (c *cmd) Run(ctx context.Context, cfg common.Config) error { // Validate command line arguments auth, err := c.flags.auth.Options() @@ -75,16 +92,24 @@ func (c *cmd) Run(ctx context.Context, cfg common.Config) error { return err } + testlist, err := loadTestList(common.DefaultTestListPath()) + if err != nil { + return err + } + + if diag := ex.Validate(); diag.NumErrors() > 0 { + diag.Print(os.Stdout, c.flags.expectations) + return fmt.Errorf("validation failed") + } + // Update the expectations file with the results - msgs, err := ex.Update(results) + diag, err := ex.Update(results, testlist) if err != nil { return err } // Print any diagnostics - for _, msg := range msgs { - fmt.Printf("%v:%v %v\n", c.flags.expectations, msg.Line, msg.Message) - } + diag.Print(os.Stdout, c.flags.expectations) // Save the updated expectations file return ex.Save(c.flags.expectations) diff --git a/tools/src/cmd/cts/validate/validate.go b/tools/src/cmd/cts/validate/validate.go new file mode 100644 index 0000000000..fb4d436551 --- /dev/null +++ b/tools/src/cmd/cts/validate/validate.go @@ -0,0 +1,66 @@ +// Copyright 2022 The Dawn Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package validate + +import ( + "context" + "flag" + "fmt" + "os" + + "dawn.googlesource.com/dawn/tools/src/cmd/cts/common" + "dawn.googlesource.com/dawn/tools/src/cts/expectations" +) + +func init() { + common.Register(&cmd{}) +} + +type cmd struct { + flags struct { + expectations string // expectations file path + } +} + +func (cmd) Name() string { + return "validate" +} + +func (cmd) Desc() string { + return "validates a WebGPU expectations.txt file" +} + +func (c *cmd) RegisterFlags(ctx context.Context, cfg common.Config) ([]string, error) { + defaultExpectations := common.DefaultExpectationsPath() + flag.StringVar(&c.flags.expectations, "expectations", defaultExpectations, "path to CTS expectations file to update") + return nil, nil +} + +func (c *cmd) Run(ctx context.Context, cfg common.Config) error { + ex, err := expectations.Load(c.flags.expectations) + if err != nil { + return err + } + diags := ex.Validate() + + // Print any diagnostics + diags.Print(os.Stdout, c.flags.expectations) + if numErrs := diags.NumErrors(); numErrs > 0 { + return fmt.Errorf("%v errors found", numErrs) + } + + fmt.Println("no issues found") + return nil +} diff --git a/tools/src/cts/expectations/diagnostic.go b/tools/src/cts/expectations/diagnostic.go index c7059aa105..03baa28479 100644 --- a/tools/src/cts/expectations/diagnostic.go +++ b/tools/src/cts/expectations/diagnostic.go @@ -16,6 +16,7 @@ package expectations import ( "fmt" + "io" "strings" ) @@ -54,3 +55,24 @@ func (e Diagnostic) String() string { // Error implements the 'error' interface. func (e Diagnostic) Error() string { return e.String() } + +// Diagnostics is a list of diagnostic +type Diagnostics []Diagnostic + +// NumErrors returns number of errors in the diagnostics +func (l Diagnostics) NumErrors() int { + count := 0 + for _, d := range l { + if d.Severity == Error { + count++ + } + } + return count +} + +// Print prints the list of diagnostics to 'w' +func (l Diagnostics) Print(w io.Writer, path string) { + for _, d := range l { + fmt.Fprintf(w, "%v:%v %v\n", path, d.Line, d.Message) + } +} diff --git a/tools/src/cts/expectations/update.go b/tools/src/cts/expectations/update.go index 98a2a80139..bb72d8c794 100644 --- a/tools/src/cts/expectations/update.go +++ b/tools/src/cts/expectations/update.go @@ -37,7 +37,11 @@ import ( // existing expectation lines. // // Update returns a list of diagnostics for things that should be addressed. -func (c *Content) Update(results result.List) ([]Diagnostic, error) { +// +// Note: Validate() should be called before attempting to update the +// expectations. If Validate() returns errors, then Update() behaviour is +// undefined. +func (c *Content) Update(results result.List, testlist []query.Query) (Diagnostics, error) { // Make a copy of the results. This code mutates the list. results = append(result.List{}, results...) @@ -53,13 +57,23 @@ func (c *Content) Update(results result.List) ([]Diagnostic, error) { tagSets[len(tagSets)-i-1] = s.Tags } - // Update those expectations! + // Scan the full result list to obtain all the test variants + // (unique tag combinations). + variants := results.Variants() + + // Add 'consumed' results for tests that were skipped. + // This ensures that skipped results are not included in reduced trees. + results = c.appendConsumedResultsForSkippedTests(results, testlist, variants) + u := updater{ - in: *c, - out: Content{}, - qt: newQueryTree(results), - tagSets: tagSets, + in: *c, + out: Content{}, + qt: newQueryTree(results), + variants: variants, + tagSets: tagSets, } + + // Update those expectations! if err := u.build(); err != nil { return nil, fmt.Errorf("while updating expectations: %w", err) } @@ -70,11 +84,45 @@ func (c *Content) Update(results result.List) ([]Diagnostic, error) { // updater holds the state used for updating the expectations type updater struct { - in Content // the original expectations Content - out Content // newly built expectations Content - qt queryTree // the query tree - diags []Diagnostic // diagnostics raised during update - tagSets []result.Tags // reverse-ordered tag-sets of 'in' + in Content // the original expectations Content + out Content // newly built expectations Content + qt queryTree // the query tree + variants []container.Set[string] + diags []Diagnostic // diagnostics raised during update + tagSets []result.Tags // reverse-ordered tag-sets of 'in' +} + +// Returns 'results' with additional 'consumed' results for tests that have +// 'Skip' expectations. This fills in gaps for results, preventing tree +// reductions from marking skipped results as failure, which could result in +// expectation collisions. +func (c *Content) appendConsumedResultsForSkippedTests(results result.List, + testlist []query.Query, + variants []container.Set[string]) result.List { + tree := query.Tree[struct{}]{} + for _, q := range testlist { + tree.Add(q, struct{}{}) + } + for _, c := range c.Chunks { + for _, ex := range c.Expectations { + if container.NewSet(ex.Status...).Contains(string(result.Skip)) { + for _, variant := range variants { + if !variant.ContainsAll(ex.Tags) { + continue + } + glob, _ := tree.Glob(query.Parse(ex.Query)) + for _, qd := range glob { + results = append(results, result.Result{ + Query: qd.Query, + Tags: variant, + Status: consumed, + }) + } + } + } + } + } + return results } // simplifyStatuses replaces all result statuses that are not one of @@ -160,14 +208,9 @@ func (qt *queryTree) glob(q query.Query) (result.List, error) { return out, nil } -// globAndCheckForCollisions returns the list of results matching the given tags -// under (or with) the given query. -// globAndCheckForCollisions will return an error if any of the results are -// already consumed by a non-zero line. The non-zero line distinguishes between -// results consumed by expectations declared in the input (non-zero line), vs -// those that were introduced by the update (zero line). We only want to error -// if there's a collision in user declared expectations. -func (qt *queryTree) globAndCheckForCollisions(q query.Query, t result.Tags) (result.List, error) { +// globTags returns the list of results matching the given tags under (or with) +// the given query. +func (qt *queryTree) globTags(q query.Query, t result.Tags) (result.List, error) { glob, err := qt.tree.Glob(q) if err != nil { return nil, err @@ -177,12 +220,6 @@ func (qt *queryTree) globAndCheckForCollisions(q query.Query, t result.Tags) (re for _, indices := range glob { for _, idx := range indices.Data { if r := qt.results[idx]; r.Tags.ContainsAll(t) { - if at := qt.consumedAt[idx]; at > 0 { - if len(t) > 0 { - return nil, fmt.Errorf("%v %v collides with expectation at line %v", t, q, at) - } - return nil, fmt.Errorf("%v collides with expectation at line %v", q, at) - } out = append(out, r) } } @@ -194,7 +231,7 @@ func (qt *queryTree) globAndCheckForCollisions(q query.Query, t result.Tags) (re // under (or with) the given query, as consumed. // line is used to record the line at which the results were consumed. If the // results were consumed as part of generating new expectations then line should -// be 0. See queryTree.globAndCheckForCollisions(). +// be 0. func (qt *queryTree) markAsConsumed(q query.Query, t result.Tags, line int) { if glob, err := qt.tree.Glob(q); err == nil { for _, indices := range glob { @@ -274,21 +311,12 @@ func (u *updater) chunk(in Chunk) Chunk { return out } -// chunk returns a new list of Expectations, based on the Expectation 'in', +// expectation returns a new list of Expectations, based on the Expectation 'in', // using the new result data. func (u *updater) expectation(in Expectation, keep bool) []Expectation { // noResults is a helper for returning when the expectation has no test // results. - // If the expectation has an expected 'Skip' result, then we're likely - // to be missing results (as the test was not run). In this situation - // the expectation is preserved, and no diagnostics are raised. - // If the expectation did not have a 'Skip' result, then a diagnostic will - // be raised and the expectation will be removed. noResults := func() []Expectation { - if container.NewSet(in.Status...).Contains(string(result.Skip)) { - return []Expectation{in} - } - // Expectation does not have a 'Skip' result. if len(in.Tags) > 0 { u.diag(Warning, in.Line, "no results found for '%v' with tags %v", in.Query, in.Tags) } else { @@ -298,12 +326,11 @@ func (u *updater) expectation(in Expectation, keep bool) []Expectation { return []Expectation{} } - // Grab all the results that match the expectation's query q := query.Parse(in.Query) // Glob the results for the expectation's query + tag combination. // Ensure that none of these are already consumed. - results, err := u.qt.globAndCheckForCollisions(q, in.Tags) + results, err := u.qt.globTags(q, in.Tags) // If we can't find any results for this query + tag combination, then bail. switch { case errors.As(err, &query.ErrNoDataForQuery{}): @@ -357,18 +384,14 @@ func (u *updater) expectation(in Expectation, keep bool) []Expectation { // addNewExpectations (potentially) appends to 'u.out' chunks for new flaky and // failing tests. func (u *updater) addNewExpectations() error { - // Scan the full result list to obtain all the test variants - // (unique tag combinations). - allVariants := u.qt.results.Variants() - // For each variant: // • Build a query tree using the results filtered to the variant, and then // reduce the tree. // • Take all the reduced-tree leaf nodes, and add these to 'roots'. // Once we've collected all the roots, we'll use these to build the // expectations across the reduced set of tags. - roots := container.NewMap[string, query.Query]() - for _, variant := range allVariants { + roots := query.Tree[bool]{} + for _, variant := range u.variants { // Build a tree from the results matching the given variant. tree, err := u.qt.results.FilterByVariant(variant).StatusTree() if err != nil { @@ -378,15 +401,16 @@ func (u *updater) addNewExpectations() error { tree.Reduce(treeReducer) // Add all the reduced leaf nodes to 'roots'. for _, qd := range tree.List() { - roots.Add(qd.Query.String(), qd.Query) + // Use Split() to ensure that only the leaves have data (true) in the tree + roots.Split(qd.Query, true) } } // Build all the expectations for each of the roots. expectations := []Expectation{} - for _, root := range roots.Values() { + for _, root := range roots.List() { expectations = append(expectations, u.expectationsForRoot( - root, // Root query + root.Query, // Root query 0, // Line number "crbug.com/dawn/0000", // Bug "", // Comment diff --git a/tools/src/cts/expectations/update_test.go b/tools/src/cts/expectations/update_test.go index 081ea11f97..6b4da10819 100644 --- a/tools/src/cts/expectations/update_test.go +++ b/tools/src/cts/expectations/update_test.go @@ -18,6 +18,7 @@ import ( "strings" "testing" + "dawn.googlesource.com/dawn/tools/src/container" "dawn.googlesource.com/dawn/tools/src/cts/expectations" "dawn.googlesource.com/dawn/tools/src/cts/query" "dawn.googlesource.com/dawn/tools/src/cts/result" @@ -41,7 +42,7 @@ func TestUpdate(t *testing.T) { expectations string results result.List updated string - diagnostics []expectations.Diagnostic + diagnostics expectations.Diagnostics err string } for _, test := range []Test{ @@ -73,7 +74,7 @@ some:other,test:* [ Failure ] updated: ` some:other,test:* [ Failure ] `, - diagnostics: []expectations.Diagnostic{ + diagnostics: expectations.Diagnostics{ { Severity: expectations.Warning, Line: headerLines + 2, @@ -109,7 +110,7 @@ some:other,test:* [ Failure ] updated: ` some:other,test:* [ Failure ] `, - diagnostics: []expectations.Diagnostic{ + diagnostics: expectations.Diagnostics{ { Severity: expectations.Warning, Line: headerLines + 3, @@ -117,55 +118,6 @@ some:other,test:* [ Failure ] }, }, }, - { ////////////////////////////////////////////////////////////////////// - name: "no results found Skip", - expectations: ` -crbug.com/a/123 a:missing,test,result:* [ Skip ] - -some:other,test:* [ Failure ] -`, - results: result.List{ - result.Result{ - Query: Q("some:other,test:*"), - Tags: result.NewTags("os-a", "gpu-a"), - Status: result.Failure, - }, - result.Result{ - Query: Q("some:other,test:*"), - Tags: result.NewTags("os-b", "gpu-b"), - Status: result.Failure, - }, - }, - updated: ` -crbug.com/a/123 a:missing,test,result:* [ Skip ] - -some:other,test:* [ Failure ] -`, - }, - { ////////////////////////////////////////////////////////////////////// - name: "simple expectation collision", - expectations: ` -a:b,c:* [ Failure ] -a:b,c:* [ Failure ] -`, - results: result.List{ - result.Result{ - Query: Q("a:b,c:d"), - Tags: result.NewTags("os-a", "os-c", "gpu-b"), - Status: result.Failure, - }, - }, - updated: ` -a:b,c:* [ Failure ] -`, - diagnostics: []expectations.Diagnostic{ - { - Severity: expectations.Error, - Line: headerLines + 3, - Message: "a:b,c:* collides with expectation at line 8", - }, - }, - }, { ////////////////////////////////////////////////////////////////////// name: "simple expectation with tags", expectations: ` @@ -182,100 +134,6 @@ a:b,c:* [ Failure ] updated: ` a:b,c:* [ Failure ] `, - diagnostics: []expectations.Diagnostic{ - { - Severity: expectations.Error, - Line: headerLines + 3, - Message: "[gpu-b] a:b,c:* collides with expectation at line 8", - }, - }, - }, - { ////////////////////////////////////////////////////////////////////// - name: "simple expectation collision KEEP", - expectations: ` -# KEEP -a:b,c:* [ Failure ] -a:b,c:* [ Failure ] -`, - results: result.List{ - result.Result{ - Query: Q("a:b,c:d"), - Tags: result.NewTags("os-a", "os-c", "gpu-b"), - Status: result.Failure, - }, - }, - updated: ` -# KEEP -a:b,c:* [ Failure ] -`, - diagnostics: []expectations.Diagnostic{ - { - Severity: expectations.Error, - Line: headerLines + 4, - Message: "a:b,c:* collides with expectation at line 9", - }, - }, - }, - { ////////////////////////////////////////////////////////////////////// - name: "collision with child-expectation", - expectations: ` -a:b:x:* [ Failure ] -a:b:* [ Failure ] -`, - results: result.List{ - result.Result{ - Query: Q("a:b:x:*"), - Tags: result.NewTags("os-a", "os-c", "gpu-b"), - Status: result.Failure, - }, - result.Result{ - Query: Q("a:b:y:*"), - Tags: result.NewTags("os-a", "os-c", "gpu-b"), - Status: result.Failure, - }, - }, - updated: ` -a:b:x:* [ Failure ] - -# New failures. Please triage: -crbug.com/dawn/0000 a:b:y:* [ Failure ] -`, - diagnostics: []expectations.Diagnostic{ - { - Severity: expectations.Error, - Line: headerLines + 3, - Message: "a:b:* collides with expectation at line 8", - }, - }, - }, - { ////////////////////////////////////////////////////////////////////// - name: "collision with parent-expectation", - expectations: ` -a:b:* [ Failure ] -a:b:x:* [ Failure ] -`, - results: result.List{ - result.Result{ - Query: Q("a:b:x:*"), - Tags: result.NewTags("os-a", "os-c", "gpu-b"), - Status: result.Failure, - }, - result.Result{ - Query: Q("a:b:y:*"), - Tags: result.NewTags("os-a", "os-c", "gpu-b"), - Status: result.Failure, - }, - }, - updated: ` -a:b:* [ Failure ] -`, - diagnostics: []expectations.Diagnostic{ - { - Severity: expectations.Error, - Line: headerLines + 3, - Message: "a:b:x:* collides with expectation at line 8", - }, - }, }, { ////////////////////////////////////////////////////////////////////// name: "expectation test now passes", @@ -345,7 +203,7 @@ crbug.com/a/123 [ gpu-b os-b ] a:b,c:d [ Failure ] crbug.com/a/123 [ gpu-a os-a ] a:b,c:d [ Failure ] crbug.com/a/123 [ gpu-b os-b ] a:b,c:d [ Failure ] `, - diagnostics: []expectations.Diagnostic{ + diagnostics: expectations.Diagnostics{ { Severity: expectations.Note, Line: headerLines + 3, @@ -369,7 +227,7 @@ crbug.com/a/123 a:b,c:d:* [ Failure ] # KEEP crbug.com/a/123 a:b,c:d:* [ Failure ] `, - diagnostics: []expectations.Diagnostic{ + diagnostics: expectations.Diagnostics{ { Severity: expectations.Note, Line: headerLines + 3, @@ -391,6 +249,11 @@ crbug.com/a/123 a:b,c:d:* [ Failure ] Tags: result.NewTags("os-a", "gpu-b"), Status: result.Abort, }, + result.Result{ + Query: Q("suite:dir_a,dir_b:test_c:case=4;*"), + Tags: result.NewTags("os-a", "gpu-a"), + Status: result.Crash, + }, result.Result{ Query: Q("suite:dir_a,dir_b:test_c:case=4;*"), Tags: result.NewTags("os-b", "gpu-b"), @@ -402,22 +265,22 @@ crbug.com/a/123 a:b,c:d:* [ Failure ] Status: result.RetryOnFailure, }, result.Result{ - Query: Q("suite:dir_a,dir_b:test_b;case=5;*"), + Query: Q("suite:dir_a,dir_b:test_b:case=5;*"), Tags: result.NewTags("os-b", "gpu-b"), Status: result.Pass, }, result.Result{ - Query: Q("suite:dir_a,dir_b:test_b:*"), + Query: Q("suite:dir_a,dir_b:test_b:case=6;*"), Tags: result.NewTags("os-a", "gpu-a"), Status: result.Slow, }, result.Result{ - Query: Q("suite:dir_a,dir_b:test_b:*"), + Query: Q("suite:dir_a,dir_b:test_b:case=6;*"), Tags: result.NewTags("os-b", "gpu-a"), Status: result.Pass, }, result.Result{ - Query: Q("suite:dir_a,dir_b:test_c:*"), + Query: Q("suite:dir_a,dir_b:test_c:case=6;*"), Tags: result.NewTags("os-a", "gpu-a"), Status: result.RetryOnFailure, }, @@ -425,14 +288,13 @@ crbug.com/a/123 a:b,c:d:* [ Failure ] updated: `# A comment # New flakes. Please triage: -crbug.com/dawn/0000 [ gpu-a os-a ] suite:dir_a,dir_b:test_c:* [ RetryOnFailure ] -crbug.com/dawn/0000 [ gpu-b os-b ] suite:dir_a,dir_b:test_c:case=5;* [ RetryOnFailure ] +crbug.com/dawn/0000 suite:dir_a,dir_b:test_c:case=5;* [ RetryOnFailure ] +crbug.com/dawn/0000 suite:dir_a,dir_b:test_c:case=6;* [ RetryOnFailure ] # New failures. Please triage: -crbug.com/dawn/0000 [ gpu-b os-a ] suite:* [ Failure ] -crbug.com/dawn/0000 [ gpu-a os-a ] suite:dir_a,dir_b:test_a:* [ Failure ] +crbug.com/dawn/0000 suite:dir_a,dir_b:test_a:* [ Failure ] crbug.com/dawn/0000 [ gpu-a os-a ] suite:dir_a,dir_b:test_b:* [ Slow ] -crbug.com/dawn/0000 [ gpu-b os-b ] suite:dir_a,dir_b:test_c:case=4;* [ Failure ] +crbug.com/dawn/0000 suite:dir_a,dir_b:test_c:case=4;* [ Failure ] `, }, { ////////////////////////////////////////////////////////////////////// @@ -650,8 +512,13 @@ crbug.com/dawn/0000 a:b,c:t29:* [ Failure ] t.Fatalf("'%v': expectations.Parse():\n%v", test.name, err) } + testList := container.NewMap[string, query.Query]() + for _, r := range test.results { + testList.Add(r.Query.String(), r.Query) + } + errMsg := "" - diagnostics, err := ex.Update(test.results) + diagnostics, err := ex.Update(test.results, testList.Values()) if err != nil { errMsg = err.Error() } diff --git a/tools/src/cts/expectations/validate.go b/tools/src/cts/expectations/validate.go new file mode 100644 index 0000000000..bb4e6e6415 --- /dev/null +++ b/tools/src/cts/expectations/validate.go @@ -0,0 +1,77 @@ +// Copyright 2022 The Dawn Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package expectations provides types and helpers for parsing, updating and +// writing WebGPU expectations files. +// +// See /webgpu-cts/expectations.txt for more information. +package expectations + +import ( + "fmt" + + "dawn.googlesource.com/dawn/tools/src/container" + "dawn.googlesource.com/dawn/tools/src/cts/query" +) + +func (c Content) tagsCollide(a, b container.Set[string]) bool { + for _, set := range c.Tags.Sets { + aSet := a.Intersection(set.Tags) + bSet := b.Intersection(set.Tags) + if len(aSet) != 0 && len(bSet) != 0 && len(aSet.Intersection(bSet)) == 0 { + return false + } + } + return true +} + +// Validate checks that the expectations do not contain errors +func (c Content) Validate() Diagnostics { + tree, _ := query.NewTree[Expectations]() + for _, chunk := range c.Chunks { + for _, ex := range chunk.Expectations { + node := tree.GetOrCreate(query.Parse(ex.Query), func() Expectations { + return Expectations{} + }) + *node = append(*node, ex) + } + } + var out Diagnostics + for _, chunk := range c.Chunks { + for _, ex := range chunk.Expectations { + glob, err := tree.Glob(query.Parse(ex.Query)) + if err != nil { + out = append(out, Diagnostic{ + Severity: Error, + Line: ex.Line, + Message: err.Error(), + }) + continue + } + for _, qd := range glob { + expectations := qd.Data + for _, other := range expectations { + if other.Line != ex.Line && c.tagsCollide(ex.Tags, other.Tags) { + out = append(out, Diagnostic{ + Severity: Error, + Line: ex.Line, + Message: fmt.Sprintf("expectation collides with expectation on line %v", other.Line), + }) + } + } + } + } + } + return out +} diff --git a/tools/src/cts/expectations/validate_test.go b/tools/src/cts/expectations/validate_test.go new file mode 100644 index 0000000000..1be5082cdb --- /dev/null +++ b/tools/src/cts/expectations/validate_test.go @@ -0,0 +1,130 @@ +// Copyright 2022 The Dawn Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package expectations_test + +import ( + "testing" + + "dawn.googlesource.com/dawn/tools/src/cts/expectations" + "github.com/google/go-cmp/cmp" +) + +func TestValidate(t *testing.T) { + header := `# BEGIN TAG HEADER +# OS +# tags: [ os-a os-b os-c ] +# GPU +# tags: [ gpu-a gpu-b gpu-c ] +# END TAG HEADER +` + + type Test struct { + name string + expectations string + diagnostics expectations.Diagnostics + } + for _, test := range []Test{ + { ////////////////////////////////////////////////////////////////////// + name: "empty", + expectations: ``, + }, + { ////////////////////////////////////////////////////////////////////// + name: "simple", + expectations: ` +crbug.com/a/123 a:b,c:d,* [ Failure ] +`, + }, + { ////////////////////////////////////////////////////////////////////// + name: "no-tag collision", + expectations: ` +crbug.com/a/123 a:b,c:d,* [ Failure ] +crbug.com/a/123 a:x,x:d,* [ Failure ] +crbug.com/a/123 a:b,c:d,* [ Failure ] +`, + diagnostics: expectations.Diagnostics{ + { + Line: 8, + Severity: expectations.Error, + Message: "expectation collides with expectation on line 10", + }, + { + Line: 10, + Severity: expectations.Error, + Message: "expectation collides with expectation on line 8", + }, + }, + }, + { ////////////////////////////////////////////////////////////////////// + name: "tag collision", + expectations: ` +crbug.com/a/123 [ os-a ] a:b,c:d,* [ Failure ] +crbug.com/a/123 a:x,x:d,* [ Failure ] +crbug.com/a/123 [ os-a ] a:b,c:d,* [ Failure ] +`, + diagnostics: expectations.Diagnostics{ + { + Line: 8, + Severity: expectations.Error, + Message: "expectation collides with expectation on line 10", + }, + { + Line: 10, + Severity: expectations.Error, + Message: "expectation collides with expectation on line 8", + }, + }, + }, + { ////////////////////////////////////////////////////////////////////// + name: "nested no-tag collision", + expectations: ` +crbug.com/a/123 a:b,c:d,e:* [ Failure ] +crbug.com/a/123 a:x,x:d,* [ Failure ] +crbug.com/a/123 a:b,c:d,* [ Failure ] +`, + diagnostics: expectations.Diagnostics{ + { + Line: 10, + Severity: expectations.Error, + Message: "expectation collides with expectation on line 8", + }, + }, + }, + { ////////////////////////////////////////////////////////////////////// + name: "tag collision", + expectations: ` +crbug.com/a/123 [ os-a ] a:b,c:d,e:* [ Failure ] +crbug.com/a/123 a:x,x:d,* [ Failure ] +crbug.com/a/123 [ os-a ] a:b,c:d,* [ Failure ] +`, + diagnostics: expectations.Diagnostics{ + { + Line: 10, + Severity: expectations.Error, + Message: "expectation collides with expectation on line 8", + }, + }, + }, + } { + ex, err := expectations.Parse(header + test.expectations) + if err != nil { + t.Fatalf("'%v': expectations.Parse():\n%v", test.name, err) + } + + diagnostics := ex.Validate() + if diff := cmp.Diff(diagnostics, test.diagnostics); diff != "" { + t.Errorf("'%v': expectations.Update() error:\n%v", test.name, diff) + } + } +} diff --git a/tools/src/cts/query/tree.go b/tools/src/cts/query/tree.go index b52d993e1e..d6733b5c3d 100644 --- a/tools/src/cts/query/tree.go +++ b/tools/src/cts/query/tree.go @@ -252,7 +252,7 @@ func NewTree[Data any](entries ...QueryData[Data]) (Tree[Data], error) { } // Add adds a new data to the tree. -// Returns ErrDuplicateData if the tree already contains a data for the given +// Returns ErrDuplicateData if the tree already contains a data for the given node at query func (t *Tree[Data]) Add(q Query, d Data) error { node := &t.TreeNode q.Walk(func(q Query, t Target, n string) error { @@ -266,6 +266,37 @@ func (t *Tree[Data]) Add(q Query, d Data) error { return nil } +// Split adds a new data to the tree, clearing any ancestor node's data. +// Returns ErrDuplicateData if the tree already contains a data for the given node at query +func (t *Tree[Data]) Split(q Query, d Data) error { + node := &t.TreeNode + q.Walk(func(q Query, t Target, n string) error { + delete(node.Children, TreeNodeChildKey{Name: "*", Target: t}) + node.Data = nil + node = node.getOrCreateChild(TreeNodeChildKey{n, t}) + return nil + }) + if node.Data != nil { + return ErrDuplicateData{node.Query} + } + node.Data = &d + return nil +} + +// GetOrCreate returns existing, or adds a new data to the tree. +func (t *Tree[Data]) GetOrCreate(q Query, create func() Data) *Data { + node := &t.TreeNode + q.Walk(func(q Query, t Target, n string) error { + node = node.getOrCreateChild(TreeNodeChildKey{n, t}) + return nil + }) + if node.Data == nil { + data := create() + node.Data = &data + } + return node.Data +} + // Reduce reduces the tree using the Merger function f. // If the Merger function returns a non-nil Data value, then this will be used // to replace the non-leaf node with a new leaf node holding the returned Data. diff --git a/tools/src/cts/query/tree_test.go b/tools/src/cts/query/tree_test.go index 80af4756a2..760dca6310 100644 --- a/tools/src/cts/query/tree_test.go +++ b/tools/src/cts/query/tree_test.go @@ -238,7 +238,6 @@ func TestNewSingle(t *testing.T) { t.Errorf("NewTree(%v) tree was not as expected:\n%v", test.in, diff) } } - } func TestNewMultiple(t *testing.T) { @@ -351,6 +350,223 @@ func TestNewWithCollision(t *testing.T) { } } +func TestSplit(t *testing.T) { + type Tree = query.Tree[string] + type Node = query.TreeNode[string] + type QueryData = query.QueryData[string] + type Children = query.TreeNodeChildren[string] + + type Test struct { + in QueryData + pre Tree + post Tree + } + for _, test := range []Test{ + { ///////////////////////////////////////////////////////////////////// + in: QueryData{ + Query: Q(`suite:*`), + Data: pass, + }, + post: Tree{ + TreeNode: Node{ + Children: Children{ + query.TreeNodeChildKey{`suite`, query.Suite}: { + Query: Q(`suite`), + Children: Children{ + query.TreeNodeChildKey{`*`, query.Files}: { + Query: Q(`suite:*`), + Data: &pass, + }, + }, + }, + }, + }, + }, + }, + { ///////////////////////////////////////////////////////////////////// + in: QueryData{ + Query: Q(`suite:a,b:*`), + Data: pass, + }, + pre: Tree{ + TreeNode: Node{ + Children: Children{ + query.TreeNodeChildKey{`suite`, query.Suite}: { + Query: Q(`suite`), + Children: Children{ + query.TreeNodeChildKey{`a`, query.Files}: { + Query: Q(`suite:a`), + Children: Children{ + query.TreeNodeChildKey{`*`, query.Files}: { + Query: Q(`suite:a,*`), + Data: &pass, + }, + }, + }, + }, + }, + }, + }, + }, + post: Tree{ + TreeNode: Node{ + Children: Children{ + query.TreeNodeChildKey{`suite`, query.Suite}: { + Query: Q(`suite`), + Children: Children{ + query.TreeNodeChildKey{`a`, query.Files}: { + Query: Q(`suite:a`), + Children: Children{ + query.TreeNodeChildKey{`b`, query.Files}: { + Query: Q(`suite:a,b`), + Children: Children{ + query.TreeNodeChildKey{`*`, query.Tests}: { + Query: Q(`suite:a,b:*`), + Data: &pass, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { ///////////////////////////////////////////////////////////////////// + in: QueryData{ + Query: Q(`suite:a:*`), + Data: pass, + }, + pre: Tree{ + TreeNode: Node{ + Children: Children{ + query.TreeNodeChildKey{`suite`, query.Suite}: { + Query: Q(`suite`), + Children: Children{ + query.TreeNodeChildKey{`a`, query.Files}: { + Query: Q(`suite:a`), + Children: Children{ + query.TreeNodeChildKey{`b`, query.Files}: { + Query: Q(`suite:a,b`), + Children: Children{ + query.TreeNodeChildKey{`*`, query.Files}: { + Query: Q(`suite:a,b,*`), + Data: &pass, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + post: Tree{ + TreeNode: Node{ + Children: Children{ + query.TreeNodeChildKey{`suite`, query.Suite}: { + Query: Q(`suite`), + Children: Children{ + query.TreeNodeChildKey{`a`, query.Files}: { + Query: Q(`suite:a`), + Children: Children{ + query.TreeNodeChildKey{`*`, query.Tests}: { + Query: Q(`suite:a:*`), + Data: &pass, + }, + query.TreeNodeChildKey{`b`, query.Files}: { + Query: Q(`suite:a,b`), + Children: Children{ + query.TreeNodeChildKey{`*`, query.Files}: { + Query: Q(`suite:a,b,*`), + Data: &pass, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { ///////////////////////////////////////////////////////////////////// + in: QueryData{ + Query: Q(`suite:a,b:c:*`), + Data: pass, + }, + pre: Tree{ + TreeNode: Node{ + Children: Children{ + query.TreeNodeChildKey{`suite`, query.Suite}: { + Query: Q(`suite`), + Children: Children{ + query.TreeNodeChildKey{`a`, query.Files}: { + Query: Q(`suite:a`), + Children: Children{ + query.TreeNodeChildKey{`b`, query.Files}: { + Query: Q(`suite:a,b`), + Children: Children{ + query.TreeNodeChildKey{`*`, query.Tests}: { + Query: Q(`suite:a,b:*`), + Data: &pass, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + post: Tree{ + TreeNode: Node{ + Children: Children{ + query.TreeNodeChildKey{`suite`, query.Suite}: { + Query: Q(`suite`), + Children: Children{ + query.TreeNodeChildKey{`a`, query.Files}: { + Query: Q(`suite:a`), + Children: Children{ + query.TreeNodeChildKey{`b`, query.Files}: { + Query: Q(`suite:a,b`), + Children: Children{ + query.TreeNodeChildKey{`c`, query.Tests}: { + Query: Q(`suite:a,b:c`), + Children: Children{ + query.TreeNodeChildKey{`*`, query.Cases}: { + Query: Q(`suite:a,b:c:*`), + Data: &pass, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } { + tree := test.pre + if err := tree.Split(test.in.Query, test.in.Data); err != nil { + t.Errorf("NewTree(%v): %v", test.in, err) + continue + } + if diff := cmp.Diff(tree, test.post); diff != "" { + t.Errorf("Split(%v) tree was not as expected:\n%v", test.in, diff) + } + } +} + func TestList(t *testing.T) { type QueryData = query.QueryData[string] diff --git a/webgpu-cts/expectations.txt b/webgpu-cts/expectations.txt index 03a497e0bd..c5b140295f 100644 --- a/webgpu-cts/expectations.txt +++ b/webgpu-cts/expectations.txt @@ -79,7 +79,8 @@ crbug.com/dawn/1487 [ intel-gen-9 win10 ] webgpu:api,operation,resource_init,tex # Flaky failures using SwiftShader (crbug.com/1344876). # KEEP ################################################################################ -crbug.com/1343969 [ win ] webgpu:web_platform,copyToTexture,ImageBitmap:* [ RetryOnFailure Slow ] +crbug.com/1343969 [ win ] webgpu:web_platform,copyToTexture,ImageBitmap:from_ImageData:* [ RetryOnFailure Slow ] +crbug.com/1343969 [ webgpu-adapter-default win ] webgpu:web_platform,copyToTexture,ImageBitmap:from_canvas:* [ RetryOnFailure Slow ] crbug.com/1344876 [ ubuntu webgpu-adapter-swiftshader ] webgpu:web_platform,copyToTexture,ImageBitmap:from_ImageData:* [ RetryOnFailure Slow ] crbug.com/1359106 [ ubuntu webgpu-adapter-default ] webgpu:web_platform,copyToTexture,ImageBitmap:from_ImageData:* [ RetryOnFailure ] @@ -389,9 +390,6 @@ crbug.com/tint/1613 webgpu:shader,execution,expression,unary,f32_arithmetic:nega ################################################################################ # shader execution failures ################################################################################ -crbug.com/dawn/0000 [ intel-gen-9 target-cpu-32 ] webgpu:shader,execution,expression,binary,f32_arithmetic:addition:inputSource="uniform";vectorize="_undef_" [ Failure ] -crbug.com/dawn/0000 [ intel-gen-9 target-cpu-32 ] webgpu:shader,execution,expression,binary,f32_arithmetic:division:inputSource="uniform";vectorize="_undef_" [ Failure ] -crbug.com/dawn/0000 [ intel-gen-9 target-cpu-32 ] webgpu:shader,execution,expression,binary,f32_arithmetic:multiplication:inputSource="uniform";vectorize="_undef_" [ Failure ] crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,binary,f32_arithmetic:remainder:inputSource="storage_r";vectorize="_undef_" [ Failure ] crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,binary,f32_arithmetic:remainder:inputSource="storage_r";vectorize=2 [ Failure ] crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,binary,f32_arithmetic:remainder:inputSource="storage_r";vectorize=3 [ Failure ] @@ -404,7 +402,6 @@ crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,binary,f32_arit crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,binary,f32_arithmetic:remainder:inputSource="uniform";vectorize=2 [ Failure ] crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,binary,f32_arithmetic:remainder:inputSource="uniform";vectorize=3 [ Failure ] crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,binary,f32_arithmetic:remainder:inputSource="uniform";vectorize=4 [ Failure ] -crbug.com/dawn/0000 [ intel-gen-9 win10 ] webgpu:shader,execution,expression,binary,f32_arithmetic:subtraction:inputSource="uniform";vectorize="_undef_" [ Failure ] crbug.com/dawn/0000 [ intel-gen-9 ubuntu ] webgpu:shader,execution,expression,call,builtin,countTrailingZeros:i32:inputSource="storage_r";vectorize="_undef_" [ Failure ] crbug.com/dawn/0000 [ intel-gen-9 ubuntu ] webgpu:shader,execution,expression,call,builtin,countTrailingZeros:i32:inputSource="storage_r";vectorize=2 [ Failure ] crbug.com/dawn/0000 [ intel-gen-9 ubuntu ] webgpu:shader,execution,expression,call,builtin,countTrailingZeros:i32:inputSource="storage_r";vectorize=3 [ Failure ] @@ -429,11 +426,12 @@ crbug.com/dawn/0000 [ intel-gen-9 ubuntu ] webgpu:shader,execution,expression,ca crbug.com/dawn/0000 [ intel-gen-9 ubuntu ] webgpu:shader,execution,expression,call,builtin,countTrailingZeros:u32:inputSource="uniform";vectorize=2 [ Failure ] crbug.com/dawn/0000 [ intel-gen-9 ubuntu ] webgpu:shader,execution,expression,call,builtin,countTrailingZeros:u32:inputSource="uniform";vectorize=3 [ Failure ] crbug.com/dawn/0000 [ intel-gen-9 ubuntu ] webgpu:shader,execution,expression,call,builtin,countTrailingZeros:u32:inputSource="uniform";vectorize=4 [ Failure ] -crbug.com/dawn/0000 [ dawn-backend-validation intel-gen-9 monterey target-cpu-64 ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec3:* [ Failure ] -crbug.com/dawn/0000 [ dawn-no-backend-validation intel-gen-9 monterey target-cpu-64 ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec3:* [ Failure ] -crbug.com/dawn/0000 [ dawn-no-backend-validation nvidia-0x2184 target-cpu-32 win10 ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec3:inputSource="uniform" [ Failure ] -crbug.com/dawn/0000 [ intel-gen-9 monterey target-cpu-64 ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec4:* [ Failure ] -crbug.com/dawn/0000 [ intel-gen-9 target-cpu-32 win10 ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec4:inputSource="uniform" [ Failure ] +crbug.com/dawn/0000 [ monterey ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec3:inputSource="storage_r" [ Failure ] +crbug.com/dawn/0000 [ monterey ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec3:inputSource="storage_rw" [ Failure ] +crbug.com/dawn/0000 [ monterey ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec3:inputSource="uniform" [ Failure ] +crbug.com/dawn/0000 [ monterey ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec4:inputSource="storage_r" [ Failure ] +crbug.com/dawn/0000 [ monterey ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec4:inputSource="storage_rw" [ Failure ] +crbug.com/dawn/0000 [ monterey ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec4:inputSource="uniform" [ Failure ] crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,extractBits:i32:inputSource="storage_r";width=1 [ Failure ] crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,extractBits:i32:inputSource="storage_r";width=2 [ Failure ] crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,extractBits:i32:inputSource="storage_r";width=3 [ Failure ] @@ -506,22 +504,33 @@ crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,in crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,insertBits:integer:inputSource="uniform";signed=true;width=2 [ Failure ] crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,insertBits:integer:inputSource="uniform";signed=true;width=3 [ Failure ] crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,insertBits:integer:inputSource="uniform";signed=true;width=4 [ Failure ] -crbug.com/dawn/0000 [ intel-gen-9 target-cpu-32 ] webgpu:shader,execution,expression,call,builtin,ldexp:f32:inputSource="uniform";vectorize="_undef_" [ Failure ] -crbug.com/dawn/0000 [ intel-gen-9 target-cpu-32 ] webgpu:shader,execution,expression,call,builtin,max:f32:inputSource="uniform";vectorize="_undef_" [ Failure ] -crbug.com/dawn/0000 [ intel-gen-9 target-cpu-32 ] webgpu:shader,execution,expression,call,builtin,min:f32:inputSource="uniform";vectorize="_undef_" [ Failure ] -crbug.com/dawn/0000 [ intel-gen-9 target-cpu-32 ] webgpu:shader,execution,expression,call,builtin,pow:f32:inputSource="uniform";vectorize="_undef_" [ Failure ] -crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="mixed";dispatch="indirect";groupSize={"x":1,"y":1,"z":1};numGroups={"x":1,"y":1,"z":1} [ Failure ] -crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="mixed";dispatch="indirect";groupSize={"x":1,"y":1,"z":1};numGroups={"x":8,"y":4,"z":2} [ Failure ] -crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="mixed";dispatch="indirect";groupSize={"x":3,"y":7,"z":5};numGroups={"x":13,"y":9,"z":11} [ Failure ] -crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="mixed";dispatch="indirect";groupSize={"x":8,"y":4,"z":2};numGroups={"x":1,"y":1,"z":1} [ Failure ] -crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="param";dispatch="indirect";groupSize={"x":1,"y":1,"z":1};numGroups={"x":1,"y":1,"z":1} [ Failure ] -crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="param";dispatch="indirect";groupSize={"x":1,"y":1,"z":1};numGroups={"x":8,"y":4,"z":2} [ Failure ] -crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="param";dispatch="indirect";groupSize={"x":3,"y":7,"z":5};numGroups={"x":13,"y":9,"z":11} [ Failure ] -crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="param";dispatch="indirect";groupSize={"x":8,"y":4,"z":2};numGroups={"x":1,"y":1,"z":1} [ Failure ] -crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="struct";dispatch="indirect";groupSize={"x":1,"y":1,"z":1};numGroups={"x":1,"y":1,"z":1} [ Failure ] -crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="struct";dispatch="indirect";groupSize={"x":1,"y":1,"z":1};numGroups={"x":8,"y":4,"z":2} [ Failure ] -crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="struct";dispatch="indirect";groupSize={"x":3,"y":7,"z":5};numGroups={"x":13,"y":9,"z":11} [ Failure ] -crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="struct";dispatch="indirect";groupSize={"x":8,"y":4,"z":2};numGroups={"x":1,"y":1,"z":1} [ Failure ] +crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,length:f32_vec2:inputSource="const" [ Failure ] +crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,length:f32_vec3:inputSource="const" [ Failure ] +crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,length:f32_vec4:inputSource="const" [ Failure ] +crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,normalize:f32_vec2:inputSource="const" [ Failure ] +crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,normalize:f32_vec3:inputSource="const" [ Failure ] +crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,normalize:f32_vec4:inputSource="const" [ Failure ] +crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,quantizeToF16:* [ Failure ] +crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,saturate:f32:inputSource="const";vectorize="_undef_" [ Failure ] +crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,saturate:f32:inputSource="const";vectorize=2 [ Failure ] +crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,saturate:f32:inputSource="const";vectorize=3 [ Failure ] +crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,saturate:f32:inputSource="const";vectorize=4 [ Failure ] +crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="storage_r";vectorize="_undef_" [ Failure ] +crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="storage_r";vectorize=2 [ Failure ] +crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="storage_r";vectorize=3 [ Failure ] +crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="storage_r";vectorize=4 [ Failure ] +crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="storage_rw";vectorize="_undef_" [ Failure ] +crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="storage_rw";vectorize=2 [ Failure ] +crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="storage_rw";vectorize=3 [ Failure ] +crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="storage_rw";vectorize=4 [ Failure ] +crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="uniform";vectorize="_undef_" [ Failure ] +crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="uniform";vectorize=2 [ Failure ] +crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="uniform";vectorize=3 [ Failure ] +crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="uniform";vectorize=4 [ Failure ] +crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,smoothstep:f32:inputSource="const";vectorize="_undef_" [ Failure ] +crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,smoothstep:f32:inputSource="const";vectorize=2 [ Failure ] +crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,smoothstep:f32:inputSource="const";vectorize=3 [ Failure ] +crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,smoothstep:f32:inputSource="const";vectorize=4 [ Failure ] ################################################################################ # webgpu:shader,execution,memory_model @@ -617,13 +626,8 @@ crbug.com/dawn/0000 [ win ] webgpu:api,operation,render_pipeline,pipeline_output crbug.com/dawn/0000 [ mac ] webgpu:web_platform,canvas,readbackFromWebGPUCanvas:* [ Failure ] crbug.com/dawn/0000 [ mac ] worker_webgpu:api,operation,render_pass,storeOp:render_pass_store_op,depth_stencil_attachment_only:depthStencilFormat="stencil8";* [ Failure ] -################################################################################ -# slow tests -# KEEP -################################################################################ -crbug.com/dawn/0000 [ win ] webgpu:shader,execution,expression,binary,f32_logical:* [ Slow ] - # New failures. Please triage: +crbug.com/dawn/0000 [ monterey ] webgpu:api,operation,command_buffer,copyTextureToTexture:color_textures,non_compressed,non_array:srcFormat="rgb9e5ufloat";dstFormat="rgb9e5ufloat";dimension="1d" [ Failure ] crbug.com/dawn/0000 [ intel-gen-9 win10 ] webgpu:api,operation,command_buffer,image_copy:mip_levels:initMethod="WriteTexture";checkMethod="PartialCopyT2B";format="rgba32uint";dimension="2d" [ Failure ] crbug.com/dawn/0000 [ win10 ] webgpu:api,validation,createBindGroup:buffer,resource_binding_size:type="read-only-storage" [ Failure ] crbug.com/dawn/0000 [ win10 ] webgpu:api,validation,createBindGroup:buffer,resource_binding_size:type="storage" [ Failure ] @@ -650,35 +654,12 @@ crbug.com/dawn/0000 webgpu:api,validation,render_pass,render_pass_descriptor:tim crbug.com/dawn/0000 webgpu:api,validation,render_pass,render_pass_descriptor:timestampWrites,query_set_type:queryTypeA="timestamp";queryTypeB="occlusion" [ Failure ] crbug.com/dawn/0000 webgpu:api,validation,render_pass,render_pass_descriptor:timestamp_writes_location:locationA="beginning";locationB="beginning" [ Failure ] crbug.com/dawn/0000 webgpu:api,validation,render_pass,render_pass_descriptor:timestamp_writes_location:locationA="end";locationB="end" [ Failure ] -crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,length:f32_vec2:inputSource="const" [ Failure ] -crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,length:f32_vec3:inputSource="const" [ Failure ] -crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,length:f32_vec4:inputSource="const" [ Failure ] -crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,normalize:f32_vec2:inputSource="const" [ Failure ] -crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,normalize:f32_vec3:inputSource="const" [ Failure ] -crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,normalize:f32_vec4:inputSource="const" [ Failure ] -crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,quantizeToF16:* [ Failure ] -crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,saturate:f32:inputSource="const";vectorize="_undef_" [ Failure ] -crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,saturate:f32:inputSource="const";vectorize=2 [ Failure ] -crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,saturate:f32:inputSource="const";vectorize=3 [ Failure ] -crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,saturate:f32:inputSource="const";vectorize=4 [ Failure ] -crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:* [ Failure ] -crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,smoothstep:f32:inputSource="const";vectorize="_undef_" [ Failure ] -crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,smoothstep:f32:inputSource="const";vectorize=2 [ Failure ] -crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,smoothstep:f32:inputSource="const";vectorize=3 [ Failure ] -crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,smoothstep:f32:inputSource="const";vectorize=4 [ Failure ] -crbug.com/dawn/0000 [ intel-gen-9 target-cpu-32 ] webgpu:shader,execution,expression,call,builtin,step:f32:inputSource="uniform";vectorize="_undef_" [ Failure ] -crbug.com/dawn/0000 [ monterey webgpu-adapter-default ] webgpu:web_platform,canvas,configure:alpha_mode:* [ Failure ] -crbug.com/dawn/0000 [ monterey webgpu-adapter-swiftshader ] webgpu:web_platform,canvas,configure:alpha_mode:* [ Failure ] -crbug.com/dawn/0000 [ monterey webgpu-adapter-default ] webgpu:web_platform,canvas,configure:defaults:* [ Failure ] -crbug.com/dawn/0000 [ monterey webgpu-adapter-swiftshader ] webgpu:web_platform,canvas,configure:defaults:* [ Failure ] -crbug.com/dawn/0000 [ monterey webgpu-adapter-default ] webgpu:web_platform,canvas,configure:device:* [ Failure ] -crbug.com/dawn/0000 [ monterey webgpu-adapter-swiftshader ] webgpu:web_platform,canvas,configure:device:* [ Failure ] -crbug.com/dawn/0000 [ monterey webgpu-adapter-default ] webgpu:web_platform,canvas,configure:format:canvasType="offscreen";format="rgba8unorm" [ Failure ] -crbug.com/dawn/0000 [ monterey webgpu-adapter-swiftshader ] webgpu:web_platform,canvas,configure:format:canvasType="offscreen";format="rgba8unorm" [ Failure ] -crbug.com/dawn/0000 [ monterey webgpu-adapter-default ] webgpu:web_platform,canvas,configure:format:canvasType="onscreen";format="rgba8unorm" [ Failure ] -crbug.com/dawn/0000 [ monterey webgpu-adapter-swiftshader ] webgpu:web_platform,canvas,configure:format:canvasType="onscreen";format="rgba8unorm" [ Failure ] +crbug.com/dawn/0000 [ monterey ] webgpu:web_platform,canvas,configure:alpha_mode:* [ Failure ] +crbug.com/dawn/0000 [ monterey ] webgpu:web_platform,canvas,configure:defaults:* [ Failure ] +crbug.com/dawn/0000 [ monterey ] webgpu:web_platform,canvas,configure:device:* [ Failure ] +crbug.com/dawn/0000 [ monterey ] webgpu:web_platform,canvas,configure:format:canvasType="offscreen";format="rgba8unorm" [ Failure ] +crbug.com/dawn/0000 [ monterey ] webgpu:web_platform,canvas,configure:format:canvasType="onscreen";format="rgba8unorm" [ Failure ] crbug.com/dawn/0000 [ monterey webgpu-adapter-default ] webgpu:web_platform,canvas,configure:usage:* [ Failure ] crbug.com/dawn/0000 [ monterey webgpu-adapter-swiftshader ] webgpu:web_platform,canvas,configure:usage:* [ Failure ] crbug.com/dawn/0000 [ ubuntu webgpu-adapter-default ] webgpu:web_platform,canvas,configure:usage:* [ Failure ] -crbug.com/dawn/0000 [ monterey webgpu-adapter-default ] webgpu:web_platform,canvas,getCurrentTexture:* [ Failure ] -crbug.com/dawn/0000 [ monterey webgpu-adapter-swiftshader ] webgpu:web_platform,canvas,getCurrentTexture:* [ Failure ] +crbug.com/dawn/0000 [ monterey ] webgpu:web_platform,canvas,getCurrentTexture:* [ Failure ]