encounter
/
dawn-cmake
mirror of https://github.com/encounter/dawn-cmake.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

tools/cts: Add `cts validate`, improvements & fixes 

• Add `cts validate` command used to check for expectation collisions.
  Can be used as a presubmit check.
  This is more tightly checked than the previous logic, as this works on just
  the expectations, instead of results.

• Fix an issue where the test result reduction could introduce collisions with
  'Skip' expectations.
  To fix this, the update process first adds 'consumed' results for the skipped
  tests, preventing test tree reduction for that part of the tree.

• Fix a bug in the generation of 'New failures' and 'New flakes' which produced
  more expectations than was necessary.
  The issue here was that the tree roots could contain overlaps, and roots could
  be processed before sub-trees, resulting in inefficient expectations.

• Fix collisions in the expectations file, and update with results from
  the most recent roll.

Change-Id: I7b64553408998fb4416458ce564fc49c8f6d4d07
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/101860
Commit-Queue: Ben Clayton <bclayton@google.com>
Kokoro: Kokoro <noreply+kokoro@google.com>
Reviewed-by: Austin Eng <enga@chromium.org>
This commit is contained in:
Ben Clayton 2022-09-17 19:30:29 +00:00 committed by Dawn LUCI CQ
parent 9ab81268ff
commit aad2e9c0b5
14 changed files with 739 additions and 274 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
tools/src/cmd/cts/common/paths.go
View File

@ -25,6 +25,10 @@ const (
// RelativeExpectationsPath is the dawn-root relative path to the
// expectations.txt file.
RelativeExpectationsPath = "webgpu-cts/expectations.txt"
// RelativeTestListPath is the dawn-root relative path to the
// test_list.txt file.
RelativeTestListPath = "third_party/gn/webgpu-cts/test_list.txt"
)
// DefaultExpectationsPath returns the default path to the expectations.txt
@ -36,3 +40,13 @@ func DefaultExpectationsPath() string {
}
return path
}
// DefaultTestListPath returns the default path to the test_list.txt
// file. Returns an empty string if the file cannot be found.
func DefaultTestListPath() string {
path := filepath.Join(utils.DawnRoot(), RelativeTestListPath)
if _, err := os.Stat(path); err != nil {
return ""
}
return path
}

2
tools/src/cmd/cts/format/format.go
View File

@ -37,7 +37,7 @@ func (cmd) Name() string {
}
func (cmd) Desc() string {
return "formats a WebGPUExpectation file"
return "formats a WebGPU expectations.txt file"
}
func (c *cmd) RegisterFlags(ctx context.Context, cfg common.Config) ([]string, error) {

1
tools/src/cmd/cts/main.go
View File

@ -35,6 +35,7 @@ import (
_ "dawn.googlesource.com/dawn/tools/src/cmd/cts/roll"
_ "dawn.googlesource.com/dawn/tools/src/cmd/cts/time"
_ "dawn.googlesource.com/dawn/tools/src/cmd/cts/update"
_ "dawn.googlesource.com/dawn/tools/src/cmd/cts/validate"
)
func main() {

13
tools/src/cmd/cts/roll/roll.go
View File

@ -34,6 +34,7 @@ import (
"dawn.googlesource.com/dawn/tools/src/cmd/cts/common"
"dawn.googlesource.com/dawn/tools/src/container"
"dawn.googlesource.com/dawn/tools/src/cts/expectations"
"dawn.googlesource.com/dawn/tools/src/cts/query"
"dawn.googlesource.com/dawn/tools/src/cts/result"
"dawn.googlesource.com/dawn/tools/src/fileutils"
"dawn.googlesource.com/dawn/tools/src/gerrit"
@ -246,6 +247,16 @@ func (r *roller) roll(ctx context.Context) error {
return err
}
// Pull out the test list from the generated files
testlist := func() []query.Query {
lines := strings.Split(generatedFiles[testListRelPath], "\n")
list := make([]query.Query, len(lines))
for i, line := range lines {
list[i] = query.Parse(line)
}
return list
}()
deletedFiles := []string{}
if currentWebTestFiles, err := r.dawn.ListFiles(ctx, dawnHash, webTestsPath); err != nil {
// If there's an error, allow NotFound. It means the directory did not exist, so no files
@ -348,7 +359,7 @@ func (r *roller) roll(ctx context.Context) error {
// Note: The new expectations are not used if the last attempt didn't
// fail, but we always want to post the diagnostics
newExpectations := ex.Clone()
diags, err := newExpectations.Update(results)
diags, err := newExpectations.Update(results, testlist)
if err != nil {
return err
}

33
tools/src/cmd/cts/update/update.go
View File

@ -18,9 +18,13 @@ import (
"context"
"flag"
"fmt"
"io/ioutil"
"os"
"strings"
"dawn.googlesource.com/dawn/tools/src/cmd/cts/common"
"dawn.googlesource.com/dawn/tools/src/cts/expectations"
"dawn.googlesource.com/dawn/tools/src/cts/query"
"dawn.googlesource.com/dawn/tools/src/cts/result"
"go.chromium.org/luci/auth/client/authcli"
)
@ -53,6 +57,19 @@ func (c *cmd) RegisterFlags(ctx context.Context, cfg common.Config) ([]string, e
return nil, nil
}
func loadTestList(path string) ([]query.Query, error) {
data, err := ioutil.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("failed to load test list: %w", err)
}
lines := strings.Split(string(data), "\n")
out := make([]query.Query, len(lines))
for i, l := range lines {
out[i] = query.Parse(l)
}
return out, nil
}
func (c *cmd) Run(ctx context.Context, cfg common.Config) error {
// Validate command line arguments
auth, err := c.flags.auth.Options()
@ -75,16 +92,24 @@ func (c *cmd) Run(ctx context.Context, cfg common.Config) error {
return err
}
testlist, err := loadTestList(common.DefaultTestListPath())
if err != nil {
return err
}
if diag := ex.Validate(); diag.NumErrors() > 0 {
diag.Print(os.Stdout, c.flags.expectations)
return fmt.Errorf("validation failed")
}
// Update the expectations file with the results
msgs, err := ex.Update(results)
diag, err := ex.Update(results, testlist)
if err != nil {
return err
}
// Print any diagnostics
for _, msg := range msgs {
fmt.Printf("%v:%v %v\n", c.flags.expectations, msg.Line, msg.Message)
}
diag.Print(os.Stdout, c.flags.expectations)
// Save the updated expectations file
return ex.Save(c.flags.expectations)

66
tools/src/cmd/cts/validate/validate.go Normal file
View File

@ -0,0 +1,66 @@
// Copyright 2022 The Dawn Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package validate
import (
"context"
"flag"
"fmt"
"os"
"dawn.googlesource.com/dawn/tools/src/cmd/cts/common"
"dawn.googlesource.com/dawn/tools/src/cts/expectations"
)
func init() {
common.Register(&cmd{})
}
type cmd struct {
flags struct {
expectations string // expectations file path
}
}
func (cmd) Name() string {
return "validate"
}
func (cmd) Desc() string {
return "validates a WebGPU expectations.txt file"
}
func (c *cmd) RegisterFlags(ctx context.Context, cfg common.Config) ([]string, error) {
defaultExpectations := common.DefaultExpectationsPath()
flag.StringVar(&c.flags.expectations, "expectations", defaultExpectations, "path to CTS expectations file to update")
return nil, nil
}
func (c *cmd) Run(ctx context.Context, cfg common.Config) error {
ex, err := expectations.Load(c.flags.expectations)
if err != nil {
return err
}
diags := ex.Validate()
// Print any diagnostics
diags.Print(os.Stdout, c.flags.expectations)
if numErrs := diags.NumErrors(); numErrs > 0 {
return fmt.Errorf("%v errors found", numErrs)
}
fmt.Println("no issues found")
return nil
}

22
tools/src/cts/expectations/diagnostic.go
View File

@ -16,6 +16,7 @@ package expectations
import (
"fmt"
"io"
"strings"
)
@ -54,3 +55,24 @@ func (e Diagnostic) String() string {
// Error implements the 'error' interface.
func (e Diagnostic) Error() string { return e.String() }
// Diagnostics is a list of diagnostic
type Diagnostics []Diagnostic
// NumErrors returns number of errors in the diagnostics
func (l Diagnostics) NumErrors() int {
count := 0
for _, d := range l {
if d.Severity == Error {
count++
}
}
return count
}
// Print prints the list of diagnostics to 'w'
func (l Diagnostics) Print(w io.Writer, path string) {
for _, d := range l {
fmt.Fprintf(w, "%v:%v %v\n", path, d.Line, d.Message)
}
}

118
tools/src/cts/expectations/update.go
View File

@ -37,7 +37,11 @@ import (
// existing expectation lines.
//
// Update returns a list of diagnostics for things that should be addressed.
func (c *Content) Update(results result.List) ([]Diagnostic, error) {
//
// Note: Validate() should be called before attempting to update the
// expectations. If Validate() returns errors, then Update() behaviour is
// undefined.
func (c *Content) Update(results result.List, testlist []query.Query) (Diagnostics, error) {
// Make a copy of the results. This code mutates the list.
results = append(result.List{}, results...)
@ -53,13 +57,23 @@ func (c *Content) Update(results result.List) ([]Diagnostic, error) {
tagSets[len(tagSets)-i-1] = s.Tags
}
// Update those expectations!
// Scan the full result list to obtain all the test variants
// (unique tag combinations).
variants := results.Variants()
// Add 'consumed' results for tests that were skipped.
// This ensures that skipped results are not included in reduced trees.
results = c.appendConsumedResultsForSkippedTests(results, testlist, variants)
u := updater{
in: *c,
out: Content{},
qt: newQueryTree(results),
tagSets: tagSets,
in: *c,
out: Content{},
qt: newQueryTree(results),
variants: variants,
tagSets: tagSets,
}
// Update those expectations!
if err := u.build(); err != nil {
return nil, fmt.Errorf("while updating expectations: %w", err)
}
@ -70,11 +84,45 @@ func (c *Content) Update(results result.List) ([]Diagnostic, error) {
// updater holds the state used for updating the expectations
type updater struct {
in Content // the original expectations Content
out Content // newly built expectations Content
qt queryTree // the query tree
diags []Diagnostic // diagnostics raised during update
tagSets []result.Tags // reverse-ordered tag-sets of 'in'
in Content // the original expectations Content
out Content // newly built expectations Content
qt queryTree // the query tree
variants []container.Set[string]
diags []Diagnostic // diagnostics raised during update
tagSets []result.Tags // reverse-ordered tag-sets of 'in'
}
// Returns 'results' with additional 'consumed' results for tests that have
// 'Skip' expectations. This fills in gaps for results, preventing tree
// reductions from marking skipped results as failure, which could result in
// expectation collisions.
func (c *Content) appendConsumedResultsForSkippedTests(results result.List,
testlist []query.Query,
variants []container.Set[string]) result.List {
tree := query.Tree[struct{}]{}
for _, q := range testlist {
tree.Add(q, struct{}{})
}
for _, c := range c.Chunks {
for _, ex := range c.Expectations {
if container.NewSet(ex.Status...).Contains(string(result.Skip)) {
for _, variant := range variants {
if !variant.ContainsAll(ex.Tags) {
continue
}
glob, _ := tree.Glob(query.Parse(ex.Query))
for _, qd := range glob {
results = append(results, result.Result{
Query: qd.Query,
Tags: variant,
Status: consumed,
})
}
}
}
}
}
return results
}
// simplifyStatuses replaces all result statuses that are not one of
@ -160,14 +208,9 @@ func (qt *queryTree) glob(q query.Query) (result.List, error) {
return out, nil
}
// globAndCheckForCollisions returns the list of results matching the given tags
// under (or with) the given query.
// globAndCheckForCollisions will return an error if any of the results are
// already consumed by a non-zero line. The non-zero line distinguishes between
// results consumed by expectations declared in the input (non-zero line), vs
// those that were introduced by the update (zero line). We only want to error
// if there's a collision in user declared expectations.
func (qt *queryTree) globAndCheckForCollisions(q query.Query, t result.Tags) (result.List, error) {
// globTags returns the list of results matching the given tags under (or with)
// the given query.
func (qt *queryTree) globTags(q query.Query, t result.Tags) (result.List, error) {
glob, err := qt.tree.Glob(q)
if err != nil {
return nil, err
@ -177,12 +220,6 @@ func (qt *queryTree) globAndCheckForCollisions(q query.Query, t result.Tags) (re
for _, indices := range glob {
for _, idx := range indices.Data {
if r := qt.results[idx]; r.Tags.ContainsAll(t) {
if at := qt.consumedAt[idx]; at > 0 {
if len(t) > 0 {
return nil, fmt.Errorf("%v %v collides with expectation at line %v", t, q, at)
}
return nil, fmt.Errorf("%v collides with expectation at line %v", q, at)
}
out = append(out, r)
}
}
@ -194,7 +231,7 @@ func (qt *queryTree) globAndCheckForCollisions(q query.Query, t result.Tags) (re
// under (or with) the given query, as consumed.
// line is used to record the line at which the results were consumed. If the
// results were consumed as part of generating new expectations then line should
// be 0. See queryTree.globAndCheckForCollisions().
// be 0.
func (qt *queryTree) markAsConsumed(q query.Query, t result.Tags, line int) {
if glob, err := qt.tree.Glob(q); err == nil {
for _, indices := range glob {
@ -274,21 +311,12 @@ func (u *updater) chunk(in Chunk) Chunk {
return out
}
// chunk returns a new list of Expectations, based on the Expectation 'in',
// expectation returns a new list of Expectations, based on the Expectation 'in',
// using the new result data.
func (u *updater) expectation(in Expectation, keep bool) []Expectation {
// noResults is a helper for returning when the expectation has no test
// results.
// If the expectation has an expected 'Skip' result, then we're likely
// to be missing results (as the test was not run). In this situation
// the expectation is preserved, and no diagnostics are raised.
// If the expectation did not have a 'Skip' result, then a diagnostic will
// be raised and the expectation will be removed.
noResults := func() []Expectation {
if container.NewSet(in.Status...).Contains(string(result.Skip)) {
return []Expectation{in}
}
// Expectation does not have a 'Skip' result.
if len(in.Tags) > 0 {
u.diag(Warning, in.Line, "no results found for '%v' with tags %v", in.Query, in.Tags)
} else {
@ -298,12 +326,11 @@ func (u *updater) expectation(in Expectation, keep bool) []Expectation {
return []Expectation{}
}
// Grab all the results that match the expectation's query
q := query.Parse(in.Query)
// Glob the results for the expectation's query + tag combination.
// Ensure that none of these are already consumed.
results, err := u.qt.globAndCheckForCollisions(q, in.Tags)
results, err := u.qt.globTags(q, in.Tags)
// If we can't find any results for this query + tag combination, then bail.
switch {
case errors.As(err, &query.ErrNoDataForQuery{}):
@ -357,18 +384,14 @@ func (u *updater) expectation(in Expectation, keep bool) []Expectation {
// addNewExpectations (potentially) appends to 'u.out' chunks for new flaky and
// failing tests.
func (u *updater) addNewExpectations() error {
// Scan the full result list to obtain all the test variants
// (unique tag combinations).
allVariants := u.qt.results.Variants()
// For each variant:
// • Build a query tree using the results filtered to the variant, and then
// reduce the tree.
// • Take all the reduced-tree leaf nodes, and add these to 'roots'.
// Once we've collected all the roots, we'll use these to build the
// expectations across the reduced set of tags.
roots := container.NewMap[string, query.Query]()
for _, variant := range allVariants {
roots := query.Tree[bool]{}
for _, variant := range u.variants {
// Build a tree from the results matching the given variant.
tree, err := u.qt.results.FilterByVariant(variant).StatusTree()
if err != nil {
@ -378,15 +401,16 @@ func (u *updater) addNewExpectations() error {
tree.Reduce(treeReducer)
// Add all the reduced leaf nodes to 'roots'.
for _, qd := range tree.List() {
roots.Add(qd.Query.String(), qd.Query)
// Use Split() to ensure that only the leaves have data (true) in the tree
roots.Split(qd.Query, true)
}
}
// Build all the expectations for each of the roots.
expectations := []Expectation{}
for _, root := range roots.Values() {
for _, root := range roots.List() {
expectations = append(expectations, u.expectationsForRoot(
root, // Root query
root.Query, // Root query
0, // Line number
"crbug.com/dawn/0000", // Bug
"", // Comment

183
tools/src/cts/expectations/update_test.go
View File

@ -18,6 +18,7 @@ import (
"strings"
"testing"
"dawn.googlesource.com/dawn/tools/src/container"
"dawn.googlesource.com/dawn/tools/src/cts/expectations"
"dawn.googlesource.com/dawn/tools/src/cts/query"
"dawn.googlesource.com/dawn/tools/src/cts/result"
@ -41,7 +42,7 @@ func TestUpdate(t *testing.T) {
expectations string
results result.List
updated string
diagnostics []expectations.Diagnostic
diagnostics expectations.Diagnostics
err string
}
for _, test := range []Test{
@ -73,7 +74,7 @@ some:other,test:* [ Failure ]
updated: `
some:other,test:* [ Failure ]
`,
diagnostics: []expectations.Diagnostic{
diagnostics: expectations.Diagnostics{
{
Severity: expectations.Warning,
Line: headerLines + 2,
@ -109,7 +110,7 @@ some:other,test:* [ Failure ]
updated: `
some:other,test:* [ Failure ]
`,
diagnostics: []expectations.Diagnostic{
diagnostics: expectations.Diagnostics{
{
Severity: expectations.Warning,
Line: headerLines + 3,
@ -117,55 +118,6 @@ some:other,test:* [ Failure ]
},
},
},
{ //////////////////////////////////////////////////////////////////////
name: "no results found Skip",
expectations: `
crbug.com/a/123 a:missing,test,result:* [ Skip ]
some:other,test:* [ Failure ]
`,
results: result.List{
result.Result{
Query: Q("some:other,test:*"),
Tags: result.NewTags("os-a", "gpu-a"),
Status: result.Failure,
},
result.Result{
Query: Q("some:other,test:*"),
Tags: result.NewTags("os-b", "gpu-b"),
Status: result.Failure,
},
},
updated: `
crbug.com/a/123 a:missing,test,result:* [ Skip ]
some:other,test:* [ Failure ]
`,
},
{ //////////////////////////////////////////////////////////////////////
name: "simple expectation collision",
expectations: `
a:b,c:* [ Failure ]
a:b,c:* [ Failure ]
`,
results: result.List{
result.Result{
Query: Q("a:b,c:d"),
Tags: result.NewTags("os-a", "os-c", "gpu-b"),
Status: result.Failure,
},
},
updated: `
a:b,c:* [ Failure ]
`,
diagnostics: []expectations.Diagnostic{
{
Severity: expectations.Error,
Line: headerLines + 3,
Message: "a:b,c:* collides with expectation at line 8",
},
},
},
{ //////////////////////////////////////////////////////////////////////
name: "simple expectation with tags",
expectations: `
@ -182,100 +134,6 @@ a:b,c:* [ Failure ]
updated: `
a:b,c:* [ Failure ]
`,
diagnostics: []expectations.Diagnostic{
{
Severity: expectations.Error,
Line: headerLines + 3,
Message: "[gpu-b] a:b,c:* collides with expectation at line 8",
},
},
},
{ //////////////////////////////////////////////////////////////////////
name: "simple expectation collision KEEP",
expectations: `
# KEEP
a:b,c:* [ Failure ]
a:b,c:* [ Failure ]
`,
results: result.List{
result.Result{
Query: Q("a:b,c:d"),
Tags: result.NewTags("os-a", "os-c", "gpu-b"),
Status: result.Failure,
},
},
updated: `
# KEEP
a:b,c:* [ Failure ]
`,
diagnostics: []expectations.Diagnostic{
{
Severity: expectations.Error,
Line: headerLines + 4,
Message: "a:b,c:* collides with expectation at line 9",
},
},
},
{ //////////////////////////////////////////////////////////////////////
name: "collision with child-expectation",
expectations: `
a:b:x:* [ Failure ]
a:b:* [ Failure ]
`,
results: result.List{
result.Result{
Query: Q("a:b:x:*"),
Tags: result.NewTags("os-a", "os-c", "gpu-b"),
Status: result.Failure,
},
result.Result{
Query: Q("a:b:y:*"),
Tags: result.NewTags("os-a", "os-c", "gpu-b"),
Status: result.Failure,
},
},
updated: `
a:b:x:* [ Failure ]
# New failures. Please triage:
crbug.com/dawn/0000 a:b:y:* [ Failure ]
`,
diagnostics: []expectations.Diagnostic{
{
Severity: expectations.Error,
Line: headerLines + 3,
Message: "a:b:* collides with expectation at line 8",
},
},
},
{ //////////////////////////////////////////////////////////////////////
name: "collision with parent-expectation",
expectations: `
a:b:* [ Failure ]
a:b:x:* [ Failure ]
`,
results: result.List{
result.Result{
Query: Q("a:b:x:*"),
Tags: result.NewTags("os-a", "os-c", "gpu-b"),
Status: result.Failure,
},
result.Result{
Query: Q("a:b:y:*"),
Tags: result.NewTags("os-a", "os-c", "gpu-b"),
Status: result.Failure,
},
},
updated: `
a:b:* [ Failure ]
`,
diagnostics: []expectations.Diagnostic{
{
Severity: expectations.Error,
Line: headerLines + 3,
Message: "a:b:x:* collides with expectation at line 8",
},
},
},
{ //////////////////////////////////////////////////////////////////////
name: "expectation test now passes",
@ -345,7 +203,7 @@ crbug.com/a/123 [ gpu-b os-b ] a:b,c:d [ Failure ]
crbug.com/a/123 [ gpu-a os-a ] a:b,c:d [ Failure ]
crbug.com/a/123 [ gpu-b os-b ] a:b,c:d [ Failure ]
`,
diagnostics: []expectations.Diagnostic{
diagnostics: expectations.Diagnostics{
{
Severity: expectations.Note,
Line: headerLines + 3,
@ -369,7 +227,7 @@ crbug.com/a/123 a:b,c:d:* [ Failure ]
# KEEP
crbug.com/a/123 a:b,c:d:* [ Failure ]
`,
diagnostics: []expectations.Diagnostic{
diagnostics: expectations.Diagnostics{
{
Severity: expectations.Note,
Line: headerLines + 3,
@ -391,6 +249,11 @@ crbug.com/a/123 a:b,c:d:* [ Failure ]
Tags: result.NewTags("os-a", "gpu-b"),
Status: result.Abort,
},
result.Result{
Query: Q("suite:dir_a,dir_b:test_c:case=4;*"),
Tags: result.NewTags("os-a", "gpu-a"),
Status: result.Crash,
},
result.Result{
Query: Q("suite:dir_a,dir_b:test_c:case=4;*"),
Tags: result.NewTags("os-b", "gpu-b"),
@ -402,22 +265,22 @@ crbug.com/a/123 a:b,c:d:* [ Failure ]
Status: result.RetryOnFailure,
},
result.Result{
Query: Q("suite:dir_a,dir_b:test_b;case=5;*"),
Query: Q("suite:dir_a,dir_b:test_b:case=5;*"),
Tags: result.NewTags("os-b", "gpu-b"),
Status: result.Pass,
},
result.Result{
Query: Q("suite:dir_a,dir_b:test_b:*"),
Query: Q("suite:dir_a,dir_b:test_b:case=6;*"),
Tags: result.NewTags("os-a", "gpu-a"),
Status: result.Slow,
},
result.Result{
Query: Q("suite:dir_a,dir_b:test_b:*"),
Query: Q("suite:dir_a,dir_b:test_b:case=6;*"),
Tags: result.NewTags("os-b", "gpu-a"),
Status: result.Pass,
},
result.Result{
Query: Q("suite:dir_a,dir_b:test_c:*"),
Query: Q("suite:dir_a,dir_b:test_c:case=6;*"),
Tags: result.NewTags("os-a", "gpu-a"),
Status: result.RetryOnFailure,
},
@ -425,14 +288,13 @@ crbug.com/a/123 a:b,c:d:* [ Failure ]
updated: `# A comment
# New flakes. Please triage:
crbug.com/dawn/0000 [ gpu-a os-a ] suite:dir_a,dir_b:test_c:* [ RetryOnFailure ]
crbug.com/dawn/0000 [ gpu-b os-b ] suite:dir_a,dir_b:test_c:case=5;* [ RetryOnFailure ]
crbug.com/dawn/0000 suite:dir_a,dir_b:test_c:case=5;* [ RetryOnFailure ]
crbug.com/dawn/0000 suite:dir_a,dir_b:test_c:case=6;* [ RetryOnFailure ]
# New failures. Please triage:
crbug.com/dawn/0000 [ gpu-b os-a ] suite:* [ Failure ]
crbug.com/dawn/0000 [ gpu-a os-a ] suite:dir_a,dir_b:test_a:* [ Failure ]
crbug.com/dawn/0000 suite:dir_a,dir_b:test_a:* [ Failure ]
crbug.com/dawn/0000 [ gpu-a os-a ] suite:dir_a,dir_b:test_b:* [ Slow ]
crbug.com/dawn/0000 [ gpu-b os-b ] suite:dir_a,dir_b:test_c:case=4;* [ Failure ]
crbug.com/dawn/0000 suite:dir_a,dir_b:test_c:case=4;* [ Failure ]
`,
},
{ //////////////////////////////////////////////////////////////////////
@ -650,8 +512,13 @@ crbug.com/dawn/0000 a:b,c:t29:* [ Failure ]
t.Fatalf("'%v': expectations.Parse():\n%v", test.name, err)
}
testList := container.NewMap[string, query.Query]()
for _, r := range test.results {
testList.Add(r.Query.String(), r.Query)
}
errMsg := ""
diagnostics, err := ex.Update(test.results)
diagnostics, err := ex.Update(test.results, testList.Values())
if err != nil {
errMsg = err.Error()
}

77
tools/src/cts/expectations/validate.go Normal file
View File

@ -0,0 +1,77 @@
// Copyright 2022 The Dawn Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package expectations provides types and helpers for parsing, updating and
// writing WebGPU expectations files.
//
// See <dawn>/webgpu-cts/expectations.txt for more information.
package expectations
import (
"fmt"
"dawn.googlesource.com/dawn/tools/src/container"
"dawn.googlesource.com/dawn/tools/src/cts/query"
)
func (c Content) tagsCollide(a, b container.Set[string]) bool {
for _, set := range c.Tags.Sets {
aSet := a.Intersection(set.Tags)
bSet := b.Intersection(set.Tags)
if len(aSet) != 0 && len(bSet) != 0 && len(aSet.Intersection(bSet)) == 0 {
return false
}
}
return true
}
// Validate checks that the expectations do not contain errors
func (c Content) Validate() Diagnostics {
tree, _ := query.NewTree[Expectations]()
for _, chunk := range c.Chunks {
for _, ex := range chunk.Expectations {
node := tree.GetOrCreate(query.Parse(ex.Query), func() Expectations {
return Expectations{}
})
*node = append(*node, ex)
}
}
var out Diagnostics
for _, chunk := range c.Chunks {
for _, ex := range chunk.Expectations {
glob, err := tree.Glob(query.Parse(ex.Query))
if err != nil {
out = append(out, Diagnostic{
Severity: Error,
Line: ex.Line,
Message: err.Error(),
})
continue
}
for _, qd := range glob {
expectations := qd.Data
for _, other := range expectations {
if other.Line != ex.Line && c.tagsCollide(ex.Tags, other.Tags) {
out = append(out, Diagnostic{
Severity: Error,
Line: ex.Line,
Message: fmt.Sprintf("expectation collides with expectation on line %v", other.Line),
})
}
}
}
}
}
return out
}

130
tools/src/cts/expectations/validate_test.go Normal file
View File

@ -0,0 +1,130 @@
// Copyright 2022 The Dawn Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package expectations_test
import (
"testing"
"dawn.googlesource.com/dawn/tools/src/cts/expectations"
"github.com/google/go-cmp/cmp"
)
func TestValidate(t *testing.T) {
header := `# BEGIN TAG HEADER
# OS
# tags: [ os-a os-b os-c ]
# GPU
# tags: [ gpu-a gpu-b gpu-c ]
# END TAG HEADER
`
type Test struct {
name string
expectations string
diagnostics expectations.Diagnostics
}
for _, test := range []Test{
{ //////////////////////////////////////////////////////////////////////
name: "empty",
expectations: ``,
},
{ //////////////////////////////////////////////////////////////////////
name: "simple",
expectations: `
crbug.com/a/123 a:b,c:d,* [ Failure ]
`,
},
{ //////////////////////////////////////////////////////////////////////
name: "no-tag collision",
expectations: `
crbug.com/a/123 a:b,c:d,* [ Failure ]
crbug.com/a/123 a:x,x:d,* [ Failure ]
crbug.com/a/123 a:b,c:d,* [ Failure ]
`,
diagnostics: expectations.Diagnostics{
{
Line: 8,
Severity: expectations.Error,
Message: "expectation collides with expectation on line 10",
},
{
Line: 10,
Severity: expectations.Error,
Message: "expectation collides with expectation on line 8",
},
},
},
{ //////////////////////////////////////////////////////////////////////
name: "tag collision",
expectations: `
crbug.com/a/123 [ os-a ] a:b,c:d,* [ Failure ]
crbug.com/a/123 a:x,x:d,* [ Failure ]
crbug.com/a/123 [ os-a ] a:b,c:d,* [ Failure ]
`,
diagnostics: expectations.Diagnostics{
{
Line: 8,
Severity: expectations.Error,
Message: "expectation collides with expectation on line 10",
},
{
Line: 10,
Severity: expectations.Error,
Message: "expectation collides with expectation on line 8",
},
},
},
{ //////////////////////////////////////////////////////////////////////
name: "nested no-tag collision",
expectations: `
crbug.com/a/123 a:b,c:d,e:* [ Failure ]
crbug.com/a/123 a:x,x:d,* [ Failure ]
crbug.com/a/123 a:b,c:d,* [ Failure ]
`,
diagnostics: expectations.Diagnostics{
{
Line: 10,
Severity: expectations.Error,
Message: "expectation collides with expectation on line 8",
},
},
},
{ //////////////////////////////////////////////////////////////////////
name: "tag collision",
expectations: `
crbug.com/a/123 [ os-a ] a:b,c:d,e:* [ Failure ]
crbug.com/a/123 a:x,x:d,* [ Failure ]
crbug.com/a/123 [ os-a ] a:b,c:d,* [ Failure ]
`,
diagnostics: expectations.Diagnostics{
{
Line: 10,
Severity: expectations.Error,
Message: "expectation collides with expectation on line 8",
},
},
},
} {
ex, err := expectations.Parse(header + test.expectations)
if err != nil {
t.Fatalf("'%v': expectations.Parse():\n%v", test.name, err)
}
diagnostics := ex.Validate()
if diff := cmp.Diff(diagnostics, test.diagnostics); diff != "" {
t.Errorf("'%v': expectations.Update() error:\n%v", test.name, diff)
}
}
}

33
tools/src/cts/query/tree.go
View File

@ -252,7 +252,7 @@ func NewTree[Data any](entries ...QueryData[Data]) (Tree[Data], error) {
}
// Add adds a new data to the tree.
// Returns ErrDuplicateData if the tree already contains a data for the given
// Returns ErrDuplicateData if the tree already contains a data for the given node at query
func (t *Tree[Data]) Add(q Query, d Data) error {
node := &t.TreeNode
q.Walk(func(q Query, t Target, n string) error {
@ -266,6 +266,37 @@ func (t *Tree[Data]) Add(q Query, d Data) error {
return nil
}
// Split adds a new data to the tree, clearing any ancestor node's data.
// Returns ErrDuplicateData if the tree already contains a data for the given node at query
func (t *Tree[Data]) Split(q Query, d Data) error {
node := &t.TreeNode
q.Walk(func(q Query, t Target, n string) error {
delete(node.Children, TreeNodeChildKey{Name: "*", Target: t})
node.Data = nil
node = node.getOrCreateChild(TreeNodeChildKey{n, t})
return nil
})
if node.Data != nil {
return ErrDuplicateData{node.Query}
}
node.Data = &d
return nil
}
// GetOrCreate returns existing, or adds a new data to the tree.
func (t *Tree[Data]) GetOrCreate(q Query, create func() Data) *Data {
node := &t.TreeNode
q.Walk(func(q Query, t Target, n string) error {
node = node.getOrCreateChild(TreeNodeChildKey{n, t})
return nil
})
if node.Data == nil {
data := create()
node.Data = &data
}
return node.Data
}
// Reduce reduces the tree using the Merger function f.
// If the Merger function returns a non-nil Data value, then this will be used
// to replace the non-leaf node with a new leaf node holding the returned Data.

218
tools/src/cts/query/tree_test.go
View File

@ -238,7 +238,6 @@ func TestNewSingle(t *testing.T) {
t.Errorf("NewTree(%v) tree was not as expected:\n%v", test.in, diff)
}
}
}
func TestNewMultiple(t *testing.T) {
@ -351,6 +350,223 @@ func TestNewWithCollision(t *testing.T) {
}
}
func TestSplit(t *testing.T) {
type Tree = query.Tree[string]
type Node = query.TreeNode[string]
type QueryData = query.QueryData[string]
type Children = query.TreeNodeChildren[string]
type Test struct {
in QueryData
pre Tree
post Tree
}
for _, test := range []Test{
{ /////////////////////////////////////////////////////////////////////
in: QueryData{
Query: Q(`suite:*`),
Data: pass,
},
post: Tree{
TreeNode: Node{
Children: Children{
query.TreeNodeChildKey{`suite`, query.Suite}: {
Query: Q(`suite`),
Children: Children{
query.TreeNodeChildKey{`*`, query.Files}: {
Query: Q(`suite:*`),
Data: &pass,
},
},
},
},
},
},
},
{ /////////////////////////////////////////////////////////////////////
in: QueryData{
Query: Q(`suite:a,b:*`),
Data: pass,
},
pre: Tree{
TreeNode: Node{
Children: Children{
query.TreeNodeChildKey{`suite`, query.Suite}: {
Query: Q(`suite`),
Children: Children{
query.TreeNodeChildKey{`a`, query.Files}: {
Query: Q(`suite:a`),
Children: Children{
query.TreeNodeChildKey{`*`, query.Files}: {
Query: Q(`suite:a,*`),
Data: &pass,
},
},
},
},
},
},
},
},
post: Tree{
TreeNode: Node{
Children: Children{
query.TreeNodeChildKey{`suite`, query.Suite}: {
Query: Q(`suite`),
Children: Children{
query.TreeNodeChildKey{`a`, query.Files}: {
Query: Q(`suite:a`),
Children: Children{
query.TreeNodeChildKey{`b`, query.Files}: {
Query: Q(`suite:a,b`),
Children: Children{
query.TreeNodeChildKey{`*`, query.Tests}: {
Query: Q(`suite:a,b:*`),
Data: &pass,
},
},
},
},
},
},
},
},
},
},
},
{ /////////////////////////////////////////////////////////////////////
in: QueryData{
Query: Q(`suite:a:*`),
Data: pass,
},
pre: Tree{
TreeNode: Node{
Children: Children{
query.TreeNodeChildKey{`suite`, query.Suite}: {
Query: Q(`suite`),
Children: Children{
query.TreeNodeChildKey{`a`, query.Files}: {
Query: Q(`suite:a`),
Children: Children{
query.TreeNodeChildKey{`b`, query.Files}: {
Query: Q(`suite:a,b`),
Children: Children{
query.TreeNodeChildKey{`*`, query.Files}: {
Query: Q(`suite:a,b,*`),
Data: &pass,
},
},
},
},
},
},
},
},
},
},
post: Tree{
TreeNode: Node{
Children: Children{
query.TreeNodeChildKey{`suite`, query.Suite}: {
Query: Q(`suite`),
Children: Children{
query.TreeNodeChildKey{`a`, query.Files}: {
Query: Q(`suite:a`),
Children: Children{
query.TreeNodeChildKey{`*`, query.Tests}: {
Query: Q(`suite:a:*`),
Data: &pass,
},
query.TreeNodeChildKey{`b`, query.Files}: {
Query: Q(`suite:a,b`),
Children: Children{
query.TreeNodeChildKey{`*`, query.Files}: {
Query: Q(`suite:a,b,*`),
Data: &pass,
},
},
},
},
},
},
},
},
},
},
},
{ /////////////////////////////////////////////////////////////////////
in: QueryData{
Query: Q(`suite:a,b:c:*`),
Data: pass,
},
pre: Tree{
TreeNode: Node{
Children: Children{
query.TreeNodeChildKey{`suite`, query.Suite}: {
Query: Q(`suite`),
Children: Children{
query.TreeNodeChildKey{`a`, query.Files}: {
Query: Q(`suite:a`),
Children: Children{
query.TreeNodeChildKey{`b`, query.Files}: {
Query: Q(`suite:a,b`),
Children: Children{
query.TreeNodeChildKey{`*`, query.Tests}: {
Query: Q(`suite:a,b:*`),
Data: &pass,
},
},
},
},
},
},
},
},
},
},
post: Tree{
TreeNode: Node{
Children: Children{
query.TreeNodeChildKey{`suite`, query.Suite}: {
Query: Q(`suite`),
Children: Children{
query.TreeNodeChildKey{`a`, query.Files}: {
Query: Q(`suite:a`),
Children: Children{
query.TreeNodeChildKey{`b`, query.Files}: {
Query: Q(`suite:a,b`),
Children: Children{
query.TreeNodeChildKey{`c`, query.Tests}: {
Query: Q(`suite:a,b:c`),
Children: Children{
query.TreeNodeChildKey{`*`, query.Cases}: {
Query: Q(`suite:a,b:c:*`),
Data: &pass,
},
},
},
},
},
},
},
},
},
},
},
},
},
} {
tree := test.pre
if err := tree.Split(test.in.Query, test.in.Data); err != nil {
t.Errorf("NewTree(%v): %v", test.in, err)
continue
}
if diff := cmp.Diff(tree, test.post); diff != "" {
t.Errorf("Split(%v) tree was not as expected:\n%v", test.in, diff)
}
}
}
func TestList(t *testing.T) {
type QueryData = query.QueryData[string]

103
webgpu-cts/expectations.txt
View File

@ -79,7 +79,8 @@ crbug.com/dawn/1487 [ intel-gen-9 win10 ] webgpu:api,operation,resource_init,tex
# Flaky failures using SwiftShader (crbug.com/1344876).
# KEEP
################################################################################
crbug.com/1343969 [ win ] webgpu:web_platform,copyToTexture,ImageBitmap:* [ RetryOnFailure Slow ]
crbug.com/1343969 [ win ] webgpu:web_platform,copyToTexture,ImageBitmap:from_ImageData:* [ RetryOnFailure Slow ]
crbug.com/1343969 [ webgpu-adapter-default win ] webgpu:web_platform,copyToTexture,ImageBitmap:from_canvas:* [ RetryOnFailure Slow ]
crbug.com/1344876 [ ubuntu webgpu-adapter-swiftshader ] webgpu:web_platform,copyToTexture,ImageBitmap:from_ImageData:* [ RetryOnFailure Slow ]
crbug.com/1359106 [ ubuntu webgpu-adapter-default ] webgpu:web_platform,copyToTexture,ImageBitmap:from_ImageData:* [ RetryOnFailure ]
@ -389,9 +390,6 @@ crbug.com/tint/1613 webgpu:shader,execution,expression,unary,f32_arithmetic:nega
################################################################################
# shader execution failures
################################################################################
crbug.com/dawn/0000 [ intel-gen-9 target-cpu-32 ] webgpu:shader,execution,expression,binary,f32_arithmetic:addition:inputSource="uniform";vectorize="_undef_" [ Failure ]
crbug.com/dawn/0000 [ intel-gen-9 target-cpu-32 ] webgpu:shader,execution,expression,binary,f32_arithmetic:division:inputSource="uniform";vectorize="_undef_" [ Failure ]
crbug.com/dawn/0000 [ intel-gen-9 target-cpu-32 ] webgpu:shader,execution,expression,binary,f32_arithmetic:multiplication:inputSource="uniform";vectorize="_undef_" [ Failure ]
crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,binary,f32_arithmetic:remainder:inputSource="storage_r";vectorize="_undef_" [ Failure ]
crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,binary,f32_arithmetic:remainder:inputSource="storage_r";vectorize=2 [ Failure ]
crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,binary,f32_arithmetic:remainder:inputSource="storage_r";vectorize=3 [ Failure ]
@ -404,7 +402,6 @@ crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,binary,f32_arit
crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,binary,f32_arithmetic:remainder:inputSource="uniform";vectorize=2 [ Failure ]
crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,binary,f32_arithmetic:remainder:inputSource="uniform";vectorize=3 [ Failure ]
crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,binary,f32_arithmetic:remainder:inputSource="uniform";vectorize=4 [ Failure ]
crbug.com/dawn/0000 [ intel-gen-9 win10 ] webgpu:shader,execution,expression,binary,f32_arithmetic:subtraction:inputSource="uniform";vectorize="_undef_" [ Failure ]
crbug.com/dawn/0000 [ intel-gen-9 ubuntu ] webgpu:shader,execution,expression,call,builtin,countTrailingZeros:i32:inputSource="storage_r";vectorize="_undef_" [ Failure ]
crbug.com/dawn/0000 [ intel-gen-9 ubuntu ] webgpu:shader,execution,expression,call,builtin,countTrailingZeros:i32:inputSource="storage_r";vectorize=2 [ Failure ]
crbug.com/dawn/0000 [ intel-gen-9 ubuntu ] webgpu:shader,execution,expression,call,builtin,countTrailingZeros:i32:inputSource="storage_r";vectorize=3 [ Failure ]
@ -429,11 +426,12 @@ crbug.com/dawn/0000 [ intel-gen-9 ubuntu ] webgpu:shader,execution,expression,ca
crbug.com/dawn/0000 [ intel-gen-9 ubuntu ] webgpu:shader,execution,expression,call,builtin,countTrailingZeros:u32:inputSource="uniform";vectorize=2 [ Failure ]
crbug.com/dawn/0000 [ intel-gen-9 ubuntu ] webgpu:shader,execution,expression,call,builtin,countTrailingZeros:u32:inputSource="uniform";vectorize=3 [ Failure ]
crbug.com/dawn/0000 [ intel-gen-9 ubuntu ] webgpu:shader,execution,expression,call,builtin,countTrailingZeros:u32:inputSource="uniform";vectorize=4 [ Failure ]
crbug.com/dawn/0000 [ dawn-backend-validation intel-gen-9 monterey target-cpu-64 ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec3:* [ Failure ]
crbug.com/dawn/0000 [ dawn-no-backend-validation intel-gen-9 monterey target-cpu-64 ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec3:* [ Failure ]
crbug.com/dawn/0000 [ dawn-no-backend-validation nvidia-0x2184 target-cpu-32 win10 ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec3:inputSource="uniform" [ Failure ]
crbug.com/dawn/0000 [ intel-gen-9 monterey target-cpu-64 ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec4:* [ Failure ]
crbug.com/dawn/0000 [ intel-gen-9 target-cpu-32 win10 ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec4:inputSource="uniform" [ Failure ]
crbug.com/dawn/0000 [ monterey ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec3:inputSource="storage_r" [ Failure ]
crbug.com/dawn/0000 [ monterey ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec3:inputSource="storage_rw" [ Failure ]
crbug.com/dawn/0000 [ monterey ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec3:inputSource="uniform" [ Failure ]
crbug.com/dawn/0000 [ monterey ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec4:inputSource="storage_r" [ Failure ]
crbug.com/dawn/0000 [ monterey ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec4:inputSource="storage_rw" [ Failure ]
crbug.com/dawn/0000 [ monterey ] webgpu:shader,execution,expression,call,builtin,dot:f32_vec4:inputSource="uniform" [ Failure ]
crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,extractBits:i32:inputSource="storage_r";width=1 [ Failure ]
crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,extractBits:i32:inputSource="storage_r";width=2 [ Failure ]
crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,extractBits:i32:inputSource="storage_r";width=3 [ Failure ]
@ -506,22 +504,33 @@ crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,in
crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,insertBits:integer:inputSource="uniform";signed=true;width=2 [ Failure ]
crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,insertBits:integer:inputSource="uniform";signed=true;width=3 [ Failure ]
crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,insertBits:integer:inputSource="uniform";signed=true;width=4 [ Failure ]
crbug.com/dawn/0000 [ intel-gen-9 target-cpu-32 ] webgpu:shader,execution,expression,call,builtin,ldexp:f32:inputSource="uniform";vectorize="_undef_" [ Failure ]
crbug.com/dawn/0000 [ intel-gen-9 target-cpu-32 ] webgpu:shader,execution,expression,call,builtin,max:f32:inputSource="uniform";vectorize="_undef_" [ Failure ]
crbug.com/dawn/0000 [ intel-gen-9 target-cpu-32 ] webgpu:shader,execution,expression,call,builtin,min:f32:inputSource="uniform";vectorize="_undef_" [ Failure ]
crbug.com/dawn/0000 [ intel-gen-9 target-cpu-32 ] webgpu:shader,execution,expression,call,builtin,pow:f32:inputSource="uniform";vectorize="_undef_" [ Failure ]
crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="mixed";dispatch="indirect";groupSize={"x":1,"y":1,"z":1};numGroups={"x":1,"y":1,"z":1} [ Failure ]
crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="mixed";dispatch="indirect";groupSize={"x":1,"y":1,"z":1};numGroups={"x":8,"y":4,"z":2} [ Failure ]
crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="mixed";dispatch="indirect";groupSize={"x":3,"y":7,"z":5};numGroups={"x":13,"y":9,"z":11} [ Failure ]
crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="mixed";dispatch="indirect";groupSize={"x":8,"y":4,"z":2};numGroups={"x":1,"y":1,"z":1} [ Failure ]
crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="param";dispatch="indirect";groupSize={"x":1,"y":1,"z":1};numGroups={"x":1,"y":1,"z":1} [ Failure ]
crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="param";dispatch="indirect";groupSize={"x":1,"y":1,"z":1};numGroups={"x":8,"y":4,"z":2} [ Failure ]
crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="param";dispatch="indirect";groupSize={"x":3,"y":7,"z":5};numGroups={"x":13,"y":9,"z":11} [ Failure ]
crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="param";dispatch="indirect";groupSize={"x":8,"y":4,"z":2};numGroups={"x":1,"y":1,"z":1} [ Failure ]
crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="struct";dispatch="indirect";groupSize={"x":1,"y":1,"z":1};numGroups={"x":1,"y":1,"z":1} [ Failure ]
crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="struct";dispatch="indirect";groupSize={"x":1,"y":1,"z":1};numGroups={"x":8,"y":4,"z":2} [ Failure ]
crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="struct";dispatch="indirect";groupSize={"x":3,"y":7,"z":5};numGroups={"x":13,"y":9,"z":11} [ Failure ]
crbug.com/tint/1500 [ nvidia-0x2184 target-cpu-32 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="struct";dispatch="indirect";groupSize={"x":8,"y":4,"z":2};numGroups={"x":1,"y":1,"z":1} [ Failure ]
crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,length:f32_vec2:inputSource="const" [ Failure ]
crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,length:f32_vec3:inputSource="const" [ Failure ]
crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,length:f32_vec4:inputSource="const" [ Failure ]
crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,normalize:f32_vec2:inputSource="const" [ Failure ]
crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,normalize:f32_vec3:inputSource="const" [ Failure ]
crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,normalize:f32_vec4:inputSource="const" [ Failure ]
crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,quantizeToF16:* [ Failure ]
crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,saturate:f32:inputSource="const";vectorize="_undef_" [ Failure ]
crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,saturate:f32:inputSource="const";vectorize=2 [ Failure ]
crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,saturate:f32:inputSource="const";vectorize=3 [ Failure ]
crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,saturate:f32:inputSource="const";vectorize=4 [ Failure ]
crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="storage_r";vectorize="_undef_" [ Failure ]
crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="storage_r";vectorize=2 [ Failure ]
crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="storage_r";vectorize=3 [ Failure ]
crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="storage_r";vectorize=4 [ Failure ]
crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="storage_rw";vectorize="_undef_" [ Failure ]
crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="storage_rw";vectorize=2 [ Failure ]
crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="storage_rw";vectorize=3 [ Failure ]
crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="storage_rw";vectorize=4 [ Failure ]
crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="uniform";vectorize="_undef_" [ Failure ]
crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="uniform";vectorize=2 [ Failure ]
crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="uniform";vectorize=3 [ Failure ]
crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:inputSource="uniform";vectorize=4 [ Failure ]
crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,smoothstep:f32:inputSource="const";vectorize="_undef_" [ Failure ]
crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,smoothstep:f32:inputSource="const";vectorize=2 [ Failure ]
crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,smoothstep:f32:inputSource="const";vectorize=3 [ Failure ]
crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,smoothstep:f32:inputSource="const";vectorize=4 [ Failure ]
################################################################################
# webgpu:shader,execution,memory_model
@ -617,13 +626,8 @@ crbug.com/dawn/0000 [ win ] webgpu:api,operation,render_pipeline,pipeline_output
crbug.com/dawn/0000 [ mac ] webgpu:web_platform,canvas,readbackFromWebGPUCanvas:* [ Failure ]
crbug.com/dawn/0000 [ mac ] worker_webgpu:api,operation,render_pass,storeOp:render_pass_store_op,depth_stencil_attachment_only:depthStencilFormat="stencil8";* [ Failure ]
################################################################################
# slow tests
# KEEP
################################################################################
crbug.com/dawn/0000 [ win ] webgpu:shader,execution,expression,binary,f32_logical:* [ Slow ]
# New failures. Please triage:
crbug.com/dawn/0000 [ monterey ] webgpu:api,operation,command_buffer,copyTextureToTexture:color_textures,non_compressed,non_array:srcFormat="rgb9e5ufloat";dstFormat="rgb9e5ufloat";dimension="1d" [ Failure ]
crbug.com/dawn/0000 [ intel-gen-9 win10 ] webgpu:api,operation,command_buffer,image_copy:mip_levels:initMethod="WriteTexture";checkMethod="PartialCopyT2B";format="rgba32uint";dimension="2d" [ Failure ]
crbug.com/dawn/0000 [ win10 ] webgpu:api,validation,createBindGroup:buffer,resource_binding_size:type="read-only-storage" [ Failure ]
crbug.com/dawn/0000 [ win10 ] webgpu:api,validation,createBindGroup:buffer,resource_binding_size:type="storage" [ Failure ]
@ -650,35 +654,12 @@ crbug.com/dawn/0000 webgpu:api,validation,render_pass,render_pass_descriptor:tim
crbug.com/dawn/0000 webgpu:api,validation,render_pass,render_pass_descriptor:timestampWrites,query_set_type:queryTypeA="timestamp";queryTypeB="occlusion" [ Failure ]
crbug.com/dawn/0000 webgpu:api,validation,render_pass,render_pass_descriptor:timestamp_writes_location:locationA="beginning";locationB="beginning" [ Failure ]
crbug.com/dawn/0000 webgpu:api,validation,render_pass,render_pass_descriptor:timestamp_writes_location:locationA="end";locationB="end" [ Failure ]
crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,length:f32_vec2:inputSource="const" [ Failure ]
crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,length:f32_vec3:inputSource="const" [ Failure ]
crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,length:f32_vec4:inputSource="const" [ Failure ]
crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,normalize:f32_vec2:inputSource="const" [ Failure ]
crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,normalize:f32_vec3:inputSource="const" [ Failure ]
crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,normalize:f32_vec4:inputSource="const" [ Failure ]
crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,quantizeToF16:* [ Failure ]
crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,saturate:f32:inputSource="const";vectorize="_undef_" [ Failure ]
crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,saturate:f32:inputSource="const";vectorize=2 [ Failure ]
crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,saturate:f32:inputSource="const";vectorize=3 [ Failure ]
crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,saturate:f32:inputSource="const";vectorize=4 [ Failure ]
crbug.com/dawn/0000 [ win10 ] webgpu:shader,execution,expression,call,builtin,sign:f32:* [ Failure ]
crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,smoothstep:f32:inputSource="const";vectorize="_undef_" [ Failure ]
crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,smoothstep:f32:inputSource="const";vectorize=2 [ Failure ]
crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,smoothstep:f32:inputSource="const";vectorize=3 [ Failure ]
crbug.com/dawn/0000 webgpu:shader,execution,expression,call,builtin,smoothstep:f32:inputSource="const";vectorize=4 [ Failure ]
crbug.com/dawn/0000 [ intel-gen-9 target-cpu-32 ] webgpu:shader,execution,expression,call,builtin,step:f32:inputSource="uniform";vectorize="_undef_" [ Failure ]
crbug.com/dawn/0000 [ monterey webgpu-adapter-default ] webgpu:web_platform,canvas,configure:alpha_mode:* [ Failure ]
crbug.com/dawn/0000 [ monterey webgpu-adapter-swiftshader ] webgpu:web_platform,canvas,configure:alpha_mode:* [ Failure ]
crbug.com/dawn/0000 [ monterey webgpu-adapter-default ] webgpu:web_platform,canvas,configure:defaults:* [ Failure ]
crbug.com/dawn/0000 [ monterey webgpu-adapter-swiftshader ] webgpu:web_platform,canvas,configure:defaults:* [ Failure ]
crbug.com/dawn/0000 [ monterey webgpu-adapter-default ] webgpu:web_platform,canvas,configure:device:* [ Failure ]
crbug.com/dawn/0000 [ monterey webgpu-adapter-swiftshader ] webgpu:web_platform,canvas,configure:device:* [ Failure ]
crbug.com/dawn/0000 [ monterey webgpu-adapter-default ] webgpu:web_platform,canvas,configure:format:canvasType="offscreen";format="rgba8unorm" [ Failure ]
crbug.com/dawn/0000 [ monterey webgpu-adapter-swiftshader ] webgpu:web_platform,canvas,configure:format:canvasType="offscreen";format="rgba8unorm" [ Failure ]
crbug.com/dawn/0000 [ monterey webgpu-adapter-default ] webgpu:web_platform,canvas,configure:format:canvasType="onscreen";format="rgba8unorm" [ Failure ]
crbug.com/dawn/0000 [ monterey webgpu-adapter-swiftshader ] webgpu:web_platform,canvas,configure:format:canvasType="onscreen";format="rgba8unorm" [ Failure ]
crbug.com/dawn/0000 [ monterey ] webgpu:web_platform,canvas,configure:alpha_mode:* [ Failure ]
crbug.com/dawn/0000 [ monterey ] webgpu:web_platform,canvas,configure:defaults:* [ Failure ]
crbug.com/dawn/0000 [ monterey ] webgpu:web_platform,canvas,configure:device:* [ Failure ]
crbug.com/dawn/0000 [ monterey ] webgpu:web_platform,canvas,configure:format:canvasType="offscreen";format="rgba8unorm" [ Failure ]
crbug.com/dawn/0000 [ monterey ] webgpu:web_platform,canvas,configure:format:canvasType="onscreen";format="rgba8unorm" [ Failure ]
crbug.com/dawn/0000 [ monterey webgpu-adapter-default ] webgpu:web_platform,canvas,configure:usage:* [ Failure ]
crbug.com/dawn/0000 [ monterey webgpu-adapter-swiftshader ] webgpu:web_platform,canvas,configure:usage:* [ Failure ]
crbug.com/dawn/0000 [ ubuntu webgpu-adapter-default ] webgpu:web_platform,canvas,configure:usage:* [ Failure ]
crbug.com/dawn/0000 [ monterey webgpu-adapter-default ] webgpu:web_platform,canvas,getCurrentTexture:* [ Failure ]
crbug.com/dawn/0000 [ monterey webgpu-adapter-swiftshader ] webgpu:web_platform,canvas,getCurrentTexture:* [ Failure ]
crbug.com/dawn/0000 [ monterey ] webgpu:web_platform,canvas,getCurrentTexture:* [ Failure ]