From 27f480b7e62f865c46d4b91eee8757ab5172a5ed Mon Sep 17 00:00:00 2001 From: Ben Clayton Date: Fri, 29 Apr 2022 11:06:34 +0000 Subject: [PATCH] tools: Add src/cts/expectations Update() Implement the logic to update an existing expectations file with new results. Bug: dawn:1342 Change-Id: Idcbad57946712539cc5f0d238f89a2adf42f5aa0 Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/87702 Reviewed-by: Dan Sinclair Commit-Queue: Ben Clayton --- tools/src/cts/expectations/diagnostic.go | 56 ++ tools/src/cts/expectations/parse.go | 22 +- tools/src/cts/expectations/parse_test.go | 10 +- tools/src/cts/expectations/update.go | 592 +++++++++++++++++++ tools/src/cts/expectations/update_test.go | 672 ++++++++++++++++++++++ tools/src/cts/result/result.go | 8 + tools/src/cts/result/result_test.go | 90 +++ 7 files changed, 1427 insertions(+), 23 deletions(-) create mode 100644 tools/src/cts/expectations/diagnostic.go create mode 100644 tools/src/cts/expectations/update.go create mode 100644 tools/src/cts/expectations/update_test.go diff --git a/tools/src/cts/expectations/diagnostic.go b/tools/src/cts/expectations/diagnostic.go new file mode 100644 index 0000000000..c7059aa105 --- /dev/null +++ b/tools/src/cts/expectations/diagnostic.go @@ -0,0 +1,56 @@ +// Copyright 2022 The Dawn Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package expectations + +import ( + "fmt" + "strings" +) + +// Severity is an enumerator of diagnostic severity +type Severity string + +const ( + Error Severity = "error" + Warning Severity = "warning" + Note Severity = "note" +) + +// Diagnostic holds a line, column, message and severity. +// Diagnostic also implements the 'error' interface. +type Diagnostic struct { + Severity Severity + Line int // 1-based + Column int // 1-based + Message string +} + +func (e Diagnostic) String() string { + sb := &strings.Builder{} + if e.Line > 0 { + fmt.Fprintf(sb, "%v", e.Line) + if e.Column > 0 { + fmt.Fprintf(sb, ":%v", e.Column) + } + sb.WriteString(" ") + } + sb.WriteString(string(e.Severity)) + sb.WriteString(": ") + sb.WriteString(e.Message) + return sb.String() +} + +// Error implements the 'error' interface. +func (e Diagnostic) Error() string { return e.String() } diff --git a/tools/src/cts/expectations/parse.go b/tools/src/cts/expectations/parse.go index 7331f0315e..aafb800a4f 100644 --- a/tools/src/cts/expectations/parse.go +++ b/tools/src/cts/expectations/parse.go @@ -15,7 +15,6 @@ package expectations import ( - "fmt" "strings" "dawn.googlesource.com/dawn/tools/src/cts/result" @@ -26,19 +25,6 @@ const ( tagHeaderEnd = `END TAG HEADER` ) -// SyntaxError is the error type returned by Parse() when a syntax error is -// encountered. -type SyntaxError struct { - Line int // 1-based - Column int // 1-based - Message string -} - -// Error implements the 'error' interface. -func (e SyntaxError) Error() string { - return fmt.Sprintf("%v:%v: %v", e.Line, e.Column, e.Message) -} - // Parse parses an expectations file, returning the Content func Parse(body string) (Content, error) { // LineType is an enumerator classifying the 'type' of the line. @@ -130,11 +116,11 @@ func Parse(body string) (Content, error) { // syntaxErr is a helper for returning a SyntaxError with the current // line and column index. syntaxErr := func(at Token, msg string) error { - column := at.start + 1 - if column == 1 { - column = len(l) + 1 + columnIdx := at.start + 1 + if columnIdx == 1 { + columnIdx = len(l) + 1 } - return SyntaxError{lineIdx, column, msg} + return Diagnostic{Error, lineIdx, columnIdx, msg} } // peek returns the next token without consuming it. diff --git a/tools/src/cts/expectations/parse_test.go b/tools/src/cts/expectations/parse_test.go index af233e1b4c..041080b5c7 100644 --- a/tools/src/cts/expectations/parse_test.go +++ b/tools/src/cts/expectations/parse_test.go @@ -432,27 +432,27 @@ crbug.com/456 [ Mac ] ghi_jkl [ PASS ] { name: "err missing tag ']'", in: `[`, - expectErr: "1:2: expected ']' for tags", + expectErr: "1:2 error: expected ']' for tags", }, ///////////////////////////////////////////////////////////////////// { name: "err missing test query", in: `[ a ]`, - expectErr: "1:6: expected test query", + expectErr: "1:6 error: expected test query", }, ///////////////////////////////////////////////////////////////////// { name: "err missing status EOL", in: `[ a ] b`, - expectErr: "1:8: expected status", + expectErr: "1:8 error: expected status", }, ///////////////////////////////////////////////////////////////////// { name: "err missing status comment", in: `[ a ] b # c`, - expectErr: "1:9: expected status", + expectErr: "1:9 error: expected status", }, ///////////////////////////////////////////////////////////////////// { name: "err missing status ']'", in: `[ a ] b [ c`, - expectErr: "1:12: expected ']' for status", + expectErr: "1:12 error: expected ']' for status", }, } { diff --git a/tools/src/cts/expectations/update.go b/tools/src/cts/expectations/update.go new file mode 100644 index 0000000000..13a11ab2a5 --- /dev/null +++ b/tools/src/cts/expectations/update.go @@ -0,0 +1,592 @@ +// Copyright 2022 The Dawn Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package expectations + +import ( + "errors" + "fmt" + "sort" + "strings" + + "dawn.googlesource.com/dawn/tools/src/container" + "dawn.googlesource.com/dawn/tools/src/cts/query" + "dawn.googlesource.com/dawn/tools/src/cts/result" +) + +// Update performs an incremental update on the expectations using the provided +// results. +// +// Update will: +// • Remove any expectation lines that have a query where no results match. +// • Remove expectations lines that are in a chunk which is not annotated with +// 'KEEP', and all test results have the status 'Pass'. +// • Remove chunks that have had all expectation lines removed. +// • Appends new chunks for flaky and failing tests which are not covered by +// existing expectation lines. +// +// Update returns a list of diagnostics for things that should be addressed. +func (c *Content) Update(results result.List) ([]Diagnostic, error) { + // Make a copy of the results. This code mutates the list. + results = append(result.List{}, results...) + + // Replace statuses that the CTS runner doesn't recognize with 'Failure' + simplifyStatuses(results) + + // Produce a list of tag sets. + // We reverse the declared order, as webgpu-cts/expectations.txt lists the + // most important first (OS, GPU, etc), and result.MinimalVariantTags will + // prioritize folding away the earlier tag-sets. + tagSets := make([]result.Tags, len(c.Tags.Sets)) + for i, s := range c.Tags.Sets { + tagSets[len(tagSets)-i-1] = s.Tags + } + + // Update those expectations! + u := updater{ + in: *c, + out: Content{}, + qt: newQueryTree(results), + tagSets: tagSets, + } + if err := u.build(); err != nil { + return nil, fmt.Errorf("while updating expectations: %w", err) + } + + *c = u.out + return u.diags, nil +} + +// updater holds the state used for updating the expectations +type updater struct { + in Content // the original expectations Content + out Content // newly built expectations Content + qt queryTree // the query tree + diags []Diagnostic // diagnostics raised during update + tagSets []result.Tags // reverse-ordered tag-sets of 'in' +} + +// simplifyStatuses replaces all result statuses that are not 'Pass', +// 'RetryOnFailure', 'Slow', 'Skip' with 'Failure'. +func simplifyStatuses(results result.List) { + for i, r := range results { + switch r.Status { + case result.Pass, result.RetryOnFailure, result.Slow, result.Skip: + // keep + default: + results[i].Status = result.Failure + } + } +} + +const ( + // Status used to mark results that have been already handled by an + // expectation. + consumed result.Status = "<>" + // Chunk comment for new flakes + newFlakesComment = "# New flakes. Please triage:" + // Chunk comment for new failures + newFailuresComment = "# New failures. Please triage:" +) + +// queryTree holds tree of queries to all results (no filtering by tag or +// status). The queryTree is used to glob all the results that match a +// particular query. +type queryTree struct { + // All the results. + results result.List + // consumedAt is a list of line numbers for the i'th result in 'results' + // Initially all line numbers are 0. When a result is consumed the line + // number is set. + consumedAt []int + // Each tree node holds a list of indices to results. + tree query.Tree[[]int] +} + +// newQueryTree builds the queryTree from the list of results. +func newQueryTree(results result.List) queryTree { + // Build a map of query to result indices + queryToIndices := map[query.Query][]int{} + for i, r := range results { + l := queryToIndices[r.Query] + l = append(l, i) + queryToIndices[r.Query] = l + } + + // Construct the query tree to result indices + tree := query.Tree[[]int]{} + for query, indices := range queryToIndices { + if err := tree.Add(query, indices); err != nil { + // Unreachable: The only error we could get is duplicate data for + // the same query, which should be impossible. + panic(err) + } + } + + consumedAt := make([]int, len(results)) + return queryTree{results, consumedAt, tree} +} + +// glob returns the list of results matching the given tags under (or with) the +// given query. +func (qt *queryTree) glob(q query.Query) (result.List, error) { + glob, err := qt.tree.Glob(q) + if err != nil { + return nil, fmt.Errorf("while gathering results for query '%v': %w", q, err) + } + + out := result.List{} + for _, indices := range glob { + for _, idx := range indices.Data { + out = append(out, qt.results[idx]) + } + } + + return out, nil +} + +// globAndCheckForCollisions returns the list of results matching the given tags +// under (or with) the given query. +// globAndCheckForCollisions will return an error if any of the results are +// already consumed by a non-zero line. The non-zero line distinguishes between +// results consumed by expectations declared in the input (non-zero line), vs +// those that were introduced by the update (zero line). We only want to error +// if there's a collision in user declared expectations. +func (qt *queryTree) globAndCheckForCollisions(q query.Query, t result.Tags) (result.List, error) { + glob, err := qt.tree.Glob(q) + if err != nil { + return nil, err + } + + out := result.List{} + for _, indices := range glob { + for _, idx := range indices.Data { + if r := qt.results[idx]; r.Tags.ContainsAll(t) { + if at := qt.consumedAt[idx]; at > 0 { + if len(t) > 0 { + return nil, fmt.Errorf("%v %v collides with expectation at line %v", t, q, at) + } + return nil, fmt.Errorf("%v collides with expectation at line %v", q, at) + } + out = append(out, r) + } + } + } + return out, nil +} + +// markAsConsumed marks all the results matching the given tags +// under (or with) the given query, as consumed. +// line is used to record the line at which the results were consumed. If the +// results were consumed as part of generating new expectations then line should +// be 0. See queryTree.globAndCheckForCollisions(). +func (qt *queryTree) markAsConsumed(q query.Query, t result.Tags, line int) { + if glob, err := qt.tree.Glob(q); err == nil { + for _, indices := range glob { + for _, idx := range indices.Data { + r := &qt.results[idx] + if r.Tags.ContainsAll(t) { + r.Status = consumed + qt.consumedAt[idx] = line + } + } + } + } +} + +// build is the updater top-level function. +// build first appends to u.out all chunks from 'u.in' with expectations updated +// using the new results, and then appends any new expectations to u.out. +func (u *updater) build() error { + // Update all the existing chunks + for _, in := range u.in.Chunks { + out := u.chunk(in) + + // If all chunk had expectations, but now they've gone, remove the chunk + if len(in.Expectations) > 0 && len(out.Expectations) == 0 { + continue + } + if out.IsBlankLine() { + u.out.MaybeAddBlankLine() + continue + } + u.out.Chunks = append(u.out.Chunks, out) + } + + // Emit new expectations (flaky, failing) + if err := u.addNewExpectations(); err != nil { + return fmt.Errorf("failed to add new expectations: %w", err) + } + + return nil +} + +// chunk returns a new Chunk, based on 'in', with the expectations updated. +func (u *updater) chunk(in Chunk) Chunk { + if len(in.Expectations) == 0 { + return in // Just a comment / blank line + } + + // Skip over any untriaged failures / flake chunks. + // We'll just rebuild them at the end. + if len(in.Comments) > 0 { + switch in.Comments[0] { + case newFailuresComment, newFlakesComment: + return Chunk{} + } + } + + keep := false // Does the chunk comment contain 'KEEP' ? + for _, l := range in.Comments { + if strings.Contains(l, "KEEP") { + keep = true + break + } + } + + // Begin building the output chunk. + // Copy over the chunk's comments. + out := Chunk{Comments: in.Comments} + + // Build the new chunk's expectations + for _, exIn := range in.Expectations { + exOut := u.expectation(exIn, keep) + out.Expectations = append(out.Expectations, exOut...) + } + + // Sort the expectations to keep things clean and tidy. + sort.Slice(out.Expectations, func(i, j int) bool { + switch { + case out.Expectations[i].Query < out.Expectations[j].Query: + return true + case out.Expectations[i].Query > out.Expectations[j].Query: + return false + } + a := result.TagsToString(out.Expectations[i].Tags) + b := result.TagsToString(out.Expectations[j].Tags) + switch { + case a < b: + return true + case a > b: + return false + } + return false + }) + + return out +} + +// chunk returns a new list of Expectations, based on the Expectation 'in', +// using the new result data. +func (u *updater) expectation(in Expectation, keep bool) []Expectation { + // noResults is a helper for returning when the expectation has no test + // results. + // If the expectation has an expected 'Skip' result, then we're likely + // to be missing results (as the test was not run). In this situation + // the expectation is preserved, and no diagnostics are raised. + // If the expectation did not have a 'Skip' result, then a diagnostic will + // be raised and the expectation will be removed. + noResults := func() []Expectation { + if container.NewSet(in.Status...).Contains(string(result.Skip)) { + return []Expectation{in} + } + // Expectation does not have a 'Skip' result. + if len(in.Tags) > 0 { + u.diag(Warning, in.Line, "no results found for '%v' with tags %v", in.Query, in.Tags) + } else { + u.diag(Warning, in.Line, "no results found for '%v'", in.Query) + } + // Remove the no-results expectation + return []Expectation{} + } + + // Grab all the results that match the expectation's query + q := query.Parse(in.Query) + + // Glob the results for the expectation's query + tag combination. + // Ensure that none of these are already consumed. + results, err := u.qt.globAndCheckForCollisions(q, in.Tags) + // If we can't find any results for this query + tag combination, then bail. + switch { + case errors.As(err, &query.ErrNoDataForQuery{}): + return noResults() + case err != nil: + u.diag(Error, in.Line, "%v", err) + return []Expectation{} + case len(results) == 0: + return noResults() + } + + // Before returning, mark all the results as consumed. + // Note: this has to happen *after* we've generated the new expectations, as + // marking the results as 'consumed' will impact the logic of + // expectationsForRoot() + defer u.qt.markAsConsumed(q, in.Tags, in.Line) + + if keep { // Expectation chunk was marked with 'KEEP' + // Add a diagnostic if all tests of the expectation were 'Pass' + if s := results.Statuses(); len(s) == 1 && s.One() == result.Pass { + if c := len(results); c > 1 { + u.diag(Note, in.Line, "all %d tests now pass", len(results)) + } else { + u.diag(Note, in.Line, "test now passes") + } + } + return []Expectation{in} + } + + // Rebuild the expectations for this query. + return u.expectationsForRoot(q, in.Line, in.Bug, in.Comment) +} + +// addNewExpectations (potentially) appends to 'u.out' chunks for new flaky and +// failing tests. +func (u *updater) addNewExpectations() error { + // Scan the full result list to obtain all the test variants + // (unique tag combinations). + allVariants := u.qt.results.Variants() + + // For each variant: + // • Build a query tree using the results filtered to the variant, and then + // reduce the tree. + // • Take all the reduced-tree leaf nodes, and add these to 'roots'. + // Once we've collected all the roots, we'll use these to build the + // expectations across the reduced set of tags. + roots := container.NewMap[string, query.Query]() + for _, variant := range allVariants { + // Build a tree from the results matching the given variant. + tree, err := u.qt.results.FilterByVariant(variant).StatusTree() + if err != nil { + return fmt.Errorf("while building tree for tags '%v': %w", variant, err) + } + // Reduce the tree. + tree.Reduce(treeReducer) + // Add all the reduced leaf nodes to 'roots'. + for _, qd := range tree.List() { + roots.Add(qd.Query.String(), qd.Query) + } + } + + // Build all the expectations for each of the roots. + expectations := []Expectation{} + for _, root := range roots.Values() { + expectations = append(expectations, u.expectationsForRoot( + root, // Root query + 0, // Line number + "crbug.com/dawn/0000", // Bug + "", // Comment + )...) + } + + // Bin the expectations by failure or flake. + flakes, failures := []Expectation{}, []Expectation{} + for _, r := range expectations { + if container.NewSet(r.Status...).Contains(string(result.RetryOnFailure)) { + flakes = append(flakes, r) + } else { + failures = append(failures, r) + } + } + + // Create chunks for any flakes and failures, in that order. + for _, group := range []struct { + results []Expectation + comment string + }{ + {flakes, newFlakesComment}, + {failures, newFailuresComment}, + } { + if len(group.results) > 0 { + u.out.MaybeAddBlankLine() + u.out.Chunks = append(u.out.Chunks, Chunk{ + Comments: []string{group.comment}, + Expectations: group.results, + }) + } + } + + return nil +} + +// expectationsForRoot builds a list of expectations that cover the failing +// tests for the results under root. +// The returned list of expectations is optimized by reducing queries to the +// most common root, and reducing tags to the smallest required set. +func (u *updater) expectationsForRoot( + root query.Query, // The sub-tree query root + line int, // The originating line, when producing diagnostics + bug string, // The bug to apply to all returned expectations + comment string, // The comment to apply to all returned expectations +) []Expectation { + results, err := u.qt.glob(root) + if err != nil { + u.diag(Error, line, "%v", err) + return nil + } + + // Using the full list of unfiltered tests, generate the minimal set of + // variants (tags) that uniquely classify the results with differing status. + minimalVariants := u. + cleanupTags(results). + MinimalVariantTags(u.tagSets) + + // For each minimized variant... + reduced := result.List{} + for _, variant := range minimalVariants { + // Build a query tree from this variant... + tree := result.StatusTree{} + filtered := results.FilterByTags(variant) + for _, r := range filtered { + // Note: variants may overlap, but overlaped queries will have + // identical statuses, so we can just ignore the error for Add(). + tree.Add(r.Query, r.Status) + } + + // ... and reduce the tree by collapsing sub-trees that have common + // statuses. + tree.ReduceUnder(root, treeReducer) + + // Append the reduced tree nodes to the results list + for _, qs := range tree.List() { + reduced = append(reduced, result.Result{ + Query: qs.Query, + Tags: variant, + Status: qs.Data, + }) + } + } + + // Filter out any results that passed or have already been consumed + filtered := reduced.Filter(func(r result.Result) bool { + return r.Status != result.Pass && r.Status != consumed + }) + + // Mark all the new expectation results as consumed. + for _, r := range filtered { + u.qt.markAsConsumed(r.Query, r.Tags, 0) + } + + // Transform the results to expectations. + return u.resultsToExpectations(filtered, bug, comment) +} + +// resultsToExpectations returns a list of expectations from the given results. +// Each expectation will have the same query, tags and status as the input +// result, along with the specified bug and comment. +// +// If the result query target is a test without a wildcard, then the expectation +// will have a wildcard automatically appended. This is to satisfy a requirement +// of the expectation validator. +func (u *updater) resultsToExpectations(results result.List, bug, comment string) []Expectation { + results.Sort() + + out := make([]Expectation, len(results)) + for i, r := range results { + q := r.Query.String() + if r.Query.Target() == query.Tests && !r.Query.IsWildcard() { + // The expectation validator wants a trailing ':' for test queries + q += query.TargetDelimiter + } + out[i] = Expectation{ + Bug: bug, + Tags: r.Tags, + Query: q, + Status: []string{string(r.Status)}, + Comment: comment, + } + } + + return out +} + +// cleanupTags returns a copy of the provided results with: +// • All tags not found in the expectations list removed +// • All but the highest priority tag for any tag-set. +// The tag sets are defined by the `BEGIN TAG HEADER` / `END TAG HEADER` +// section at the top of the expectations file. +func (u *updater) cleanupTags(results result.List) result.List { + return results.TransformTags(func(t result.Tags) result.Tags { + type HighestPrioritySetTag struct { + tag string + priority int + } + // Set name to highest priority tag for that set + best := map[string]HighestPrioritySetTag{} + for tag := range t { + sp, ok := u.in.Tags.ByName[tag] + if ok { + if set := best[sp.Set]; sp.Priority >= set.priority { + best[sp.Set] = HighestPrioritySetTag{tag, sp.Priority} + } + } + } + t = result.NewTags() + for _, ts := range best { + t.Add(ts.tag) + } + return t + }) +} + +// treeReducer is a function that can be used by StatusTree.Reduce() to reduce +// tree nodes with the same status. +// treeReducer will collapse trees nodes if any of the following are true: +// • All child nodes have the same status +// • More than 75% of the child nodes have a non-pass status, and none of the +// children are consumed. +// • There are more than 20 child nodes with a non-pass status, and none of the +// children are consumed. +func treeReducer(statuses []result.Status) *result.Status { + counts := map[result.Status]int{} + for _, s := range statuses { + counts[s] = counts[s] + 1 + } + if len(counts) == 1 { + return &statuses[0] // All the same status + } + if counts[consumed] > 0 { + return nil // Partially consumed trees cannot be merged + } + highestNonPassCount := 0 + highestNonPassStatus := result.Failure + for s, n := range counts { + if s != result.Pass { + if percent := (100 * n) / len(statuses); percent > 75 { + // Over 75% of all the children are of non-pass status s. + return &s + } + if n > highestNonPassCount { + highestNonPassCount = n + highestNonPassStatus = s + } + } + } + + if highestNonPassCount > 20 { + // Over 20 child node failed. + return &highestNonPassStatus + } + + return nil +} + +// diag appends a new diagnostic to u.diags with the given severity, line and +// message. +func (u *updater) diag(severity Severity, line int, msg string, args ...interface{}) { + u.diags = append(u.diags, Diagnostic{ + Severity: severity, + Line: line, + Message: fmt.Sprintf(msg, args...), + }) +} diff --git a/tools/src/cts/expectations/update_test.go b/tools/src/cts/expectations/update_test.go new file mode 100644 index 0000000000..69cc314e6e --- /dev/null +++ b/tools/src/cts/expectations/update_test.go @@ -0,0 +1,672 @@ +// Copyright 2022 The Dawn Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package expectations_test + +import ( + "strings" + "testing" + + "dawn.googlesource.com/dawn/tools/src/cts/expectations" + "dawn.googlesource.com/dawn/tools/src/cts/query" + "dawn.googlesource.com/dawn/tools/src/cts/result" + "github.com/google/go-cmp/cmp" +) + +var Q = query.Parse + +func TestUpdate(t *testing.T) { + header := `# BEGIN TAG HEADER +# OS +# tags: [ os-a os-b os-c ] +# GPU +# tags: [ gpu-a gpu-b gpu-c ] +# END TAG HEADER +` + headerLines := strings.Count(header, "\n") + + type Test struct { + name string + expectations string + results result.List + updated string + diagnostics []expectations.Diagnostic + err string + } + for _, test := range []Test{ + { ////////////////////////////////////////////////////////////////////// + name: "empty results", + expectations: ``, + results: result.List{}, + }, + { ////////////////////////////////////////////////////////////////////// + name: "no results found", + expectations: ` +crbug.com/a/123 a:missing,test,result:* [ Failure ] +crbug.com/a/123 [ tag ] another:missing,test,result:* [ Failure ] + +some:other,test:* [ Failure ] +`, + results: result.List{ + result.Result{ + Query: Q("some:other,test:*"), + Tags: result.NewTags("os-a", "gpu-a"), + Status: result.Failure, + }, + result.Result{ + Query: Q("some:other,test:*"), + Tags: result.NewTags("os-b", "gpu-b"), + Status: result.Failure, + }, + }, + updated: ` +some:other,test:* [ Failure ] +`, + diagnostics: []expectations.Diagnostic{ + { + Severity: expectations.Warning, + Line: headerLines + 2, + Message: "no results found for 'a:missing,test,result:*'", + }, + { + Severity: expectations.Warning, + Line: headerLines + 3, + Message: "no results found for 'another:missing,test,result:*' with tags [tag]", + }, + }, + }, + { ////////////////////////////////////////////////////////////////////// + name: "no results found KEEP", + expectations: ` +# KEEP +crbug.com/a/123 a:missing,test,result:* [ Failure ] + +some:other,test:* [ Failure ] +`, + results: result.List{ + result.Result{ + Query: Q("some:other,test:*"), + Tags: result.NewTags("os-a", "gpu-a"), + Status: result.Failure, + }, + result.Result{ + Query: Q("some:other,test:*"), + Tags: result.NewTags("os-b", "gpu-b"), + Status: result.Failure, + }, + }, + updated: ` +some:other,test:* [ Failure ] +`, + diagnostics: []expectations.Diagnostic{ + { + Severity: expectations.Warning, + Line: headerLines + 3, + Message: "no results found for 'a:missing,test,result:*'", + }, + }, + }, + { ////////////////////////////////////////////////////////////////////// + name: "no results found Skip", + expectations: ` +crbug.com/a/123 a:missing,test,result:* [ Skip ] + +some:other,test:* [ Failure ] +`, + results: result.List{ + result.Result{ + Query: Q("some:other,test:*"), + Tags: result.NewTags("os-a", "gpu-a"), + Status: result.Failure, + }, + result.Result{ + Query: Q("some:other,test:*"), + Tags: result.NewTags("os-b", "gpu-b"), + Status: result.Failure, + }, + }, + updated: ` +crbug.com/a/123 a:missing,test,result:* [ Skip ] + +some:other,test:* [ Failure ] +`, + }, + { ////////////////////////////////////////////////////////////////////// + name: "simple expectation collision", + expectations: ` +a:b,c:* [ Failure ] +a:b,c:* [ Failure ] +`, + results: result.List{ + result.Result{ + Query: Q("a:b,c:d"), + Tags: result.NewTags("os-a", "os-c", "gpu-b"), + Status: result.Failure, + }, + }, + updated: ` +a:b,c:* [ Failure ] +`, + diagnostics: []expectations.Diagnostic{ + { + Severity: expectations.Error, + Line: headerLines + 3, + Message: "a:b,c:* collides with expectation at line 8", + }, + }, + }, + { ////////////////////////////////////////////////////////////////////// + name: "simple expectation with tags", + expectations: ` +[ os-a ] a:b,c:* [ Failure ] +[ gpu-b ] a:b,c:* [ Failure ] +`, + results: result.List{ + result.Result{ + Query: Q("a:b,c:d"), + Tags: result.NewTags("os-a", "os-c", "gpu-b"), + Status: result.Failure, + }, + }, + updated: ` +a:b,c:* [ Failure ] +`, + diagnostics: []expectations.Diagnostic{ + { + Severity: expectations.Error, + Line: headerLines + 3, + Message: "[gpu-b] a:b,c:* collides with expectation at line 8", + }, + }, + }, + { ////////////////////////////////////////////////////////////////////// + name: "simple expectation collision KEEP", + expectations: ` +# KEEP +a:b,c:* [ Failure ] +a:b,c:* [ Failure ] +`, + results: result.List{ + result.Result{ + Query: Q("a:b,c:d"), + Tags: result.NewTags("os-a", "os-c", "gpu-b"), + Status: result.Failure, + }, + }, + updated: ` +# KEEP +a:b,c:* [ Failure ] +`, + diagnostics: []expectations.Diagnostic{ + { + Severity: expectations.Error, + Line: headerLines + 4, + Message: "a:b,c:* collides with expectation at line 9", + }, + }, + }, + { ////////////////////////////////////////////////////////////////////// + name: "collision with child-expectation", + expectations: ` +a:b:x:* [ Failure ] +a:b:* [ Failure ] +`, + results: result.List{ + result.Result{ + Query: Q("a:b:x:*"), + Tags: result.NewTags("os-a", "os-c", "gpu-b"), + Status: result.Failure, + }, + result.Result{ + Query: Q("a:b:y:*"), + Tags: result.NewTags("os-a", "os-c", "gpu-b"), + Status: result.Failure, + }, + }, + updated: ` +a:b:x:* [ Failure ] + +# New failures. Please triage: +crbug.com/dawn/0000 a:b:y:* [ Failure ] +`, + diagnostics: []expectations.Diagnostic{ + { + Severity: expectations.Error, + Line: headerLines + 3, + Message: "a:b:* collides with expectation at line 8", + }, + }, + }, + { ////////////////////////////////////////////////////////////////////// + name: "collision with parent-expectation", + expectations: ` +a:b:* [ Failure ] +a:b:x:* [ Failure ] +`, + results: result.List{ + result.Result{ + Query: Q("a:b:x:*"), + Tags: result.NewTags("os-a", "os-c", "gpu-b"), + Status: result.Failure, + }, + result.Result{ + Query: Q("a:b:y:*"), + Tags: result.NewTags("os-a", "os-c", "gpu-b"), + Status: result.Failure, + }, + }, + updated: ` +a:b:* [ Failure ] +`, + diagnostics: []expectations.Diagnostic{ + { + Severity: expectations.Error, + Line: headerLines + 3, + Message: "a:b:x:* collides with expectation at line 8", + }, + }, + }, + { ////////////////////////////////////////////////////////////////////// + name: "expectation test now passes", + expectations: ` +crbug.com/a/123 [ gpu-a os-a ] a:b,c:* [ Failure ] +crbug.com/a/123 [ gpu-b os-b ] a:b,c:* [ Failure ] +`, + results: result.List{ + result.Result{ + Query: Q("a:b,c:*"), + Tags: result.NewTags("os-a", "gpu-a"), + Status: result.Pass, + }, + result.Result{ + Query: Q("a:b,c:*"), + Tags: result.NewTags("os-b", "gpu-b"), + Status: result.Abort, + }, + }, + updated: ` +crbug.com/a/123 [ os-b ] a:b,c:* [ Failure ] +`, + }, + { ////////////////////////////////////////////////////////////////////// + name: "expectation case now passes", + expectations: ` +crbug.com/a/123 [ gpu-a os-a ] a:b,c:d [ Failure ] +crbug.com/a/123 [ gpu-b os-b ] a:b,c:d [ Failure ] +`, + results: result.List{ + result.Result{ + Query: Q("a:b,c:d"), + Tags: result.NewTags("os-a", "gpu-a"), + Status: result.Pass, + }, + result.Result{ + Query: Q("a:b,c:d"), + Tags: result.NewTags("os-b", "gpu-b"), + Status: result.Abort, + }, + }, + updated: ` +crbug.com/a/123 [ os-b ] a:b,c:d: [ Failure ] +`, + }, + { ////////////////////////////////////////////////////////////////////// + name: "expectation case now passes KEEP - single", + expectations: ` +# KEEP +crbug.com/a/123 [ gpu-a os-a ] a:b,c:d [ Failure ] +crbug.com/a/123 [ gpu-b os-b ] a:b,c:d [ Failure ] +`, + results: result.List{ + result.Result{ + Query: Q("a:b,c:d"), + Tags: result.NewTags("os-a", "gpu-a"), + Status: result.Pass, + }, + result.Result{ + Query: Q("a:b,c:d"), + Tags: result.NewTags("os-b", "gpu-b"), + Status: result.Abort, + }, + }, + updated: ` +# KEEP +crbug.com/a/123 [ gpu-a os-a ] a:b,c:d [ Failure ] +crbug.com/a/123 [ gpu-b os-b ] a:b,c:d [ Failure ] +`, + diagnostics: []expectations.Diagnostic{ + { + Severity: expectations.Note, + Line: headerLines + 3, + Message: "test now passes", + }, + }, + }, + { ////////////////////////////////////////////////////////////////////// + name: "expectation case now passes KEEP - multiple", + expectations: ` +# KEEP +crbug.com/a/123 a:b,c:d:* [ Failure ] +`, + results: result.List{ + result.Result{Query: Q("a:b,c:d:a"), Status: result.Pass}, + result.Result{Query: Q("a:b,c:d:b"), Status: result.Pass}, + result.Result{Query: Q("a:b,c:d:c"), Status: result.Pass}, + result.Result{Query: Q("a:b,c:d:d"), Status: result.Pass}, + }, + updated: ` +# KEEP +crbug.com/a/123 a:b,c:d:* [ Failure ] +`, + diagnostics: []expectations.Diagnostic{ + { + Severity: expectations.Note, + Line: headerLines + 3, + Message: "all 4 tests now pass", + }, + }, + }, + { ////////////////////////////////////////////////////////////////////// + name: "new test results", + expectations: `# A comment`, + results: result.List{ + result.Result{ + Query: Q("suite:dir_a,dir_b:test_a:*"), + Tags: result.NewTags("os-a", "gpu-a"), + Status: result.Abort, + }, + result.Result{ + Query: Q("suite:dir_a,dir_b:test_a:*"), + Tags: result.NewTags("os-a", "gpu-b"), + Status: result.Abort, + }, + result.Result{ + Query: Q("suite:dir_a,dir_b:test_c:case=4;*"), + Tags: result.NewTags("os-b", "gpu-b"), + Status: result.Crash, + }, + result.Result{ + Query: Q("suite:dir_a,dir_b:test_c:case=5;*"), + Tags: result.NewTags("os-b", "gpu-b"), + Status: result.RetryOnFailure, + }, + result.Result{ + Query: Q("suite:dir_a,dir_b:test_b;case=5;*"), + Tags: result.NewTags("os-b", "gpu-b"), + Status: result.Pass, + }, + result.Result{ + Query: Q("suite:dir_a,dir_b:test_b:*"), + Tags: result.NewTags("os-a", "gpu-a"), + Status: result.Skip, + }, + result.Result{ + Query: Q("suite:dir_a,dir_b:test_b:*"), + Tags: result.NewTags("os-b", "gpu-a"), + Status: result.Pass, + }, + result.Result{ + Query: Q("suite:dir_a,dir_b:test_c:*"), + Tags: result.NewTags("os-a", "gpu-a"), + Status: result.RetryOnFailure, + }, + }, + updated: `# A comment + +# New flakes. Please triage: +crbug.com/dawn/0000 [ gpu-a os-a ] suite:dir_a,dir_b:test_c:* [ RetryOnFailure ] +crbug.com/dawn/0000 [ gpu-b os-b ] suite:dir_a,dir_b:test_c:case=5;* [ RetryOnFailure ] + +# New failures. Please triage: +crbug.com/dawn/0000 [ gpu-b os-a ] suite:* [ Failure ] +crbug.com/dawn/0000 [ gpu-a os-a ] suite:dir_a,dir_b:test_a:* [ Failure ] +crbug.com/dawn/0000 [ gpu-a os-a ] suite:dir_a,dir_b:test_b:* [ Skip ] +crbug.com/dawn/0000 [ gpu-b os-b ] suite:dir_a,dir_b:test_c:case=4;* [ Failure ] +`, + }, + { ////////////////////////////////////////////////////////////////////// + name: "filter unknown tags", + expectations: ``, + results: result.List{ + result.Result{ + Query: Q("a:b,c:*"), + Tags: result.NewTags("os-a", "gpu-x"), + Status: result.Failure, + }, + result.Result{ + Query: Q("a:b,c:*"), + Tags: result.NewTags("os-b", "gpu-x"), + Status: result.Crash, + }, + result.Result{ + Query: Q("a:b,c:*"), + Tags: result.NewTags("os-x", "gpu-b"), + Status: result.Failure, + }, + result.Result{ + Query: Q("a:b,c:*"), + Tags: result.NewTags("os-x", "gpu-a"), + Status: result.Crash, + }, + result.Result{ + Query: Q("a:b,c:*"), + Tags: result.NewTags("os-c", "gpu-c"), + Status: result.Pass, + }, + }, + updated: ` +# New failures. Please triage: +crbug.com/dawn/0000 [ gpu-a ] a:* [ Failure ] +crbug.com/dawn/0000 [ gpu-b ] a:* [ Failure ] +crbug.com/dawn/0000 [ os-a ] a:* [ Failure ] +crbug.com/dawn/0000 [ os-b ] a:* [ Failure ] +`, + }, + { ////////////////////////////////////////////////////////////////////// + name: "prioritize tag sets", + expectations: ``, + results: result.List{ + result.Result{ + Query: Q("a:b,c:*"), + Tags: result.NewTags("os-a", "os-c", "gpu-b"), + Status: result.Failure, + }, + result.Result{ + Query: Q("a:b,c:*"), + Tags: result.NewTags("gpu-a", "os-b", "gpu-c"), + Status: result.Failure, + }, + result.Result{ + Query: Q("a:b,c:*"), + Tags: result.NewTags("os-c", "gpu-c"), + Status: result.Pass, + }, + }, + updated: ` +# New failures. Please triage: +crbug.com/dawn/0000 [ gpu-b os-c ] a:* [ Failure ] +crbug.com/dawn/0000 [ gpu-c os-b ] a:* [ Failure ] +`, + }, + { ////////////////////////////////////////////////////////////////////// + name: "merge when over 75% of children fail", + expectations: ``, + results: result.List{ + result.Result{Query: Q("a:b,c:t0:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t1:*"), Status: result.Pass}, + result.Result{Query: Q("a:b,c:t2:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t3:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t4:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t5:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t6:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t7:*"), Status: result.Pass}, + result.Result{Query: Q("a:b,c:t8:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t9:*"), Status: result.Failure}, + }, + updated: ` +# New failures. Please triage: +crbug.com/dawn/0000 a:* [ Failure ] +`, + }, + { ////////////////////////////////////////////////////////////////////// + name: "don't merge when under 75% of children fail", + expectations: ``, + results: result.List{ + result.Result{Query: Q("a:b,c:t0:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t1:*"), Status: result.Pass}, + result.Result{Query: Q("a:b,c:t2:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t3:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t4:*"), Status: result.Pass}, + result.Result{Query: Q("a:b,c:t5:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t6:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t7:*"), Status: result.Pass}, + result.Result{Query: Q("a:b,c:t8:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t9:*"), Status: result.Failure}, + }, + updated: ` +# New failures. Please triage: +crbug.com/dawn/0000 a:b,c:t0:* [ Failure ] +crbug.com/dawn/0000 a:b,c:t2:* [ Failure ] +crbug.com/dawn/0000 a:b,c:t3:* [ Failure ] +crbug.com/dawn/0000 a:b,c:t5:* [ Failure ] +crbug.com/dawn/0000 a:b,c:t6:* [ Failure ] +crbug.com/dawn/0000 a:b,c:t8:* [ Failure ] +crbug.com/dawn/0000 a:b,c:t9:* [ Failure ] +`, + }, + { ////////////////////////////////////////////////////////////////////// + name: "merge when over 20 children fail", + expectations: ``, + results: result.List{ // 21 failures, 70% fail + result.Result{Query: Q("a:b,c:t00:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t01:*"), Status: result.Pass}, + result.Result{Query: Q("a:b,c:t02:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t03:*"), Status: result.Pass}, + result.Result{Query: Q("a:b,c:t04:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t05:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t06:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t07:*"), Status: result.Pass}, + result.Result{Query: Q("a:b,c:t08:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t09:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t10:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t11:*"), Status: result.Pass}, + result.Result{Query: Q("a:b,c:t12:*"), Status: result.Pass}, + result.Result{Query: Q("a:b,c:t13:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t14:*"), Status: result.Pass}, + result.Result{Query: Q("a:b,c:t15:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t16:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t17:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t18:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t19:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t20:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t21:*"), Status: result.Pass}, + result.Result{Query: Q("a:b,c:t22:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t23:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t24:*"), Status: result.Pass}, + result.Result{Query: Q("a:b,c:t25:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t26:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t27:*"), Status: result.Pass}, + result.Result{Query: Q("a:b,c:t28:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t29:*"), Status: result.Failure}, + }, + updated: ` +# New failures. Please triage: +crbug.com/dawn/0000 a:* [ Failure ] +`, + }, + { ////////////////////////////////////////////////////////////////////// + name: "dont merge when under 21 children fail", + expectations: ``, + results: result.List{ // 20 failures, 66% fail + result.Result{Query: Q("a:b,c:t00:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t01:*"), Status: result.Pass}, + result.Result{Query: Q("a:b,c:t02:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t03:*"), Status: result.Pass}, + result.Result{Query: Q("a:b,c:t04:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t05:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t06:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t07:*"), Status: result.Pass}, + result.Result{Query: Q("a:b,c:t08:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t09:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t10:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t11:*"), Status: result.Pass}, + result.Result{Query: Q("a:b,c:t12:*"), Status: result.Pass}, + result.Result{Query: Q("a:b,c:t13:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t14:*"), Status: result.Pass}, + result.Result{Query: Q("a:b,c:t15:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t16:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t17:*"), Status: result.Pass}, + result.Result{Query: Q("a:b,c:t18:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t19:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t20:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t21:*"), Status: result.Pass}, + result.Result{Query: Q("a:b,c:t22:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t23:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t24:*"), Status: result.Pass}, + result.Result{Query: Q("a:b,c:t25:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t26:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t27:*"), Status: result.Pass}, + result.Result{Query: Q("a:b,c:t28:*"), Status: result.Failure}, + result.Result{Query: Q("a:b,c:t29:*"), Status: result.Failure}, + }, + updated: ` +# New failures. Please triage: +crbug.com/dawn/0000 a:b,c:t00:* [ Failure ] +crbug.com/dawn/0000 a:b,c:t02:* [ Failure ] +crbug.com/dawn/0000 a:b,c:t04:* [ Failure ] +crbug.com/dawn/0000 a:b,c:t05:* [ Failure ] +crbug.com/dawn/0000 a:b,c:t06:* [ Failure ] +crbug.com/dawn/0000 a:b,c:t08:* [ Failure ] +crbug.com/dawn/0000 a:b,c:t09:* [ Failure ] +crbug.com/dawn/0000 a:b,c:t10:* [ Failure ] +crbug.com/dawn/0000 a:b,c:t13:* [ Failure ] +crbug.com/dawn/0000 a:b,c:t15:* [ Failure ] +crbug.com/dawn/0000 a:b,c:t16:* [ Failure ] +crbug.com/dawn/0000 a:b,c:t18:* [ Failure ] +crbug.com/dawn/0000 a:b,c:t19:* [ Failure ] +crbug.com/dawn/0000 a:b,c:t20:* [ Failure ] +crbug.com/dawn/0000 a:b,c:t22:* [ Failure ] +crbug.com/dawn/0000 a:b,c:t23:* [ Failure ] +crbug.com/dawn/0000 a:b,c:t25:* [ Failure ] +crbug.com/dawn/0000 a:b,c:t26:* [ Failure ] +crbug.com/dawn/0000 a:b,c:t28:* [ Failure ] +crbug.com/dawn/0000 a:b,c:t29:* [ Failure ] +`, + }, + } { + ex, err := expectations.Parse(header + test.expectations) + if err != nil { + t.Fatalf("'%v': expectations.Parse():\n%v", test.name, err) + } + + errMsg := "" + diagnostics, err := ex.Update(test.results) + if err != nil { + errMsg = err.Error() + } + if diff := cmp.Diff(errMsg, test.err); diff != "" { + t.Errorf("'%v': expectations.Update() error:\n%v", test.name, diff) + } + + if diff := cmp.Diff(diagnostics, test.diagnostics); diff != "" { + t.Errorf("'%v': diagnostics were not as expected:\n%v", test.name, diff) + } + + if diff := cmp.Diff( + strings.Split(ex.String(), "\n"), + strings.Split(header+test.updated, "\n")); diff != "" { + t.Errorf("'%v': updated was not as expected:\n%v", test.name, diff) + } + } +} diff --git a/tools/src/cts/result/result.go b/tools/src/cts/result/result.go index 0ec6095551..e234c42df3 100644 --- a/tools/src/cts/result/result.go +++ b/tools/src/cts/result/result.go @@ -221,6 +221,14 @@ func (l List) FilterByTags(tags Tags) List { }) } +// FilterByVariant returns the results that exactly match the given tags +func (l List) FilterByVariant(tags Tags) List { + str := TagsToString(tags) + return l.Filter(func(r Result) bool { + return len(r.Tags) == len(tags) && TagsToString(r.Tags) == str + }) +} + // Statuses is a set of Status type Statuses = container.Set[Status] diff --git a/tools/src/cts/result/result_test.go b/tools/src/cts/result/result_test.go index 7eae4185a9..bd1eb27f29 100644 --- a/tools/src/cts/result/result_test.go +++ b/tools/src/cts/result/result_test.go @@ -782,6 +782,96 @@ func TestFilterByTags(t *testing.T) { } } +func TestFilterByVariant(t *testing.T) { + type Test struct { + results result.List + tags result.Tags + expect result.List + } + for _, test := range []Test{ + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{ + Query: Q(`a`), + Status: result.Pass, + Tags: result.NewTags("x"), + }, + result.Result{ + Query: Q(`b`), + Status: result.Failure, + Tags: result.NewTags("y"), + }, + result.Result{ + Query: Q(`c`), + Status: result.Pass, + Tags: result.NewTags("x", "y"), + }, + }, + tags: result.NewTags("x", "y"), + expect: result.List{ + result.Result{ + Query: Q(`c`), + Status: result.Pass, + Tags: result.NewTags("x", "y"), + }, + }, + }, + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{ + Query: Q(`a`), + Status: result.Pass, + Tags: result.NewTags("x"), + }, + result.Result{ + Query: Q(`b`), + Status: result.Failure, + Tags: result.NewTags("y"), + }, + result.Result{ + Query: Q(`c`), + Status: result.Pass, + Tags: result.NewTags("x", "y"), + }, + }, + tags: result.NewTags("x"), + expect: result.List{ + result.Result{ + Query: Q(`a`), + Status: result.Pass, + Tags: result.NewTags("x"), + }, + }, + }, + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{ + Query: Q(`a`), + Status: result.Pass, + Tags: result.NewTags("x"), + }, + result.Result{ + Query: Q(`b`), + Status: result.Failure, + Tags: result.NewTags("y"), + }, + result.Result{ + Query: Q(`c`), + Status: result.Pass, + Tags: result.NewTags("x", "y"), + }, + }, + tags: result.NewTags("q"), + expect: result.List{}, + }, + } { + got := test.results.FilterByVariant(test.tags) + if diff := cmp.Diff(got, test.expect); diff != "" { + t.Errorf("Results:\n%v\nFilterByVariant(%v) was not as expected:\n%v", test.results, test.tags, diff) + } + } +} + func TestStatuses(t *testing.T) { type Test struct { results result.List