tools: Add src/cts/expectations Update()
Implement the logic to update an existing expectations file with new results. Bug: dawn:1342 Change-Id: Idcbad57946712539cc5f0d238f89a2adf42f5aa0 Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/87702 Reviewed-by: Dan Sinclair <dsinclair@chromium.org> Commit-Queue: Ben Clayton <bclayton@google.com>
This commit is contained in:
Ben Clayton
Dawn LUCI CQ
parent
9432887ce8
commit
27f480b7e6
|
|
@ -0,0 +1,56 @@
|
|||
// Copyright 2022 The Dawn Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package expectations
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Severity is an enumerator of diagnostic severity
|
||||
type Severity string
|
||||
|
||||
const (
|
||||
Error Severity = "error"
|
||||
Warning Severity = "warning"
|
||||
Note Severity = "note"
|
||||
)
|
||||
|
||||
// Diagnostic holds a line, column, message and severity.
|
||||
// Diagnostic also implements the 'error' interface.
|
||||
type Diagnostic struct {
|
||||
Severity Severity
|
||||
Line int // 1-based
|
||||
Column int // 1-based
|
||||
Message string
|
||||
}
|
||||
|
||||
func (e Diagnostic) String() string {
|
||||
sb := &strings.Builder{}
|
||||
if e.Line > 0 {
|
||||
fmt.Fprintf(sb, "%v", e.Line)
|
||||
if e.Column > 0 {
|
||||
fmt.Fprintf(sb, ":%v", e.Column)
|
||||
}
|
||||
sb.WriteString(" ")
|
||||
}
|
||||
sb.WriteString(string(e.Severity))
|
||||
sb.WriteString(": ")
|
||||
sb.WriteString(e.Message)
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
// Error implements the 'error' interface.
|
||||
func (e Diagnostic) Error() string { return e.String() }
|
||||
|
|
@ -15,7 +15,6 @@
|
|||
package expectations
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"dawn.googlesource.com/dawn/tools/src/cts/result"
|
||||
|
|
@ -26,19 +25,6 @@ const (
|
|||
tagHeaderEnd = `END TAG HEADER`
|
||||
)
|
||||
|
||||
// SyntaxError is the error type returned by Parse() when a syntax error is
|
||||
// encountered.
|
||||
type SyntaxError struct {
|
||||
Line int // 1-based
|
||||
Column int // 1-based
|
||||
Message string
|
||||
}
|
||||
|
||||
// Error implements the 'error' interface.
|
||||
func (e SyntaxError) Error() string {
|
||||
return fmt.Sprintf("%v:%v: %v", e.Line, e.Column, e.Message)
|
||||
}
|
||||
|
||||
// Parse parses an expectations file, returning the Content
|
||||
func Parse(body string) (Content, error) {
|
||||
// LineType is an enumerator classifying the 'type' of the line.
|
||||
|
|
@ -130,11 +116,11 @@ func Parse(body string) (Content, error) {
|
|||
// syntaxErr is a helper for returning a SyntaxError with the current
|
||||
// line and column index.
|
||||
syntaxErr := func(at Token, msg string) error {
|
||||
column := at.start + 1
|
||||
if column == 1 {
|
||||
column = len(l) + 1
|
||||
columnIdx := at.start + 1
|
||||
if columnIdx == 1 {
|
||||
columnIdx = len(l) + 1
|
||||
}
|
||||
return SyntaxError{lineIdx, column, msg}
|
||||
return Diagnostic{Error, lineIdx, columnIdx, msg}
|
||||
}
|
||||
|
||||
// peek returns the next token without consuming it.
|
||||
|
|
|
|||
|
|
@ -432,27 +432,27 @@ crbug.com/456 [ Mac ] ghi_jkl [ PASS ]
|
|||
{
|
||||
name: "err missing tag ']'",
|
||||
in: `[`,
|
||||
expectErr: "1:2: expected ']' for tags",
|
||||
expectErr: "1:2 error: expected ']' for tags",
|
||||
}, /////////////////////////////////////////////////////////////////////
|
||||
{
|
||||
name: "err missing test query",
|
||||
in: `[ a ]`,
|
||||
expectErr: "1:6: expected test query",
|
||||
expectErr: "1:6 error: expected test query",
|
||||
}, /////////////////////////////////////////////////////////////////////
|
||||
{
|
||||
name: "err missing status EOL",
|
||||
in: `[ a ] b`,
|
||||
expectErr: "1:8: expected status",
|
||||
expectErr: "1:8 error: expected status",
|
||||
}, /////////////////////////////////////////////////////////////////////
|
||||
{
|
||||
name: "err missing status comment",
|
||||
in: `[ a ] b # c`,
|
||||
expectErr: "1:9: expected status",
|
||||
expectErr: "1:9 error: expected status",
|
||||
}, /////////////////////////////////////////////////////////////////////
|
||||
{
|
||||
name: "err missing status ']'",
|
||||
in: `[ a ] b [ c`,
|
||||
expectErr: "1:12: expected ']' for status",
|
||||
expectErr: "1:12 error: expected ']' for status",
|
||||
},
|
||||
} {
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,592 @@
|
|||
// Copyright 2022 The Dawn Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package expectations
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"dawn.googlesource.com/dawn/tools/src/container"
|
||||
"dawn.googlesource.com/dawn/tools/src/cts/query"
|
||||
"dawn.googlesource.com/dawn/tools/src/cts/result"
|
||||
)
|
||||
|
||||
// Update performs an incremental update on the expectations using the provided
|
||||
// results.
|
||||
//
|
||||
// Update will:
|
||||
// • Remove any expectation lines that have a query where no results match.
|
||||
// • Remove expectations lines that are in a chunk which is not annotated with
|
||||
// 'KEEP', and all test results have the status 'Pass'.
|
||||
// • Remove chunks that have had all expectation lines removed.
|
||||
// • Appends new chunks for flaky and failing tests which are not covered by
|
||||
// existing expectation lines.
|
||||
//
|
||||
// Update returns a list of diagnostics for things that should be addressed.
|
||||
func (c *Content) Update(results result.List) ([]Diagnostic, error) {
|
||||
// Make a copy of the results. This code mutates the list.
|
||||
results = append(result.List{}, results...)
|
||||
|
||||
// Replace statuses that the CTS runner doesn't recognize with 'Failure'
|
||||
simplifyStatuses(results)
|
||||
|
||||
// Produce a list of tag sets.
|
||||
// We reverse the declared order, as webgpu-cts/expectations.txt lists the
|
||||
// most important first (OS, GPU, etc), and result.MinimalVariantTags will
|
||||
// prioritize folding away the earlier tag-sets.
|
||||
tagSets := make([]result.Tags, len(c.Tags.Sets))
|
||||
for i, s := range c.Tags.Sets {
|
||||
tagSets[len(tagSets)-i-1] = s.Tags
|
||||
}
|
||||
|
||||
// Update those expectations!
|
||||
u := updater{
|
||||
in: *c,
|
||||
out: Content{},
|
||||
qt: newQueryTree(results),
|
||||
tagSets: tagSets,
|
||||
}
|
||||
if err := u.build(); err != nil {
|
||||
return nil, fmt.Errorf("while updating expectations: %w", err)
|
||||
}
|
||||
|
||||
*c = u.out
|
||||
return u.diags, nil
|
||||
}
|
||||
|
||||
// updater holds the state used for updating the expectations
|
||||
type updater struct {
|
||||
in Content // the original expectations Content
|
||||
out Content // newly built expectations Content
|
||||
qt queryTree // the query tree
|
||||
diags []Diagnostic // diagnostics raised during update
|
||||
tagSets []result.Tags // reverse-ordered tag-sets of 'in'
|
||||
}
|
||||
|
||||
// simplifyStatuses replaces all result statuses that are not 'Pass',
|
||||
// 'RetryOnFailure', 'Slow', 'Skip' with 'Failure'.
|
||||
func simplifyStatuses(results result.List) {
|
||||
for i, r := range results {
|
||||
switch r.Status {
|
||||
case result.Pass, result.RetryOnFailure, result.Slow, result.Skip:
|
||||
// keep
|
||||
default:
|
||||
results[i].Status = result.Failure
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
// Status used to mark results that have been already handled by an
|
||||
// expectation.
|
||||
consumed result.Status = "<<consumed>>"
|
||||
// Chunk comment for new flakes
|
||||
newFlakesComment = "# New flakes. Please triage:"
|
||||
// Chunk comment for new failures
|
||||
newFailuresComment = "# New failures. Please triage:"
|
||||
)
|
||||
|
||||
// queryTree holds tree of queries to all results (no filtering by tag or
|
||||
// status). The queryTree is used to glob all the results that match a
|
||||
// particular query.
|
||||
type queryTree struct {
|
||||
// All the results.
|
||||
results result.List
|
||||
// consumedAt is a list of line numbers for the i'th result in 'results'
|
||||
// Initially all line numbers are 0. When a result is consumed the line
|
||||
// number is set.
|
||||
consumedAt []int
|
||||
// Each tree node holds a list of indices to results.
|
||||
tree query.Tree[[]int]
|
||||
}
|
||||
|
||||
// newQueryTree builds the queryTree from the list of results.
|
||||
func newQueryTree(results result.List) queryTree {
|
||||
// Build a map of query to result indices
|
||||
queryToIndices := map[query.Query][]int{}
|
||||
for i, r := range results {
|
||||
l := queryToIndices[r.Query]
|
||||
l = append(l, i)
|
||||
queryToIndices[r.Query] = l
|
||||
}
|
||||
|
||||
// Construct the query tree to result indices
|
||||
tree := query.Tree[[]int]{}
|
||||
for query, indices := range queryToIndices {
|
||||
if err := tree.Add(query, indices); err != nil {
|
||||
// Unreachable: The only error we could get is duplicate data for
|
||||
// the same query, which should be impossible.
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
consumedAt := make([]int, len(results))
|
||||
return queryTree{results, consumedAt, tree}
|
||||
}
|
||||
|
||||
// glob returns the list of results matching the given tags under (or with) the
|
||||
// given query.
|
||||
func (qt *queryTree) glob(q query.Query) (result.List, error) {
|
||||
glob, err := qt.tree.Glob(q)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("while gathering results for query '%v': %w", q, err)
|
||||
}
|
||||
|
||||
out := result.List{}
|
||||
for _, indices := range glob {
|
||||
for _, idx := range indices.Data {
|
||||
out = append(out, qt.results[idx])
|
||||
}
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// globAndCheckForCollisions returns the list of results matching the given tags
|
||||
// under (or with) the given query.
|
||||
// globAndCheckForCollisions will return an error if any of the results are
|
||||
// already consumed by a non-zero line. The non-zero line distinguishes between
|
||||
// results consumed by expectations declared in the input (non-zero line), vs
|
||||
// those that were introduced by the update (zero line). We only want to error
|
||||
// if there's a collision in user declared expectations.
|
||||
func (qt *queryTree) globAndCheckForCollisions(q query.Query, t result.Tags) (result.List, error) {
|
||||
glob, err := qt.tree.Glob(q)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out := result.List{}
|
||||
for _, indices := range glob {
|
||||
for _, idx := range indices.Data {
|
||||
if r := qt.results[idx]; r.Tags.ContainsAll(t) {
|
||||
if at := qt.consumedAt[idx]; at > 0 {
|
||||
if len(t) > 0 {
|
||||
return nil, fmt.Errorf("%v %v collides with expectation at line %v", t, q, at)
|
||||
}
|
||||
return nil, fmt.Errorf("%v collides with expectation at line %v", q, at)
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// markAsConsumed marks all the results matching the given tags
|
||||
// under (or with) the given query, as consumed.
|
||||
// line is used to record the line at which the results were consumed. If the
|
||||
// results were consumed as part of generating new expectations then line should
|
||||
// be 0. See queryTree.globAndCheckForCollisions().
|
||||
func (qt *queryTree) markAsConsumed(q query.Query, t result.Tags, line int) {
|
||||
if glob, err := qt.tree.Glob(q); err == nil {
|
||||
for _, indices := range glob {
|
||||
for _, idx := range indices.Data {
|
||||
r := &qt.results[idx]
|
||||
if r.Tags.ContainsAll(t) {
|
||||
r.Status = consumed
|
||||
qt.consumedAt[idx] = line
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// build is the updater top-level function.
|
||||
// build first appends to u.out all chunks from 'u.in' with expectations updated
|
||||
// using the new results, and then appends any new expectations to u.out.
|
||||
func (u *updater) build() error {
|
||||
// Update all the existing chunks
|
||||
for _, in := range u.in.Chunks {
|
||||
out := u.chunk(in)
|
||||
|
||||
// If all chunk had expectations, but now they've gone, remove the chunk
|
||||
if len(in.Expectations) > 0 && len(out.Expectations) == 0 {
|
||||
continue
|
||||
}
|
||||
if out.IsBlankLine() {
|
||||
u.out.MaybeAddBlankLine()
|
||||
continue
|
||||
}
|
||||
u.out.Chunks = append(u.out.Chunks, out)
|
||||
}
|
||||
|
||||
// Emit new expectations (flaky, failing)
|
||||
if err := u.addNewExpectations(); err != nil {
|
||||
return fmt.Errorf("failed to add new expectations: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// chunk returns a new Chunk, based on 'in', with the expectations updated.
|
||||
func (u *updater) chunk(in Chunk) Chunk {
|
||||
if len(in.Expectations) == 0 {
|
||||
return in // Just a comment / blank line
|
||||
}
|
||||
|
||||
// Skip over any untriaged failures / flake chunks.
|
||||
// We'll just rebuild them at the end.
|
||||
if len(in.Comments) > 0 {
|
||||
switch in.Comments[0] {
|
||||
case newFailuresComment, newFlakesComment:
|
||||
return Chunk{}
|
||||
}
|
||||
}
|
||||
|
||||
keep := false // Does the chunk comment contain 'KEEP' ?
|
||||
for _, l := range in.Comments {
|
||||
if strings.Contains(l, "KEEP") {
|
||||
keep = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Begin building the output chunk.
|
||||
// Copy over the chunk's comments.
|
||||
out := Chunk{Comments: in.Comments}
|
||||
|
||||
// Build the new chunk's expectations
|
||||
for _, exIn := range in.Expectations {
|
||||
exOut := u.expectation(exIn, keep)
|
||||
out.Expectations = append(out.Expectations, exOut...)
|
||||
}
|
||||
|
||||
// Sort the expectations to keep things clean and tidy.
|
||||
sort.Slice(out.Expectations, func(i, j int) bool {
|
||||
switch {
|
||||
case out.Expectations[i].Query < out.Expectations[j].Query:
|
||||
return true
|
||||
case out.Expectations[i].Query > out.Expectations[j].Query:
|
||||
return false
|
||||
}
|
||||
a := result.TagsToString(out.Expectations[i].Tags)
|
||||
b := result.TagsToString(out.Expectations[j].Tags)
|
||||
switch {
|
||||
case a < b:
|
||||
return true
|
||||
case a > b:
|
||||
return false
|
||||
}
|
||||
return false
|
||||
})
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// chunk returns a new list of Expectations, based on the Expectation 'in',
|
||||
// using the new result data.
|
||||
func (u *updater) expectation(in Expectation, keep bool) []Expectation {
|
||||
// noResults is a helper for returning when the expectation has no test
|
||||
// results.
|
||||
// If the expectation has an expected 'Skip' result, then we're likely
|
||||
// to be missing results (as the test was not run). In this situation
|
||||
// the expectation is preserved, and no diagnostics are raised.
|
||||
// If the expectation did not have a 'Skip' result, then a diagnostic will
|
||||
// be raised and the expectation will be removed.
|
||||
noResults := func() []Expectation {
|
||||
if container.NewSet(in.Status...).Contains(string(result.Skip)) {
|
||||
return []Expectation{in}
|
||||
}
|
||||
// Expectation does not have a 'Skip' result.
|
||||
if len(in.Tags) > 0 {
|
||||
u.diag(Warning, in.Line, "no results found for '%v' with tags %v", in.Query, in.Tags)
|
||||
} else {
|
||||
u.diag(Warning, in.Line, "no results found for '%v'", in.Query)
|
||||
}
|
||||
// Remove the no-results expectation
|
||||
return []Expectation{}
|
||||
}
|
||||
|
||||
// Grab all the results that match the expectation's query
|
||||
q := query.Parse(in.Query)
|
||||
|
||||
// Glob the results for the expectation's query + tag combination.
|
||||
// Ensure that none of these are already consumed.
|
||||
results, err := u.qt.globAndCheckForCollisions(q, in.Tags)
|
||||
// If we can't find any results for this query + tag combination, then bail.
|
||||
switch {
|
||||
case errors.As(err, &query.ErrNoDataForQuery{}):
|
||||
return noResults()
|
||||
case err != nil:
|
||||
u.diag(Error, in.Line, "%v", err)
|
||||
return []Expectation{}
|
||||
case len(results) == 0:
|
||||
return noResults()
|
||||
}
|
||||
|
||||
// Before returning, mark all the results as consumed.
|
||||
// Note: this has to happen *after* we've generated the new expectations, as
|
||||
// marking the results as 'consumed' will impact the logic of
|
||||
// expectationsForRoot()
|
||||
defer u.qt.markAsConsumed(q, in.Tags, in.Line)
|
||||
|
||||
if keep { // Expectation chunk was marked with 'KEEP'
|
||||
// Add a diagnostic if all tests of the expectation were 'Pass'
|
||||
if s := results.Statuses(); len(s) == 1 && s.One() == result.Pass {
|
||||
if c := len(results); c > 1 {
|
||||
u.diag(Note, in.Line, "all %d tests now pass", len(results))
|
||||
} else {
|
||||
u.diag(Note, in.Line, "test now passes")
|
||||
}
|
||||
}
|
||||
return []Expectation{in}
|
||||
}
|
||||
|
||||
// Rebuild the expectations for this query.
|
||||
return u.expectationsForRoot(q, in.Line, in.Bug, in.Comment)
|
||||
}
|
||||
|
||||
// addNewExpectations (potentially) appends to 'u.out' chunks for new flaky and
|
||||
// failing tests.
|
||||
func (u *updater) addNewExpectations() error {
|
||||
// Scan the full result list to obtain all the test variants
|
||||
// (unique tag combinations).
|
||||
allVariants := u.qt.results.Variants()
|
||||
|
||||
// For each variant:
|
||||
// • Build a query tree using the results filtered to the variant, and then
|
||||
// reduce the tree.
|
||||
// • Take all the reduced-tree leaf nodes, and add these to 'roots'.
|
||||
// Once we've collected all the roots, we'll use these to build the
|
||||
// expectations across the reduced set of tags.
|
||||
roots := container.NewMap[string, query.Query]()
|
||||
for _, variant := range allVariants {
|
||||
// Build a tree from the results matching the given variant.
|
||||
tree, err := u.qt.results.FilterByVariant(variant).StatusTree()
|
||||
if err != nil {
|
||||
return fmt.Errorf("while building tree for tags '%v': %w", variant, err)
|
||||
}
|
||||
// Reduce the tree.
|
||||
tree.Reduce(treeReducer)
|
||||
// Add all the reduced leaf nodes to 'roots'.
|
||||
for _, qd := range tree.List() {
|
||||
roots.Add(qd.Query.String(), qd.Query)
|
||||
}
|
||||
}
|
||||
|
||||
// Build all the expectations for each of the roots.
|
||||
expectations := []Expectation{}
|
||||
for _, root := range roots.Values() {
|
||||
expectations = append(expectations, u.expectationsForRoot(
|
||||
root, // Root query
|
||||
0, // Line number
|
||||
"crbug.com/dawn/0000", // Bug
|
||||
"", // Comment
|
||||
)...)
|
||||
}
|
||||
|
||||
// Bin the expectations by failure or flake.
|
||||
flakes, failures := []Expectation{}, []Expectation{}
|
||||
for _, r := range expectations {
|
||||
if container.NewSet(r.Status...).Contains(string(result.RetryOnFailure)) {
|
||||
flakes = append(flakes, r)
|
||||
} else {
|
||||
failures = append(failures, r)
|
||||
}
|
||||
}
|
||||
|
||||
// Create chunks for any flakes and failures, in that order.
|
||||
for _, group := range []struct {
|
||||
results []Expectation
|
||||
comment string
|
||||
}{
|
||||
{flakes, newFlakesComment},
|
||||
{failures, newFailuresComment},
|
||||
} {
|
||||
if len(group.results) > 0 {
|
||||
u.out.MaybeAddBlankLine()
|
||||
u.out.Chunks = append(u.out.Chunks, Chunk{
|
||||
Comments: []string{group.comment},
|
||||
Expectations: group.results,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// expectationsForRoot builds a list of expectations that cover the failing
|
||||
// tests for the results under root.
|
||||
// The returned list of expectations is optimized by reducing queries to the
|
||||
// most common root, and reducing tags to the smallest required set.
|
||||
func (u *updater) expectationsForRoot(
|
||||
root query.Query, // The sub-tree query root
|
||||
line int, // The originating line, when producing diagnostics
|
||||
bug string, // The bug to apply to all returned expectations
|
||||
comment string, // The comment to apply to all returned expectations
|
||||
) []Expectation {
|
||||
results, err := u.qt.glob(root)
|
||||
if err != nil {
|
||||
u.diag(Error, line, "%v", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Using the full list of unfiltered tests, generate the minimal set of
|
||||
// variants (tags) that uniquely classify the results with differing status.
|
||||
minimalVariants := u.
|
||||
cleanupTags(results).
|
||||
MinimalVariantTags(u.tagSets)
|
||||
|
||||
// For each minimized variant...
|
||||
reduced := result.List{}
|
||||
for _, variant := range minimalVariants {
|
||||
// Build a query tree from this variant...
|
||||
tree := result.StatusTree{}
|
||||
filtered := results.FilterByTags(variant)
|
||||
for _, r := range filtered {
|
||||
// Note: variants may overlap, but overlaped queries will have
|
||||
// identical statuses, so we can just ignore the error for Add().
|
||||
tree.Add(r.Query, r.Status)
|
||||
}
|
||||
|
||||
// ... and reduce the tree by collapsing sub-trees that have common
|
||||
// statuses.
|
||||
tree.ReduceUnder(root, treeReducer)
|
||||
|
||||
// Append the reduced tree nodes to the results list
|
||||
for _, qs := range tree.List() {
|
||||
reduced = append(reduced, result.Result{
|
||||
Query: qs.Query,
|
||||
Tags: variant,
|
||||
Status: qs.Data,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Filter out any results that passed or have already been consumed
|
||||
filtered := reduced.Filter(func(r result.Result) bool {
|
||||
return r.Status != result.Pass && r.Status != consumed
|
||||
})
|
||||
|
||||
// Mark all the new expectation results as consumed.
|
||||
for _, r := range filtered {
|
||||
u.qt.markAsConsumed(r.Query, r.Tags, 0)
|
||||
}
|
||||
|
||||
// Transform the results to expectations.
|
||||
return u.resultsToExpectations(filtered, bug, comment)
|
||||
}
|
||||
|
||||
// resultsToExpectations returns a list of expectations from the given results.
|
||||
// Each expectation will have the same query, tags and status as the input
|
||||
// result, along with the specified bug and comment.
|
||||
//
|
||||
// If the result query target is a test without a wildcard, then the expectation
|
||||
// will have a wildcard automatically appended. This is to satisfy a requirement
|
||||
// of the expectation validator.
|
||||
func (u *updater) resultsToExpectations(results result.List, bug, comment string) []Expectation {
|
||||
results.Sort()
|
||||
|
||||
out := make([]Expectation, len(results))
|
||||
for i, r := range results {
|
||||
q := r.Query.String()
|
||||
if r.Query.Target() == query.Tests && !r.Query.IsWildcard() {
|
||||
// The expectation validator wants a trailing ':' for test queries
|
||||
q += query.TargetDelimiter
|
||||
}
|
||||
out[i] = Expectation{
|
||||
Bug: bug,
|
||||
Tags: r.Tags,
|
||||
Query: q,
|
||||
Status: []string{string(r.Status)},
|
||||
Comment: comment,
|
||||
}
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// cleanupTags returns a copy of the provided results with:
|
||||
// • All tags not found in the expectations list removed
|
||||
// • All but the highest priority tag for any tag-set.
|
||||
// The tag sets are defined by the `BEGIN TAG HEADER` / `END TAG HEADER`
|
||||
// section at the top of the expectations file.
|
||||
func (u *updater) cleanupTags(results result.List) result.List {
|
||||
return results.TransformTags(func(t result.Tags) result.Tags {
|
||||
type HighestPrioritySetTag struct {
|
||||
tag string
|
||||
priority int
|
||||
}
|
||||
// Set name to highest priority tag for that set
|
||||
best := map[string]HighestPrioritySetTag{}
|
||||
for tag := range t {
|
||||
sp, ok := u.in.Tags.ByName[tag]
|
||||
if ok {
|
||||
if set := best[sp.Set]; sp.Priority >= set.priority {
|
||||
best[sp.Set] = HighestPrioritySetTag{tag, sp.Priority}
|
||||
}
|
||||
}
|
||||
}
|
||||
t = result.NewTags()
|
||||
for _, ts := range best {
|
||||
t.Add(ts.tag)
|
||||
}
|
||||
return t
|
||||
})
|
||||
}
|
||||
|
||||
// treeReducer is a function that can be used by StatusTree.Reduce() to reduce
|
||||
// tree nodes with the same status.
|
||||
// treeReducer will collapse trees nodes if any of the following are true:
|
||||
// • All child nodes have the same status
|
||||
// • More than 75% of the child nodes have a non-pass status, and none of the
|
||||
// children are consumed.
|
||||
// • There are more than 20 child nodes with a non-pass status, and none of the
|
||||
// children are consumed.
|
||||
func treeReducer(statuses []result.Status) *result.Status {
|
||||
counts := map[result.Status]int{}
|
||||
for _, s := range statuses {
|
||||
counts[s] = counts[s] + 1
|
||||
}
|
||||
if len(counts) == 1 {
|
||||
return &statuses[0] // All the same status
|
||||
}
|
||||
if counts[consumed] > 0 {
|
||||
return nil // Partially consumed trees cannot be merged
|
||||
}
|
||||
highestNonPassCount := 0
|
||||
highestNonPassStatus := result.Failure
|
||||
for s, n := range counts {
|
||||
if s != result.Pass {
|
||||
if percent := (100 * n) / len(statuses); percent > 75 {
|
||||
// Over 75% of all the children are of non-pass status s.
|
||||
return &s
|
||||
}
|
||||
if n > highestNonPassCount {
|
||||
highestNonPassCount = n
|
||||
highestNonPassStatus = s
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if highestNonPassCount > 20 {
|
||||
// Over 20 child node failed.
|
||||
return &highestNonPassStatus
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// diag appends a new diagnostic to u.diags with the given severity, line and
|
||||
// message.
|
||||
func (u *updater) diag(severity Severity, line int, msg string, args ...interface{}) {
|
||||
u.diags = append(u.diags, Diagnostic{
|
||||
Severity: severity,
|
||||
Line: line,
|
||||
Message: fmt.Sprintf(msg, args...),
|
||||
})
|
||||
}
|
||||
|
|
@ -0,0 +1,672 @@
|
|||
// Copyright 2022 The Dawn Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package expectations_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"dawn.googlesource.com/dawn/tools/src/cts/expectations"
|
||||
"dawn.googlesource.com/dawn/tools/src/cts/query"
|
||||
"dawn.googlesource.com/dawn/tools/src/cts/result"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
)
|
||||
|
||||
var Q = query.Parse
|
||||
|
||||
func TestUpdate(t *testing.T) {
|
||||
header := `# BEGIN TAG HEADER
|
||||
# OS
|
||||
# tags: [ os-a os-b os-c ]
|
||||
# GPU
|
||||
# tags: [ gpu-a gpu-b gpu-c ]
|
||||
# END TAG HEADER
|
||||
`
|
||||
headerLines := strings.Count(header, "\n")
|
||||
|
||||
type Test struct {
|
||||
name string
|
||||
expectations string
|
||||
results result.List
|
||||
updated string
|
||||
diagnostics []expectations.Diagnostic
|
||||
err string
|
||||
}
|
||||
for _, test := range []Test{
|
||||
{ //////////////////////////////////////////////////////////////////////
|
||||
name: "empty results",
|
||||
expectations: ``,
|
||||
results: result.List{},
|
||||
},
|
||||
{ //////////////////////////////////////////////////////////////////////
|
||||
name: "no results found",
|
||||
expectations: `
|
||||
crbug.com/a/123 a:missing,test,result:* [ Failure ]
|
||||
crbug.com/a/123 [ tag ] another:missing,test,result:* [ Failure ]
|
||||
|
||||
some:other,test:* [ Failure ]
|
||||
`,
|
||||
results: result.List{
|
||||
result.Result{
|
||||
Query: Q("some:other,test:*"),
|
||||
Tags: result.NewTags("os-a", "gpu-a"),
|
||||
Status: result.Failure,
|
||||
},
|
||||
result.Result{
|
||||
Query: Q("some:other,test:*"),
|
||||
Tags: result.NewTags("os-b", "gpu-b"),
|
||||
Status: result.Failure,
|
||||
},
|
||||
},
|
||||
updated: `
|
||||
some:other,test:* [ Failure ]
|
||||
`,
|
||||
diagnostics: []expectations.Diagnostic{
|
||||
{
|
||||
Severity: expectations.Warning,
|
||||
Line: headerLines + 2,
|
||||
Message: "no results found for 'a:missing,test,result:*'",
|
||||
},
|
||||
{
|
||||
Severity: expectations.Warning,
|
||||
Line: headerLines + 3,
|
||||
Message: "no results found for 'another:missing,test,result:*' with tags [tag]",
|
||||
},
|
||||
},
|
||||
},
|
||||
{ //////////////////////////////////////////////////////////////////////
|
||||
name: "no results found KEEP",
|
||||
expectations: `
|
||||
# KEEP
|
||||
crbug.com/a/123 a:missing,test,result:* [ Failure ]
|
||||
|
||||
some:other,test:* [ Failure ]
|
||||
`,
|
||||
results: result.List{
|
||||
result.Result{
|
||||
Query: Q("some:other,test:*"),
|
||||
Tags: result.NewTags("os-a", "gpu-a"),
|
||||
Status: result.Failure,
|
||||
},
|
||||
result.Result{
|
||||
Query: Q("some:other,test:*"),
|
||||
Tags: result.NewTags("os-b", "gpu-b"),
|
||||
Status: result.Failure,
|
||||
},
|
||||
},
|
||||
updated: `
|
||||
some:other,test:* [ Failure ]
|
||||
`,
|
||||
diagnostics: []expectations.Diagnostic{
|
||||
{
|
||||
Severity: expectations.Warning,
|
||||
Line: headerLines + 3,
|
||||
Message: "no results found for 'a:missing,test,result:*'",
|
||||
},
|
||||
},
|
||||
},
|
||||
{ //////////////////////////////////////////////////////////////////////
|
||||
name: "no results found Skip",
|
||||
expectations: `
|
||||
crbug.com/a/123 a:missing,test,result:* [ Skip ]
|
||||
|
||||
some:other,test:* [ Failure ]
|
||||
`,
|
||||
results: result.List{
|
||||
result.Result{
|
||||
Query: Q("some:other,test:*"),
|
||||
Tags: result.NewTags("os-a", "gpu-a"),
|
||||
Status: result.Failure,
|
||||
},
|
||||
result.Result{
|
||||
Query: Q("some:other,test:*"),
|
||||
Tags: result.NewTags("os-b", "gpu-b"),
|
||||
Status: result.Failure,
|
||||
},
|
||||
},
|
||||
updated: `
|
||||
crbug.com/a/123 a:missing,test,result:* [ Skip ]
|
||||
|
||||
some:other,test:* [ Failure ]
|
||||
`,
|
||||
},
|
||||
{ //////////////////////////////////////////////////////////////////////
|
||||
name: "simple expectation collision",
|
||||
expectations: `
|
||||
a:b,c:* [ Failure ]
|
||||
a:b,c:* [ Failure ]
|
||||
`,
|
||||
results: result.List{
|
||||
result.Result{
|
||||
Query: Q("a:b,c:d"),
|
||||
Tags: result.NewTags("os-a", "os-c", "gpu-b"),
|
||||
Status: result.Failure,
|
||||
},
|
||||
},
|
||||
updated: `
|
||||
a:b,c:* [ Failure ]
|
||||
`,
|
||||
diagnostics: []expectations.Diagnostic{
|
||||
{
|
||||
Severity: expectations.Error,
|
||||
Line: headerLines + 3,
|
||||
Message: "a:b,c:* collides with expectation at line 8",
|
||||
},
|
||||
},
|
||||
},
|
||||
{ //////////////////////////////////////////////////////////////////////
|
||||
name: "simple expectation with tags",
|
||||
expectations: `
|
||||
[ os-a ] a:b,c:* [ Failure ]
|
||||
[ gpu-b ] a:b,c:* [ Failure ]
|
||||
`,
|
||||
results: result.List{
|
||||
result.Result{
|
||||
Query: Q("a:b,c:d"),
|
||||
Tags: result.NewTags("os-a", "os-c", "gpu-b"),
|
||||
Status: result.Failure,
|
||||
},
|
||||
},
|
||||
updated: `
|
||||
a:b,c:* [ Failure ]
|
||||
`,
|
||||
diagnostics: []expectations.Diagnostic{
|
||||
{
|
||||
Severity: expectations.Error,
|
||||
Line: headerLines + 3,
|
||||
Message: "[gpu-b] a:b,c:* collides with expectation at line 8",
|
||||
},
|
||||
},
|
||||
},
|
||||
{ //////////////////////////////////////////////////////////////////////
|
||||
name: "simple expectation collision KEEP",
|
||||
expectations: `
|
||||
# KEEP
|
||||
a:b,c:* [ Failure ]
|
||||
a:b,c:* [ Failure ]
|
||||
`,
|
||||
results: result.List{
|
||||
result.Result{
|
||||
Query: Q("a:b,c:d"),
|
||||
Tags: result.NewTags("os-a", "os-c", "gpu-b"),
|
||||
Status: result.Failure,
|
||||
},
|
||||
},
|
||||
updated: `
|
||||
# KEEP
|
||||
a:b,c:* [ Failure ]
|
||||
`,
|
||||
diagnostics: []expectations.Diagnostic{
|
||||
{
|
||||
Severity: expectations.Error,
|
||||
Line: headerLines + 4,
|
||||
Message: "a:b,c:* collides with expectation at line 9",
|
||||
},
|
||||
},
|
||||
},
|
||||
{ //////////////////////////////////////////////////////////////////////
|
||||
name: "collision with child-expectation",
|
||||
expectations: `
|
||||
a:b:x:* [ Failure ]
|
||||
a:b:* [ Failure ]
|
||||
`,
|
||||
results: result.List{
|
||||
result.Result{
|
||||
Query: Q("a:b:x:*"),
|
||||
Tags: result.NewTags("os-a", "os-c", "gpu-b"),
|
||||
Status: result.Failure,
|
||||
},
|
||||
result.Result{
|
||||
Query: Q("a:b:y:*"),
|
||||
Tags: result.NewTags("os-a", "os-c", "gpu-b"),
|
||||
Status: result.Failure,
|
||||
},
|
||||
},
|
||||
updated: `
|
||||
a:b:x:* [ Failure ]
|
||||
|
||||
# New failures. Please triage:
|
||||
crbug.com/dawn/0000 a:b:y:* [ Failure ]
|
||||
`,
|
||||
diagnostics: []expectations.Diagnostic{
|
||||
{
|
||||
Severity: expectations.Error,
|
||||
Line: headerLines + 3,
|
||||
Message: "a:b:* collides with expectation at line 8",
|
||||
},
|
||||
},
|
||||
},
|
||||
{ //////////////////////////////////////////////////////////////////////
|
||||
name: "collision with parent-expectation",
|
||||
expectations: `
|
||||
a:b:* [ Failure ]
|
||||
a:b:x:* [ Failure ]
|
||||
`,
|
||||
results: result.List{
|
||||
result.Result{
|
||||
Query: Q("a:b:x:*"),
|
||||
Tags: result.NewTags("os-a", "os-c", "gpu-b"),
|
||||
Status: result.Failure,
|
||||
},
|
||||
result.Result{
|
||||
Query: Q("a:b:y:*"),
|
||||
Tags: result.NewTags("os-a", "os-c", "gpu-b"),
|
||||
Status: result.Failure,
|
||||
},
|
||||
},
|
||||
updated: `
|
||||
a:b:* [ Failure ]
|
||||
`,
|
||||
diagnostics: []expectations.Diagnostic{
|
||||
{
|
||||
Severity: expectations.Error,
|
||||
Line: headerLines + 3,
|
||||
Message: "a:b:x:* collides with expectation at line 8",
|
||||
},
|
||||
},
|
||||
},
|
||||
{ //////////////////////////////////////////////////////////////////////
|
||||
name: "expectation test now passes",
|
||||
expectations: `
|
||||
crbug.com/a/123 [ gpu-a os-a ] a:b,c:* [ Failure ]
|
||||
crbug.com/a/123 [ gpu-b os-b ] a:b,c:* [ Failure ]
|
||||
`,
|
||||
results: result.List{
|
||||
result.Result{
|
||||
Query: Q("a:b,c:*"),
|
||||
Tags: result.NewTags("os-a", "gpu-a"),
|
||||
Status: result.Pass,
|
||||
},
|
||||
result.Result{
|
||||
Query: Q("a:b,c:*"),
|
||||
Tags: result.NewTags("os-b", "gpu-b"),
|
||||
Status: result.Abort,
|
||||
},
|
||||
},
|
||||
updated: `
|
||||
crbug.com/a/123 [ os-b ] a:b,c:* [ Failure ]
|
||||
`,
|
||||
},
|
||||
{ //////////////////////////////////////////////////////////////////////
|
||||
name: "expectation case now passes",
|
||||
expectations: `
|
||||
crbug.com/a/123 [ gpu-a os-a ] a:b,c:d [ Failure ]
|
||||
crbug.com/a/123 [ gpu-b os-b ] a:b,c:d [ Failure ]
|
||||
`,
|
||||
results: result.List{
|
||||
result.Result{
|
||||
Query: Q("a:b,c:d"),
|
||||
Tags: result.NewTags("os-a", "gpu-a"),
|
||||
Status: result.Pass,
|
||||
},
|
||||
result.Result{
|
||||
Query: Q("a:b,c:d"),
|
||||
Tags: result.NewTags("os-b", "gpu-b"),
|
||||
Status: result.Abort,
|
||||
},
|
||||
},
|
||||
updated: `
|
||||
crbug.com/a/123 [ os-b ] a:b,c:d: [ Failure ]
|
||||
`,
|
||||
},
|
||||
{ //////////////////////////////////////////////////////////////////////
|
||||
name: "expectation case now passes KEEP - single",
|
||||
expectations: `
|
||||
# KEEP
|
||||
crbug.com/a/123 [ gpu-a os-a ] a:b,c:d [ Failure ]
|
||||
crbug.com/a/123 [ gpu-b os-b ] a:b,c:d [ Failure ]
|
||||
`,
|
||||
results: result.List{
|
||||
result.Result{
|
||||
Query: Q("a:b,c:d"),
|
||||
Tags: result.NewTags("os-a", "gpu-a"),
|
||||
Status: result.Pass,
|
||||
},
|
||||
result.Result{
|
||||
Query: Q("a:b,c:d"),
|
||||
Tags: result.NewTags("os-b", "gpu-b"),
|
||||
Status: result.Abort,
|
||||
},
|
||||
},
|
||||
updated: `
|
||||
# KEEP
|
||||
crbug.com/a/123 [ gpu-a os-a ] a:b,c:d [ Failure ]
|
||||
crbug.com/a/123 [ gpu-b os-b ] a:b,c:d [ Failure ]
|
||||
`,
|
||||
diagnostics: []expectations.Diagnostic{
|
||||
{
|
||||
Severity: expectations.Note,
|
||||
Line: headerLines + 3,
|
||||
Message: "test now passes",
|
||||
},
|
||||
},
|
||||
},
|
||||
{ //////////////////////////////////////////////////////////////////////
|
||||
name: "expectation case now passes KEEP - multiple",
|
||||
expectations: `
|
||||
# KEEP
|
||||
crbug.com/a/123 a:b,c:d:* [ Failure ]
|
||||
`,
|
||||
results: result.List{
|
||||
result.Result{Query: Q("a:b,c:d:a"), Status: result.Pass},
|
||||
result.Result{Query: Q("a:b,c:d:b"), Status: result.Pass},
|
||||
result.Result{Query: Q("a:b,c:d:c"), Status: result.Pass},
|
||||
result.Result{Query: Q("a:b,c:d:d"), Status: result.Pass},
|
||||
},
|
||||
updated: `
|
||||
# KEEP
|
||||
crbug.com/a/123 a:b,c:d:* [ Failure ]
|
||||
`,
|
||||
diagnostics: []expectations.Diagnostic{
|
||||
{
|
||||
Severity: expectations.Note,
|
||||
Line: headerLines + 3,
|
||||
Message: "all 4 tests now pass",
|
||||
},
|
||||
},
|
||||
},
|
||||
{ //////////////////////////////////////////////////////////////////////
|
||||
name: "new test results",
|
||||
expectations: `# A comment`,
|
||||
results: result.List{
|
||||
result.Result{
|
||||
Query: Q("suite:dir_a,dir_b:test_a:*"),
|
||||
Tags: result.NewTags("os-a", "gpu-a"),
|
||||
Status: result.Abort,
|
||||
},
|
||||
result.Result{
|
||||
Query: Q("suite:dir_a,dir_b:test_a:*"),
|
||||
Tags: result.NewTags("os-a", "gpu-b"),
|
||||
Status: result.Abort,
|
||||
},
|
||||
result.Result{
|
||||
Query: Q("suite:dir_a,dir_b:test_c:case=4;*"),
|
||||
Tags: result.NewTags("os-b", "gpu-b"),
|
||||
Status: result.Crash,
|
||||
},
|
||||
result.Result{
|
||||
Query: Q("suite:dir_a,dir_b:test_c:case=5;*"),
|
||||
Tags: result.NewTags("os-b", "gpu-b"),
|
||||
Status: result.RetryOnFailure,
|
||||
},
|
||||
result.Result{
|
||||
Query: Q("suite:dir_a,dir_b:test_b;case=5;*"),
|
||||
Tags: result.NewTags("os-b", "gpu-b"),
|
||||
Status: result.Pass,
|
||||
},
|
||||
result.Result{
|
||||
Query: Q("suite:dir_a,dir_b:test_b:*"),
|
||||
Tags: result.NewTags("os-a", "gpu-a"),
|
||||
Status: result.Skip,
|
||||
},
|
||||
result.Result{
|
||||
Query: Q("suite:dir_a,dir_b:test_b:*"),
|
||||
Tags: result.NewTags("os-b", "gpu-a"),
|
||||
Status: result.Pass,
|
||||
},
|
||||
result.Result{
|
||||
Query: Q("suite:dir_a,dir_b:test_c:*"),
|
||||
Tags: result.NewTags("os-a", "gpu-a"),
|
||||
Status: result.RetryOnFailure,
|
||||
},
|
||||
},
|
||||
updated: `# A comment
|
||||
|
||||
# New flakes. Please triage:
|
||||
crbug.com/dawn/0000 [ gpu-a os-a ] suite:dir_a,dir_b:test_c:* [ RetryOnFailure ]
|
||||
crbug.com/dawn/0000 [ gpu-b os-b ] suite:dir_a,dir_b:test_c:case=5;* [ RetryOnFailure ]
|
||||
|
||||
# New failures. Please triage:
|
||||
crbug.com/dawn/0000 [ gpu-b os-a ] suite:* [ Failure ]
|
||||
crbug.com/dawn/0000 [ gpu-a os-a ] suite:dir_a,dir_b:test_a:* [ Failure ]
|
||||
crbug.com/dawn/0000 [ gpu-a os-a ] suite:dir_a,dir_b:test_b:* [ Skip ]
|
||||
crbug.com/dawn/0000 [ gpu-b os-b ] suite:dir_a,dir_b:test_c:case=4;* [ Failure ]
|
||||
`,
|
||||
},
|
||||
{ //////////////////////////////////////////////////////////////////////
|
||||
name: "filter unknown tags",
|
||||
expectations: ``,
|
||||
results: result.List{
|
||||
result.Result{
|
||||
Query: Q("a:b,c:*"),
|
||||
Tags: result.NewTags("os-a", "gpu-x"),
|
||||
Status: result.Failure,
|
||||
},
|
||||
result.Result{
|
||||
Query: Q("a:b,c:*"),
|
||||
Tags: result.NewTags("os-b", "gpu-x"),
|
||||
Status: result.Crash,
|
||||
},
|
||||
result.Result{
|
||||
Query: Q("a:b,c:*"),
|
||||
Tags: result.NewTags("os-x", "gpu-b"),
|
||||
Status: result.Failure,
|
||||
},
|
||||
result.Result{
|
||||
Query: Q("a:b,c:*"),
|
||||
Tags: result.NewTags("os-x", "gpu-a"),
|
||||
Status: result.Crash,
|
||||
},
|
||||
result.Result{
|
||||
Query: Q("a:b,c:*"),
|
||||
Tags: result.NewTags("os-c", "gpu-c"),
|
||||
Status: result.Pass,
|
||||
},
|
||||
},
|
||||
updated: `
|
||||
# New failures. Please triage:
|
||||
crbug.com/dawn/0000 [ gpu-a ] a:* [ Failure ]
|
||||
crbug.com/dawn/0000 [ gpu-b ] a:* [ Failure ]
|
||||
crbug.com/dawn/0000 [ os-a ] a:* [ Failure ]
|
||||
crbug.com/dawn/0000 [ os-b ] a:* [ Failure ]
|
||||
`,
|
||||
},
|
||||
{ //////////////////////////////////////////////////////////////////////
|
||||
name: "prioritize tag sets",
|
||||
expectations: ``,
|
||||
results: result.List{
|
||||
result.Result{
|
||||
Query: Q("a:b,c:*"),
|
||||
Tags: result.NewTags("os-a", "os-c", "gpu-b"),
|
||||
Status: result.Failure,
|
||||
},
|
||||
result.Result{
|
||||
Query: Q("a:b,c:*"),
|
||||
Tags: result.NewTags("gpu-a", "os-b", "gpu-c"),
|
||||
Status: result.Failure,
|
||||
},
|
||||
result.Result{
|
||||
Query: Q("a:b,c:*"),
|
||||
Tags: result.NewTags("os-c", "gpu-c"),
|
||||
Status: result.Pass,
|
||||
},
|
||||
},
|
||||
updated: `
|
||||
# New failures. Please triage:
|
||||
crbug.com/dawn/0000 [ gpu-b os-c ] a:* [ Failure ]
|
||||
crbug.com/dawn/0000 [ gpu-c os-b ] a:* [ Failure ]
|
||||
`,
|
||||
},
|
||||
{ //////////////////////////////////////////////////////////////////////
|
||||
name: "merge when over 75% of children fail",
|
||||
expectations: ``,
|
||||
results: result.List{
|
||||
result.Result{Query: Q("a:b,c:t0:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t1:*"), Status: result.Pass},
|
||||
result.Result{Query: Q("a:b,c:t2:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t3:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t4:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t5:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t6:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t7:*"), Status: result.Pass},
|
||||
result.Result{Query: Q("a:b,c:t8:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t9:*"), Status: result.Failure},
|
||||
},
|
||||
updated: `
|
||||
# New failures. Please triage:
|
||||
crbug.com/dawn/0000 a:* [ Failure ]
|
||||
`,
|
||||
},
|
||||
{ //////////////////////////////////////////////////////////////////////
|
||||
name: "don't merge when under 75% of children fail",
|
||||
expectations: ``,
|
||||
results: result.List{
|
||||
result.Result{Query: Q("a:b,c:t0:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t1:*"), Status: result.Pass},
|
||||
result.Result{Query: Q("a:b,c:t2:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t3:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t4:*"), Status: result.Pass},
|
||||
result.Result{Query: Q("a:b,c:t5:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t6:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t7:*"), Status: result.Pass},
|
||||
result.Result{Query: Q("a:b,c:t8:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t9:*"), Status: result.Failure},
|
||||
},
|
||||
updated: `
|
||||
# New failures. Please triage:
|
||||
crbug.com/dawn/0000 a:b,c:t0:* [ Failure ]
|
||||
crbug.com/dawn/0000 a:b,c:t2:* [ Failure ]
|
||||
crbug.com/dawn/0000 a:b,c:t3:* [ Failure ]
|
||||
crbug.com/dawn/0000 a:b,c:t5:* [ Failure ]
|
||||
crbug.com/dawn/0000 a:b,c:t6:* [ Failure ]
|
||||
crbug.com/dawn/0000 a:b,c:t8:* [ Failure ]
|
||||
crbug.com/dawn/0000 a:b,c:t9:* [ Failure ]
|
||||
`,
|
||||
},
|
||||
{ //////////////////////////////////////////////////////////////////////
|
||||
name: "merge when over 20 children fail",
|
||||
expectations: ``,
|
||||
results: result.List{ // 21 failures, 70% fail
|
||||
result.Result{Query: Q("a:b,c:t00:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t01:*"), Status: result.Pass},
|
||||
result.Result{Query: Q("a:b,c:t02:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t03:*"), Status: result.Pass},
|
||||
result.Result{Query: Q("a:b,c:t04:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t05:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t06:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t07:*"), Status: result.Pass},
|
||||
result.Result{Query: Q("a:b,c:t08:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t09:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t10:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t11:*"), Status: result.Pass},
|
||||
result.Result{Query: Q("a:b,c:t12:*"), Status: result.Pass},
|
||||
result.Result{Query: Q("a:b,c:t13:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t14:*"), Status: result.Pass},
|
||||
result.Result{Query: Q("a:b,c:t15:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t16:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t17:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t18:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t19:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t20:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t21:*"), Status: result.Pass},
|
||||
result.Result{Query: Q("a:b,c:t22:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t23:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t24:*"), Status: result.Pass},
|
||||
result.Result{Query: Q("a:b,c:t25:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t26:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t27:*"), Status: result.Pass},
|
||||
result.Result{Query: Q("a:b,c:t28:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t29:*"), Status: result.Failure},
|
||||
},
|
||||
updated: `
|
||||
# New failures. Please triage:
|
||||
crbug.com/dawn/0000 a:* [ Failure ]
|
||||
`,
|
||||
},
|
||||
{ //////////////////////////////////////////////////////////////////////
|
||||
name: "dont merge when under 21 children fail",
|
||||
expectations: ``,
|
||||
results: result.List{ // 20 failures, 66% fail
|
||||
result.Result{Query: Q("a:b,c:t00:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t01:*"), Status: result.Pass},
|
||||
result.Result{Query: Q("a:b,c:t02:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t03:*"), Status: result.Pass},
|
||||
result.Result{Query: Q("a:b,c:t04:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t05:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t06:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t07:*"), Status: result.Pass},
|
||||
result.Result{Query: Q("a:b,c:t08:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t09:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t10:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t11:*"), Status: result.Pass},
|
||||
result.Result{Query: Q("a:b,c:t12:*"), Status: result.Pass},
|
||||
result.Result{Query: Q("a:b,c:t13:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t14:*"), Status: result.Pass},
|
||||
result.Result{Query: Q("a:b,c:t15:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t16:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t17:*"), Status: result.Pass},
|
||||
result.Result{Query: Q("a:b,c:t18:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t19:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t20:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t21:*"), Status: result.Pass},
|
||||
result.Result{Query: Q("a:b,c:t22:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t23:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t24:*"), Status: result.Pass},
|
||||
result.Result{Query: Q("a:b,c:t25:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t26:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t27:*"), Status: result.Pass},
|
||||
result.Result{Query: Q("a:b,c:t28:*"), Status: result.Failure},
|
||||
result.Result{Query: Q("a:b,c:t29:*"), Status: result.Failure},
|
||||
},
|
||||
updated: `
|
||||
# New failures. Please triage:
|
||||
crbug.com/dawn/0000 a:b,c:t00:* [ Failure ]
|
||||
crbug.com/dawn/0000 a:b,c:t02:* [ Failure ]
|
||||
crbug.com/dawn/0000 a:b,c:t04:* [ Failure ]
|
||||
crbug.com/dawn/0000 a:b,c:t05:* [ Failure ]
|
||||
crbug.com/dawn/0000 a:b,c:t06:* [ Failure ]
|
||||
crbug.com/dawn/0000 a:b,c:t08:* [ Failure ]
|
||||
crbug.com/dawn/0000 a:b,c:t09:* [ Failure ]
|
||||
crbug.com/dawn/0000 a:b,c:t10:* [ Failure ]
|
||||
crbug.com/dawn/0000 a:b,c:t13:* [ Failure ]
|
||||
crbug.com/dawn/0000 a:b,c:t15:* [ Failure ]
|
||||
crbug.com/dawn/0000 a:b,c:t16:* [ Failure ]
|
||||
crbug.com/dawn/0000 a:b,c:t18:* [ Failure ]
|
||||
crbug.com/dawn/0000 a:b,c:t19:* [ Failure ]
|
||||
crbug.com/dawn/0000 a:b,c:t20:* [ Failure ]
|
||||
crbug.com/dawn/0000 a:b,c:t22:* [ Failure ]
|
||||
crbug.com/dawn/0000 a:b,c:t23:* [ Failure ]
|
||||
crbug.com/dawn/0000 a:b,c:t25:* [ Failure ]
|
||||
crbug.com/dawn/0000 a:b,c:t26:* [ Failure ]
|
||||
crbug.com/dawn/0000 a:b,c:t28:* [ Failure ]
|
||||
crbug.com/dawn/0000 a:b,c:t29:* [ Failure ]
|
||||
`,
|
||||
},
|
||||
} {
|
||||
ex, err := expectations.Parse(header + test.expectations)
|
||||
if err != nil {
|
||||
t.Fatalf("'%v': expectations.Parse():\n%v", test.name, err)
|
||||
}
|
||||
|
||||
errMsg := ""
|
||||
diagnostics, err := ex.Update(test.results)
|
||||
if err != nil {
|
||||
errMsg = err.Error()
|
||||
}
|
||||
if diff := cmp.Diff(errMsg, test.err); diff != "" {
|
||||
t.Errorf("'%v': expectations.Update() error:\n%v", test.name, diff)
|
||||
}
|
||||
|
||||
if diff := cmp.Diff(diagnostics, test.diagnostics); diff != "" {
|
||||
t.Errorf("'%v': diagnostics were not as expected:\n%v", test.name, diff)
|
||||
}
|
||||
|
||||
if diff := cmp.Diff(
|
||||
strings.Split(ex.String(), "\n"),
|
||||
strings.Split(header+test.updated, "\n")); diff != "" {
|
||||
t.Errorf("'%v': updated was not as expected:\n%v", test.name, diff)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -221,6 +221,14 @@ func (l List) FilterByTags(tags Tags) List {
|
|||
})
|
||||
}
|
||||
|
||||
// FilterByVariant returns the results that exactly match the given tags
|
||||
func (l List) FilterByVariant(tags Tags) List {
|
||||
str := TagsToString(tags)
|
||||
return l.Filter(func(r Result) bool {
|
||||
return len(r.Tags) == len(tags) && TagsToString(r.Tags) == str
|
||||
})
|
||||
}
|
||||
|
||||
// Statuses is a set of Status
|
||||
type Statuses = container.Set[Status]
|
||||
|
||||
|
|
|
|||
|
|
@ -782,6 +782,96 @@ func TestFilterByTags(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestFilterByVariant(t *testing.T) {
|
||||
type Test struct {
|
||||
results result.List
|
||||
tags result.Tags
|
||||
expect result.List
|
||||
}
|
||||
for _, test := range []Test{
|
||||
{ //////////////////////////////////////////////////////////////////////
|
||||
results: result.List{
|
||||
result.Result{
|
||||
Query: Q(`a`),
|
||||
Status: result.Pass,
|
||||
Tags: result.NewTags("x"),
|
||||
},
|
||||
result.Result{
|
||||
Query: Q(`b`),
|
||||
Status: result.Failure,
|
||||
Tags: result.NewTags("y"),
|
||||
},
|
||||
result.Result{
|
||||
Query: Q(`c`),
|
||||
Status: result.Pass,
|
||||
Tags: result.NewTags("x", "y"),
|
||||
},
|
||||
},
|
||||
tags: result.NewTags("x", "y"),
|
||||
expect: result.List{
|
||||
result.Result{
|
||||
Query: Q(`c`),
|
||||
Status: result.Pass,
|
||||
Tags: result.NewTags("x", "y"),
|
||||
},
|
||||
},
|
||||
},
|
||||
{ //////////////////////////////////////////////////////////////////////
|
||||
results: result.List{
|
||||
result.Result{
|
||||
Query: Q(`a`),
|
||||
Status: result.Pass,
|
||||
Tags: result.NewTags("x"),
|
||||
},
|
||||
result.Result{
|
||||
Query: Q(`b`),
|
||||
Status: result.Failure,
|
||||
Tags: result.NewTags("y"),
|
||||
},
|
||||
result.Result{
|
||||
Query: Q(`c`),
|
||||
Status: result.Pass,
|
||||
Tags: result.NewTags("x", "y"),
|
||||
},
|
||||
},
|
||||
tags: result.NewTags("x"),
|
||||
expect: result.List{
|
||||
result.Result{
|
||||
Query: Q(`a`),
|
||||
Status: result.Pass,
|
||||
Tags: result.NewTags("x"),
|
||||
},
|
||||
},
|
||||
},
|
||||
{ //////////////////////////////////////////////////////////////////////
|
||||
results: result.List{
|
||||
result.Result{
|
||||
Query: Q(`a`),
|
||||
Status: result.Pass,
|
||||
Tags: result.NewTags("x"),
|
||||
},
|
||||
result.Result{
|
||||
Query: Q(`b`),
|
||||
Status: result.Failure,
|
||||
Tags: result.NewTags("y"),
|
||||
},
|
||||
result.Result{
|
||||
Query: Q(`c`),
|
||||
Status: result.Pass,
|
||||
Tags: result.NewTags("x", "y"),
|
||||
},
|
||||
},
|
||||
tags: result.NewTags("q"),
|
||||
expect: result.List{},
|
||||
},
|
||||
} {
|
||||
got := test.results.FilterByVariant(test.tags)
|
||||
if diff := cmp.Diff(got, test.expect); diff != "" {
|
||||
t.Errorf("Results:\n%v\nFilterByVariant(%v) was not as expected:\n%v", test.results, test.tags, diff)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatuses(t *testing.T) {
|
||||
type Test struct {
|
||||
results result.List
|
||||
|
|
|
|||
Loading…
Reference in New Issue