encounter
/
dawn-cmake
mirror of https://github.com/encounter/dawn-cmake.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

tools: Add src/cts/expectations Update() 

Implement the logic to update an existing expectations file with new
results.

Bug: dawn:1342
Change-Id: Idcbad57946712539cc5f0d238f89a2adf42f5aa0
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/87702
Reviewed-by: Dan Sinclair <dsinclair@chromium.org>
Commit-Queue: Ben Clayton <bclayton@google.com>
This commit is contained in:
Ben Clayton 2022-04-29 11:06:34 +00:00 committed by Dawn LUCI CQ
parent 9432887ce8
commit 27f480b7e6
7 changed files with 1427 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
tools/src/cts/expectations/diagnostic.go Normal file
View File

@ -0,0 +1,56 @@
// Copyright 2022 The Dawn Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package expectations
import (
"fmt"
"strings"
)
// Severity is an enumerator of diagnostic severity
type Severity string
const (
Error Severity = "error"
Warning Severity = "warning"
Note Severity = "note"
)
// Diagnostic holds a line, column, message and severity.
// Diagnostic also implements the 'error' interface.
type Diagnostic struct {
Severity Severity
Line int // 1-based
Column int // 1-based
Message string
}
func (e Diagnostic) String() string {
sb := &strings.Builder{}
if e.Line > 0 {
fmt.Fprintf(sb, "%v", e.Line)
if e.Column > 0 {
fmt.Fprintf(sb, ":%v", e.Column)
}
sb.WriteString(" ")
}
sb.WriteString(string(e.Severity))
sb.WriteString(": ")
sb.WriteString(e.Message)
return sb.String()
}
// Error implements the 'error' interface.
func (e Diagnostic) Error() string { return e.String() }

22
tools/src/cts/expectations/parse.go
View File

@ -15,7 +15,6 @@
package expectations
import (
"fmt"
"strings"
"dawn.googlesource.com/dawn/tools/src/cts/result"
@ -26,19 +25,6 @@ const (
tagHeaderEnd = `END TAG HEADER`
)
// SyntaxError is the error type returned by Parse() when a syntax error is
// encountered.
type SyntaxError struct {
Line int // 1-based
Column int // 1-based
Message string
}
// Error implements the 'error' interface.
func (e SyntaxError) Error() string {
return fmt.Sprintf("%v:%v: %v", e.Line, e.Column, e.Message)
}
// Parse parses an expectations file, returning the Content
func Parse(body string) (Content, error) {
// LineType is an enumerator classifying the 'type' of the line.
@ -130,11 +116,11 @@ func Parse(body string) (Content, error) {
// syntaxErr is a helper for returning a SyntaxError with the current
// line and column index.
syntaxErr := func(at Token, msg string) error {
column := at.start + 1
if column == 1 {
column = len(l) + 1
columnIdx := at.start + 1
if columnIdx == 1 {
columnIdx = len(l) + 1
}
return SyntaxError{lineIdx, column, msg}
return Diagnostic{Error, lineIdx, columnIdx, msg}
}
// peek returns the next token without consuming it.

10
tools/src/cts/expectations/parse_test.go
View File

@ -432,27 +432,27 @@ crbug.com/456 [ Mac ] ghi_jkl [ PASS ]
{
name: "err missing tag ']'",
in: `[`,
expectErr: "1:2: expected ']' for tags",
expectErr: "1:2 error: expected ']' for tags",
}, /////////////////////////////////////////////////////////////////////
{
name: "err missing test query",
in: `[ a ]`,
expectErr: "1:6: expected test query",
expectErr: "1:6 error: expected test query",
}, /////////////////////////////////////////////////////////////////////
{
name: "err missing status EOL",
in: `[ a ] b`,
expectErr: "1:8: expected status",
expectErr: "1:8 error: expected status",
}, /////////////////////////////////////////////////////////////////////
{
name: "err missing status comment",
in: `[ a ] b # c`,
expectErr: "1:9: expected status",
expectErr: "1:9 error: expected status",
}, /////////////////////////////////////////////////////////////////////
{
name: "err missing status ']'",
in: `[ a ] b [ c`,
expectErr: "1:12: expected ']' for status",
expectErr: "1:12 error: expected ']' for status",
},
} {

592
tools/src/cts/expectations/update.go Normal file
View File

@ -0,0 +1,592 @@
// Copyright 2022 The Dawn Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package expectations
import (
"errors"
"fmt"
"sort"
"strings"
"dawn.googlesource.com/dawn/tools/src/container"
"dawn.googlesource.com/dawn/tools/src/cts/query"
"dawn.googlesource.com/dawn/tools/src/cts/result"
)
// Update performs an incremental update on the expectations using the provided
// results.
//
// Update will:
// • Remove any expectation lines that have a query where no results match.
// • Remove expectations lines that are in a chunk which is not annotated with
// 'KEEP', and all test results have the status 'Pass'.
// • Remove chunks that have had all expectation lines removed.
// • Appends new chunks for flaky and failing tests which are not covered by
// existing expectation lines.
//
// Update returns a list of diagnostics for things that should be addressed.
func (c *Content) Update(results result.List) ([]Diagnostic, error) {
// Make a copy of the results. This code mutates the list.
results = append(result.List{}, results...)
// Replace statuses that the CTS runner doesn't recognize with 'Failure'
simplifyStatuses(results)
// Produce a list of tag sets.
// We reverse the declared order, as webgpu-cts/expectations.txt lists the
// most important first (OS, GPU, etc), and result.MinimalVariantTags will
// prioritize folding away the earlier tag-sets.
tagSets := make([]result.Tags, len(c.Tags.Sets))
for i, s := range c.Tags.Sets {
tagSets[len(tagSets)-i-1] = s.Tags
}
// Update those expectations!
u := updater{
in: *c,
out: Content{},
qt: newQueryTree(results),
tagSets: tagSets,
}
if err := u.build(); err != nil {
return nil, fmt.Errorf("while updating expectations: %w", err)
}
*c = u.out
return u.diags, nil
}
// updater holds the state used for updating the expectations
type updater struct {
in Content // the original expectations Content
out Content // newly built expectations Content
qt queryTree // the query tree
diags []Diagnostic // diagnostics raised during update
tagSets []result.Tags // reverse-ordered tag-sets of 'in'
}
// simplifyStatuses replaces all result statuses that are not 'Pass',
// 'RetryOnFailure', 'Slow', 'Skip' with 'Failure'.
func simplifyStatuses(results result.List) {
for i, r := range results {
switch r.Status {
case result.Pass, result.RetryOnFailure, result.Slow, result.Skip:
// keep
default:
results[i].Status = result.Failure
}
}
}
const (
// Status used to mark results that have been already handled by an
// expectation.
consumed result.Status = "<<consumed>>"
// Chunk comment for new flakes
newFlakesComment = "# New flakes. Please triage:"
// Chunk comment for new failures
newFailuresComment = "# New failures. Please triage:"
)
// queryTree holds tree of queries to all results (no filtering by tag or
// status). The queryTree is used to glob all the results that match a
// particular query.
type queryTree struct {
// All the results.
results result.List
// consumedAt is a list of line numbers for the i'th result in 'results'
// Initially all line numbers are 0. When a result is consumed the line
// number is set.
consumedAt []int
// Each tree node holds a list of indices to results.
tree query.Tree[[]int]
}
// newQueryTree builds the queryTree from the list of results.
func newQueryTree(results result.List) queryTree {
// Build a map of query to result indices
queryToIndices := map[query.Query][]int{}
for i, r := range results {
l := queryToIndices[r.Query]
l = append(l, i)
queryToIndices[r.Query] = l
}
// Construct the query tree to result indices
tree := query.Tree[[]int]{}
for query, indices := range queryToIndices {
if err := tree.Add(query, indices); err != nil {
// Unreachable: The only error we could get is duplicate data for
// the same query, which should be impossible.
panic(err)
}
}
consumedAt := make([]int, len(results))
return queryTree{results, consumedAt, tree}
}
// glob returns the list of results matching the given tags under (or with) the
// given query.
func (qt *queryTree) glob(q query.Query) (result.List, error) {
glob, err := qt.tree.Glob(q)
if err != nil {
return nil, fmt.Errorf("while gathering results for query '%v': %w", q, err)
}
out := result.List{}
for _, indices := range glob {
for _, idx := range indices.Data {
out = append(out, qt.results[idx])
}
}
return out, nil
}
// globAndCheckForCollisions returns the list of results matching the given tags
// under (or with) the given query.
// globAndCheckForCollisions will return an error if any of the results are
// already consumed by a non-zero line. The non-zero line distinguishes between
// results consumed by expectations declared in the input (non-zero line), vs
// those that were introduced by the update (zero line). We only want to error
// if there's a collision in user declared expectations.
func (qt *queryTree) globAndCheckForCollisions(q query.Query, t result.Tags) (result.List, error) {
glob, err := qt.tree.Glob(q)
if err != nil {
return nil, err
}
out := result.List{}
for _, indices := range glob {
for _, idx := range indices.Data {
if r := qt.results[idx]; r.Tags.ContainsAll(t) {
if at := qt.consumedAt[idx]; at > 0 {
if len(t) > 0 {
return nil, fmt.Errorf("%v %v collides with expectation at line %v", t, q, at)
}
return nil, fmt.Errorf("%v collides with expectation at line %v", q, at)
}
out = append(out, r)
}
}
}
return out, nil
}
// markAsConsumed marks all the results matching the given tags
// under (or with) the given query, as consumed.
// line is used to record the line at which the results were consumed. If the
// results were consumed as part of generating new expectations then line should
// be 0. See queryTree.globAndCheckForCollisions().
func (qt *queryTree) markAsConsumed(q query.Query, t result.Tags, line int) {
if glob, err := qt.tree.Glob(q); err == nil {
for _, indices := range glob {
for _, idx := range indices.Data {
r := &qt.results[idx]
if r.Tags.ContainsAll(t) {
r.Status = consumed
qt.consumedAt[idx] = line
}
}
}
}
}
// build is the updater top-level function.
// build first appends to u.out all chunks from 'u.in' with expectations updated
// using the new results, and then appends any new expectations to u.out.
func (u *updater) build() error {
// Update all the existing chunks
for _, in := range u.in.Chunks {
out := u.chunk(in)
// If all chunk had expectations, but now they've gone, remove the chunk
if len(in.Expectations) > 0 && len(out.Expectations) == 0 {
continue
}
if out.IsBlankLine() {
u.out.MaybeAddBlankLine()
continue
}
u.out.Chunks = append(u.out.Chunks, out)
}
// Emit new expectations (flaky, failing)
if err := u.addNewExpectations(); err != nil {
return fmt.Errorf("failed to add new expectations: %w", err)
}
return nil
}
// chunk returns a new Chunk, based on 'in', with the expectations updated.
func (u *updater) chunk(in Chunk) Chunk {
if len(in.Expectations) == 0 {
return in // Just a comment / blank line
}
// Skip over any untriaged failures / flake chunks.
// We'll just rebuild them at the end.
if len(in.Comments) > 0 {
switch in.Comments[0] {
case newFailuresComment, newFlakesComment:
return Chunk{}
}
}
keep := false // Does the chunk comment contain 'KEEP' ?
for _, l := range in.Comments {
if strings.Contains(l, "KEEP") {
keep = true
break
}
}
// Begin building the output chunk.
// Copy over the chunk's comments.
out := Chunk{Comments: in.Comments}
// Build the new chunk's expectations
for _, exIn := range in.Expectations {
exOut := u.expectation(exIn, keep)
out.Expectations = append(out.Expectations, exOut...)
}
// Sort the expectations to keep things clean and tidy.
sort.Slice(out.Expectations, func(i, j int) bool {
switch {
case out.Expectations[i].Query < out.Expectations[j].Query:
return true
case out.Expectations[i].Query > out.Expectations[j].Query:
return false
}
a := result.TagsToString(out.Expectations[i].Tags)
b := result.TagsToString(out.Expectations[j].Tags)
switch {
case a < b:
return true
case a > b:
return false
}
return false
})
return out
}
// chunk returns a new list of Expectations, based on the Expectation 'in',
// using the new result data.
func (u *updater) expectation(in Expectation, keep bool) []Expectation {
// noResults is a helper for returning when the expectation has no test
// results.
// If the expectation has an expected 'Skip' result, then we're likely
// to be missing results (as the test was not run). In this situation
// the expectation is preserved, and no diagnostics are raised.
// If the expectation did not have a 'Skip' result, then a diagnostic will
// be raised and the expectation will be removed.
noResults := func() []Expectation {
if container.NewSet(in.Status...).Contains(string(result.Skip)) {
return []Expectation{in}
}
// Expectation does not have a 'Skip' result.
if len(in.Tags) > 0 {
u.diag(Warning, in.Line, "no results found for '%v' with tags %v", in.Query, in.Tags)
} else {
u.diag(Warning, in.Line, "no results found for '%v'", in.Query)
}
// Remove the no-results expectation
return []Expectation{}
}
// Grab all the results that match the expectation's query
q := query.Parse(in.Query)
// Glob the results for the expectation's query + tag combination.
// Ensure that none of these are already consumed.
results, err := u.qt.globAndCheckForCollisions(q, in.Tags)
// If we can't find any results for this query + tag combination, then bail.
switch {
case errors.As(err, &query.ErrNoDataForQuery{}):
return noResults()
case err != nil:
u.diag(Error, in.Line, "%v", err)
return []Expectation{}
case len(results) == 0:
return noResults()
}
// Before returning, mark all the results as consumed.
// Note: this has to happen *after* we've generated the new expectations, as
// marking the results as 'consumed' will impact the logic of
// expectationsForRoot()
defer u.qt.markAsConsumed(q, in.Tags, in.Line)
if keep { // Expectation chunk was marked with 'KEEP'
// Add a diagnostic if all tests of the expectation were 'Pass'
if s := results.Statuses(); len(s) == 1 && s.One() == result.Pass {
if c := len(results); c > 1 {
u.diag(Note, in.Line, "all %d tests now pass", len(results))
} else {
u.diag(Note, in.Line, "test now passes")
}
}
return []Expectation{in}
}
// Rebuild the expectations for this query.
return u.expectationsForRoot(q, in.Line, in.Bug, in.Comment)
}
// addNewExpectations (potentially) appends to 'u.out' chunks for new flaky and
// failing tests.
func (u *updater) addNewExpectations() error {
// Scan the full result list to obtain all the test variants
// (unique tag combinations).
allVariants := u.qt.results.Variants()
// For each variant:
// • Build a query tree using the results filtered to the variant, and then
// reduce the tree.
// • Take all the reduced-tree leaf nodes, and add these to 'roots'.
// Once we've collected all the roots, we'll use these to build the
// expectations across the reduced set of tags.
roots := container.NewMap[string, query.Query]()
for _, variant := range allVariants {
// Build a tree from the results matching the given variant.
tree, err := u.qt.results.FilterByVariant(variant).StatusTree()
if err != nil {
return fmt.Errorf("while building tree for tags '%v': %w", variant, err)
}
// Reduce the tree.
tree.Reduce(treeReducer)
// Add all the reduced leaf nodes to 'roots'.
for _, qd := range tree.List() {
roots.Add(qd.Query.String(), qd.Query)
}
}
// Build all the expectations for each of the roots.
expectations := []Expectation{}
for _, root := range roots.Values() {
expectations = append(expectations, u.expectationsForRoot(
root, // Root query
0, // Line number
"crbug.com/dawn/0000", // Bug
"", // Comment
)...)
}
// Bin the expectations by failure or flake.
flakes, failures := []Expectation{}, []Expectation{}
for _, r := range expectations {
if container.NewSet(r.Status...).Contains(string(result.RetryOnFailure)) {
flakes = append(flakes, r)
} else {
failures = append(failures, r)
}
}
// Create chunks for any flakes and failures, in that order.
for _, group := range []struct {
results []Expectation
comment string
}{
{flakes, newFlakesComment},
{failures, newFailuresComment},
} {
if len(group.results) > 0 {
u.out.MaybeAddBlankLine()
u.out.Chunks = append(u.out.Chunks, Chunk{
Comments: []string{group.comment},
Expectations: group.results,
})
}
}
return nil
}
// expectationsForRoot builds a list of expectations that cover the failing
// tests for the results under root.
// The returned list of expectations is optimized by reducing queries to the
// most common root, and reducing tags to the smallest required set.
func (u *updater) expectationsForRoot(
root query.Query, // The sub-tree query root
line int, // The originating line, when producing diagnostics
bug string, // The bug to apply to all returned expectations
comment string, // The comment to apply to all returned expectations
) []Expectation {
results, err := u.qt.glob(root)
if err != nil {
u.diag(Error, line, "%v", err)
return nil
}
// Using the full list of unfiltered tests, generate the minimal set of
// variants (tags) that uniquely classify the results with differing status.
minimalVariants := u.
cleanupTags(results).
MinimalVariantTags(u.tagSets)
// For each minimized variant...
reduced := result.List{}
for _, variant := range minimalVariants {
// Build a query tree from this variant...
tree := result.StatusTree{}
filtered := results.FilterByTags(variant)
for _, r := range filtered {
// Note: variants may overlap, but overlaped queries will have
// identical statuses, so we can just ignore the error for Add().
tree.Add(r.Query, r.Status)
}
// ... and reduce the tree by collapsing sub-trees that have common
// statuses.
tree.ReduceUnder(root, treeReducer)
// Append the reduced tree nodes to the results list
for _, qs := range tree.List() {
reduced = append(reduced, result.Result{
Query: qs.Query,
Tags: variant,
Status: qs.Data,
})
}
}
// Filter out any results that passed or have already been consumed
filtered := reduced.Filter(func(r result.Result) bool {
return r.Status != result.Pass && r.Status != consumed
})
// Mark all the new expectation results as consumed.
for _, r := range filtered {
u.qt.markAsConsumed(r.Query, r.Tags, 0)
}
// Transform the results to expectations.
return u.resultsToExpectations(filtered, bug, comment)
}
// resultsToExpectations returns a list of expectations from the given results.
// Each expectation will have the same query, tags and status as the input
// result, along with the specified bug and comment.
//
// If the result query target is a test without a wildcard, then the expectation
// will have a wildcard automatically appended. This is to satisfy a requirement
// of the expectation validator.
func (u *updater) resultsToExpectations(results result.List, bug, comment string) []Expectation {
results.Sort()
out := make([]Expectation, len(results))
for i, r := range results {
q := r.Query.String()
if r.Query.Target() == query.Tests && !r.Query.IsWildcard() {
// The expectation validator wants a trailing ':' for test queries
q += query.TargetDelimiter
}
out[i] = Expectation{
Bug: bug,
Tags: r.Tags,
Query: q,
Status: []string{string(r.Status)},
Comment: comment,
}
}
return out
}
// cleanupTags returns a copy of the provided results with:
// • All tags not found in the expectations list removed
// • All but the highest priority tag for any tag-set.
// The tag sets are defined by the `BEGIN TAG HEADER` / `END TAG HEADER`
// section at the top of the expectations file.
func (u *updater) cleanupTags(results result.List) result.List {
return results.TransformTags(func(t result.Tags) result.Tags {
type HighestPrioritySetTag struct {
tag string
priority int
}
// Set name to highest priority tag for that set
best := map[string]HighestPrioritySetTag{}
for tag := range t {
sp, ok := u.in.Tags.ByName[tag]
if ok {
if set := best[sp.Set]; sp.Priority >= set.priority {
best[sp.Set] = HighestPrioritySetTag{tag, sp.Priority}
}
}
}
t = result.NewTags()
for _, ts := range best {
t.Add(ts.tag)
}
return t
})
}
// treeReducer is a function that can be used by StatusTree.Reduce() to reduce
// tree nodes with the same status.
// treeReducer will collapse trees nodes if any of the following are true:
// • All child nodes have the same status
// • More than 75% of the child nodes have a non-pass status, and none of the
// children are consumed.
// • There are more than 20 child nodes with a non-pass status, and none of the
// children are consumed.
func treeReducer(statuses []result.Status) *result.Status {
counts := map[result.Status]int{}
for _, s := range statuses {
counts[s] = counts[s] + 1
}
if len(counts) == 1 {
return &statuses[0] // All the same status
}
if counts[consumed] > 0 {
return nil // Partially consumed trees cannot be merged
}
highestNonPassCount := 0
highestNonPassStatus := result.Failure
for s, n := range counts {
if s != result.Pass {
if percent := (100 * n) / len(statuses); percent > 75 {
// Over 75% of all the children are of non-pass status s.
return &s
}
if n > highestNonPassCount {
highestNonPassCount = n
highestNonPassStatus = s
}
}
}
if highestNonPassCount > 20 {
// Over 20 child node failed.
return &highestNonPassStatus
}
return nil
}
// diag appends a new diagnostic to u.diags with the given severity, line and
// message.
func (u *updater) diag(severity Severity, line int, msg string, args ...interface{}) {
u.diags = append(u.diags, Diagnostic{
Severity: severity,
Line: line,
Message: fmt.Sprintf(msg, args...),
})
}

672
tools/src/cts/expectations/update_test.go Normal file
View File

@ -0,0 +1,672 @@
// Copyright 2022 The Dawn Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package expectations_test
import (
"strings"
"testing"
"dawn.googlesource.com/dawn/tools/src/cts/expectations"
"dawn.googlesource.com/dawn/tools/src/cts/query"
"dawn.googlesource.com/dawn/tools/src/cts/result"
"github.com/google/go-cmp/cmp"
)
var Q = query.Parse
func TestUpdate(t *testing.T) {
header := `# BEGIN TAG HEADER
# OS
# tags: [ os-a os-b os-c ]
# GPU
# tags: [ gpu-a gpu-b gpu-c ]
# END TAG HEADER
`
headerLines := strings.Count(header, "\n")
type Test struct {
name string
expectations string
results result.List
updated string
diagnostics []expectations.Diagnostic
err string
}
for _, test := range []Test{
{ //////////////////////////////////////////////////////////////////////
name: "empty results",
expectations: ``,
results: result.List{},
},
{ //////////////////////////////////////////////////////////////////////
name: "no results found",
expectations: `
crbug.com/a/123 a:missing,test,result:* [ Failure ]
crbug.com/a/123 [ tag ] another:missing,test,result:* [ Failure ]
some:other,test:* [ Failure ]
`,
results: result.List{
result.Result{
Query: Q("some:other,test:*"),
Tags: result.NewTags("os-a", "gpu-a"),
Status: result.Failure,
},
result.Result{
Query: Q("some:other,test:*"),
Tags: result.NewTags("os-b", "gpu-b"),
Status: result.Failure,
},
},
updated: `
some:other,test:* [ Failure ]
`,
diagnostics: []expectations.Diagnostic{
{
Severity: expectations.Warning,
Line: headerLines + 2,
Message: "no results found for 'a:missing,test,result:*'",
},
{
Severity: expectations.Warning,
Line: headerLines + 3,
Message: "no results found for 'another:missing,test,result:*' with tags [tag]",
},
},
},
{ //////////////////////////////////////////////////////////////////////
name: "no results found KEEP",
expectations: `
# KEEP
crbug.com/a/123 a:missing,test,result:* [ Failure ]
some:other,test:* [ Failure ]
`,
results: result.List{
result.Result{
Query: Q("some:other,test:*"),
Tags: result.NewTags("os-a", "gpu-a"),
Status: result.Failure,
},
result.Result{
Query: Q("some:other,test:*"),
Tags: result.NewTags("os-b", "gpu-b"),
Status: result.Failure,
},
},
updated: `
some:other,test:* [ Failure ]
`,
diagnostics: []expectations.Diagnostic{
{
Severity: expectations.Warning,
Line: headerLines + 3,
Message: "no results found for 'a:missing,test,result:*'",
},
},
},
{ //////////////////////////////////////////////////////////////////////
name: "no results found Skip",
expectations: `
crbug.com/a/123 a:missing,test,result:* [ Skip ]
some:other,test:* [ Failure ]
`,
results: result.List{
result.Result{
Query: Q("some:other,test:*"),
Tags: result.NewTags("os-a", "gpu-a"),
Status: result.Failure,
},
result.Result{
Query: Q("some:other,test:*"),
Tags: result.NewTags("os-b", "gpu-b"),
Status: result.Failure,
},
},
updated: `
crbug.com/a/123 a:missing,test,result:* [ Skip ]
some:other,test:* [ Failure ]
`,
},
{ //////////////////////////////////////////////////////////////////////
name: "simple expectation collision",
expectations: `
a:b,c:* [ Failure ]
a:b,c:* [ Failure ]
`,
results: result.List{
result.Result{
Query: Q("a:b,c:d"),
Tags: result.NewTags("os-a", "os-c", "gpu-b"),
Status: result.Failure,
},
},
updated: `
a:b,c:* [ Failure ]
`,
diagnostics: []expectations.Diagnostic{
{
Severity: expectations.Error,
Line: headerLines + 3,
Message: "a:b,c:* collides with expectation at line 8",
},
},
},
{ //////////////////////////////////////////////////////////////////////
name: "simple expectation with tags",
expectations: `
[ os-a ] a:b,c:* [ Failure ]
[ gpu-b ] a:b,c:* [ Failure ]
`,
results: result.List{
result.Result{
Query: Q("a:b,c:d"),
Tags: result.NewTags("os-a", "os-c", "gpu-b"),
Status: result.Failure,
},
},
updated: `
a:b,c:* [ Failure ]
`,
diagnostics: []expectations.Diagnostic{
{
Severity: expectations.Error,
Line: headerLines + 3,
Message: "[gpu-b] a:b,c:* collides with expectation at line 8",
},
},
},
{ //////////////////////////////////////////////////////////////////////
name: "simple expectation collision KEEP",
expectations: `
# KEEP
a:b,c:* [ Failure ]
a:b,c:* [ Failure ]
`,
results: result.List{
result.Result{
Query: Q("a:b,c:d"),
Tags: result.NewTags("os-a", "os-c", "gpu-b"),
Status: result.Failure,
},
},
updated: `
# KEEP
a:b,c:* [ Failure ]
`,
diagnostics: []expectations.Diagnostic{
{
Severity: expectations.Error,
Line: headerLines + 4,
Message: "a:b,c:* collides with expectation at line 9",
},
},
},
{ //////////////////////////////////////////////////////////////////////
name: "collision with child-expectation",
expectations: `
a:b:x:* [ Failure ]
a:b:* [ Failure ]
`,
results: result.List{
result.Result{
Query: Q("a:b:x:*"),
Tags: result.NewTags("os-a", "os-c", "gpu-b"),
Status: result.Failure,
},
result.Result{
Query: Q("a:b:y:*"),
Tags: result.NewTags("os-a", "os-c", "gpu-b"),
Status: result.Failure,
},
},
updated: `
a:b:x:* [ Failure ]
# New failures. Please triage:
crbug.com/dawn/0000 a:b:y:* [ Failure ]
`,
diagnostics: []expectations.Diagnostic{
{
Severity: expectations.Error,
Line: headerLines + 3,
Message: "a:b:* collides with expectation at line 8",
},
},
},
{ //////////////////////////////////////////////////////////////////////
name: "collision with parent-expectation",
expectations: `
a:b:* [ Failure ]
a:b:x:* [ Failure ]
`,
results: result.List{
result.Result{
Query: Q("a:b:x:*"),
Tags: result.NewTags("os-a", "os-c", "gpu-b"),
Status: result.Failure,
},
result.Result{
Query: Q("a:b:y:*"),
Tags: result.NewTags("os-a", "os-c", "gpu-b"),
Status: result.Failure,
},
},
updated: `
a:b:* [ Failure ]
`,
diagnostics: []expectations.Diagnostic{
{
Severity: expectations.Error,
Line: headerLines + 3,
Message: "a:b:x:* collides with expectation at line 8",
},
},
},
{ //////////////////////////////////////////////////////////////////////
name: "expectation test now passes",
expectations: `
crbug.com/a/123 [ gpu-a os-a ] a:b,c:* [ Failure ]
crbug.com/a/123 [ gpu-b os-b ] a:b,c:* [ Failure ]
`,
results: result.List{
result.Result{
Query: Q("a:b,c:*"),
Tags: result.NewTags("os-a", "gpu-a"),
Status: result.Pass,
},
result.Result{
Query: Q("a:b,c:*"),
Tags: result.NewTags("os-b", "gpu-b"),
Status: result.Abort,
},
},
updated: `
crbug.com/a/123 [ os-b ] a:b,c:* [ Failure ]
`,
},
{ //////////////////////////////////////////////////////////////////////
name: "expectation case now passes",
expectations: `
crbug.com/a/123 [ gpu-a os-a ] a:b,c:d [ Failure ]
crbug.com/a/123 [ gpu-b os-b ] a:b,c:d [ Failure ]
`,
results: result.List{
result.Result{
Query: Q("a:b,c:d"),
Tags: result.NewTags("os-a", "gpu-a"),
Status: result.Pass,
},
result.Result{
Query: Q("a:b,c:d"),
Tags: result.NewTags("os-b", "gpu-b"),
Status: result.Abort,
},
},
updated: `
crbug.com/a/123 [ os-b ] a:b,c:d: [ Failure ]
`,
},
{ //////////////////////////////////////////////////////////////////////
name: "expectation case now passes KEEP - single",
expectations: `
# KEEP
crbug.com/a/123 [ gpu-a os-a ] a:b,c:d [ Failure ]
crbug.com/a/123 [ gpu-b os-b ] a:b,c:d [ Failure ]
`,
results: result.List{
result.Result{
Query: Q("a:b,c:d"),
Tags: result.NewTags("os-a", "gpu-a"),
Status: result.Pass,
},
result.Result{
Query: Q("a:b,c:d"),
Tags: result.NewTags("os-b", "gpu-b"),
Status: result.Abort,
},
},
updated: `
# KEEP
crbug.com/a/123 [ gpu-a os-a ] a:b,c:d [ Failure ]
crbug.com/a/123 [ gpu-b os-b ] a:b,c:d [ Failure ]
`,
diagnostics: []expectations.Diagnostic{
{
Severity: expectations.Note,
Line: headerLines + 3,
Message: "test now passes",
},
},
},
{ //////////////////////////////////////////////////////////////////////
name: "expectation case now passes KEEP - multiple",
expectations: `
# KEEP
crbug.com/a/123 a:b,c:d:* [ Failure ]
`,
results: result.List{
result.Result{Query: Q("a:b,c:d:a"), Status: result.Pass},
result.Result{Query: Q("a:b,c:d:b"), Status: result.Pass},
result.Result{Query: Q("a:b,c:d:c"), Status: result.Pass},
result.Result{Query: Q("a:b,c:d:d"), Status: result.Pass},
},
updated: `
# KEEP
crbug.com/a/123 a:b,c:d:* [ Failure ]
`,
diagnostics: []expectations.Diagnostic{
{
Severity: expectations.Note,
Line: headerLines + 3,
Message: "all 4 tests now pass",
},
},
},
{ //////////////////////////////////////////////////////////////////////
name: "new test results",
expectations: `# A comment`,
results: result.List{
result.Result{
Query: Q("suite:dir_a,dir_b:test_a:*"),
Tags: result.NewTags("os-a", "gpu-a"),
Status: result.Abort,
},
result.Result{
Query: Q("suite:dir_a,dir_b:test_a:*"),
Tags: result.NewTags("os-a", "gpu-b"),
Status: result.Abort,
},
result.Result{
Query: Q("suite:dir_a,dir_b:test_c:case=4;*"),
Tags: result.NewTags("os-b", "gpu-b"),
Status: result.Crash,
},
result.Result{
Query: Q("suite:dir_a,dir_b:test_c:case=5;*"),
Tags: result.NewTags("os-b", "gpu-b"),
Status: result.RetryOnFailure,
},
result.Result{
Query: Q("suite:dir_a,dir_b:test_b;case=5;*"),
Tags: result.NewTags("os-b", "gpu-b"),
Status: result.Pass,
},
result.Result{
Query: Q("suite:dir_a,dir_b:test_b:*"),
Tags: result.NewTags("os-a", "gpu-a"),
Status: result.Skip,
},
result.Result{
Query: Q("suite:dir_a,dir_b:test_b:*"),
Tags: result.NewTags("os-b", "gpu-a"),
Status: result.Pass,
},
result.Result{
Query: Q("suite:dir_a,dir_b:test_c:*"),
Tags: result.NewTags("os-a", "gpu-a"),
Status: result.RetryOnFailure,
},
},
updated: `# A comment
# New flakes. Please triage:
crbug.com/dawn/0000 [ gpu-a os-a ] suite:dir_a,dir_b:test_c:* [ RetryOnFailure ]
crbug.com/dawn/0000 [ gpu-b os-b ] suite:dir_a,dir_b:test_c:case=5;* [ RetryOnFailure ]
# New failures. Please triage:
crbug.com/dawn/0000 [ gpu-b os-a ] suite:* [ Failure ]
crbug.com/dawn/0000 [ gpu-a os-a ] suite:dir_a,dir_b:test_a:* [ Failure ]
crbug.com/dawn/0000 [ gpu-a os-a ] suite:dir_a,dir_b:test_b:* [ Skip ]
crbug.com/dawn/0000 [ gpu-b os-b ] suite:dir_a,dir_b:test_c:case=4;* [ Failure ]
`,
},
{ //////////////////////////////////////////////////////////////////////
name: "filter unknown tags",
expectations: ``,
results: result.List{
result.Result{
Query: Q("a:b,c:*"),
Tags: result.NewTags("os-a", "gpu-x"),
Status: result.Failure,
},
result.Result{
Query: Q("a:b,c:*"),
Tags: result.NewTags("os-b", "gpu-x"),
Status: result.Crash,
},
result.Result{
Query: Q("a:b,c:*"),
Tags: result.NewTags("os-x", "gpu-b"),
Status: result.Failure,
},
result.Result{
Query: Q("a:b,c:*"),
Tags: result.NewTags("os-x", "gpu-a"),
Status: result.Crash,
},
result.Result{
Query: Q("a:b,c:*"),
Tags: result.NewTags("os-c", "gpu-c"),
Status: result.Pass,
},
},
updated: `
# New failures. Please triage:
crbug.com/dawn/0000 [ gpu-a ] a:* [ Failure ]
crbug.com/dawn/0000 [ gpu-b ] a:* [ Failure ]
crbug.com/dawn/0000 [ os-a ] a:* [ Failure ]
crbug.com/dawn/0000 [ os-b ] a:* [ Failure ]
`,
},
{ //////////////////////////////////////////////////////////////////////
name: "prioritize tag sets",
expectations: ``,
results: result.List{
result.Result{
Query: Q("a:b,c:*"),
Tags: result.NewTags("os-a", "os-c", "gpu-b"),
Status: result.Failure,
},
result.Result{
Query: Q("a:b,c:*"),
Tags: result.NewTags("gpu-a", "os-b", "gpu-c"),
Status: result.Failure,
},
result.Result{
Query: Q("a:b,c:*"),
Tags: result.NewTags("os-c", "gpu-c"),
Status: result.Pass,
},
},
updated: `
# New failures. Please triage:
crbug.com/dawn/0000 [ gpu-b os-c ] a:* [ Failure ]
crbug.com/dawn/0000 [ gpu-c os-b ] a:* [ Failure ]
`,
},
{ //////////////////////////////////////////////////////////////////////
name: "merge when over 75% of children fail",
expectations: ``,
results: result.List{
result.Result{Query: Q("a:b,c:t0:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t1:*"), Status: result.Pass},
result.Result{Query: Q("a:b,c:t2:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t3:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t4:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t5:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t6:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t7:*"), Status: result.Pass},
result.Result{Query: Q("a:b,c:t8:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t9:*"), Status: result.Failure},
},
updated: `
# New failures. Please triage:
crbug.com/dawn/0000 a:* [ Failure ]
`,
},
{ //////////////////////////////////////////////////////////////////////
name: "don't merge when under 75% of children fail",
expectations: ``,
results: result.List{
result.Result{Query: Q("a:b,c:t0:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t1:*"), Status: result.Pass},
result.Result{Query: Q("a:b,c:t2:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t3:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t4:*"), Status: result.Pass},
result.Result{Query: Q("a:b,c:t5:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t6:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t7:*"), Status: result.Pass},
result.Result{Query: Q("a:b,c:t8:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t9:*"), Status: result.Failure},
},
updated: `
# New failures. Please triage:
crbug.com/dawn/0000 a:b,c:t0:* [ Failure ]
crbug.com/dawn/0000 a:b,c:t2:* [ Failure ]
crbug.com/dawn/0000 a:b,c:t3:* [ Failure ]
crbug.com/dawn/0000 a:b,c:t5:* [ Failure ]
crbug.com/dawn/0000 a:b,c:t6:* [ Failure ]
crbug.com/dawn/0000 a:b,c:t8:* [ Failure ]
crbug.com/dawn/0000 a:b,c:t9:* [ Failure ]
`,
},
{ //////////////////////////////////////////////////////////////////////
name: "merge when over 20 children fail",
expectations: ``,
results: result.List{ // 21 failures, 70% fail
result.Result{Query: Q("a:b,c:t00:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t01:*"), Status: result.Pass},
result.Result{Query: Q("a:b,c:t02:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t03:*"), Status: result.Pass},
result.Result{Query: Q("a:b,c:t04:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t05:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t06:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t07:*"), Status: result.Pass},
result.Result{Query: Q("a:b,c:t08:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t09:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t10:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t11:*"), Status: result.Pass},
result.Result{Query: Q("a:b,c:t12:*"), Status: result.Pass},
result.Result{Query: Q("a:b,c:t13:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t14:*"), Status: result.Pass},
result.Result{Query: Q("a:b,c:t15:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t16:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t17:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t18:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t19:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t20:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t21:*"), Status: result.Pass},
result.Result{Query: Q("a:b,c:t22:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t23:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t24:*"), Status: result.Pass},
result.Result{Query: Q("a:b,c:t25:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t26:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t27:*"), Status: result.Pass},
result.Result{Query: Q("a:b,c:t28:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t29:*"), Status: result.Failure},
},
updated: `
# New failures. Please triage:
crbug.com/dawn/0000 a:* [ Failure ]
`,
},
{ //////////////////////////////////////////////////////////////////////
name: "dont merge when under 21 children fail",
expectations: ``,
results: result.List{ // 20 failures, 66% fail
result.Result{Query: Q("a:b,c:t00:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t01:*"), Status: result.Pass},
result.Result{Query: Q("a:b,c:t02:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t03:*"), Status: result.Pass},
result.Result{Query: Q("a:b,c:t04:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t05:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t06:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t07:*"), Status: result.Pass},
result.Result{Query: Q("a:b,c:t08:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t09:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t10:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t11:*"), Status: result.Pass},
result.Result{Query: Q("a:b,c:t12:*"), Status: result.Pass},
result.Result{Query: Q("a:b,c:t13:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t14:*"), Status: result.Pass},
result.Result{Query: Q("a:b,c:t15:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t16:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t17:*"), Status: result.Pass},
result.Result{Query: Q("a:b,c:t18:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t19:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t20:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t21:*"), Status: result.Pass},
result.Result{Query: Q("a:b,c:t22:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t23:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t24:*"), Status: result.Pass},
result.Result{Query: Q("a:b,c:t25:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t26:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t27:*"), Status: result.Pass},
result.Result{Query: Q("a:b,c:t28:*"), Status: result.Failure},
result.Result{Query: Q("a:b,c:t29:*"), Status: result.Failure},
},
updated: `
# New failures. Please triage:
crbug.com/dawn/0000 a:b,c:t00:* [ Failure ]
crbug.com/dawn/0000 a:b,c:t02:* [ Failure ]
crbug.com/dawn/0000 a:b,c:t04:* [ Failure ]
crbug.com/dawn/0000 a:b,c:t05:* [ Failure ]
crbug.com/dawn/0000 a:b,c:t06:* [ Failure ]
crbug.com/dawn/0000 a:b,c:t08:* [ Failure ]
crbug.com/dawn/0000 a:b,c:t09:* [ Failure ]
crbug.com/dawn/0000 a:b,c:t10:* [ Failure ]
crbug.com/dawn/0000 a:b,c:t13:* [ Failure ]
crbug.com/dawn/0000 a:b,c:t15:* [ Failure ]
crbug.com/dawn/0000 a:b,c:t16:* [ Failure ]
crbug.com/dawn/0000 a:b,c:t18:* [ Failure ]
crbug.com/dawn/0000 a:b,c:t19:* [ Failure ]
crbug.com/dawn/0000 a:b,c:t20:* [ Failure ]
crbug.com/dawn/0000 a:b,c:t22:* [ Failure ]
crbug.com/dawn/0000 a:b,c:t23:* [ Failure ]
crbug.com/dawn/0000 a:b,c:t25:* [ Failure ]
crbug.com/dawn/0000 a:b,c:t26:* [ Failure ]
crbug.com/dawn/0000 a:b,c:t28:* [ Failure ]
crbug.com/dawn/0000 a:b,c:t29:* [ Failure ]
`,
},
} {
ex, err := expectations.Parse(header + test.expectations)
if err != nil {
t.Fatalf("'%v': expectations.Parse():\n%v", test.name, err)
}
errMsg := ""
diagnostics, err := ex.Update(test.results)
if err != nil {
errMsg = err.Error()
}
if diff := cmp.Diff(errMsg, test.err); diff != "" {
t.Errorf("'%v': expectations.Update() error:\n%v", test.name, diff)
}
if diff := cmp.Diff(diagnostics, test.diagnostics); diff != "" {
t.Errorf("'%v': diagnostics were not as expected:\n%v", test.name, diff)
}
if diff := cmp.Diff(
strings.Split(ex.String(), "\n"),
strings.Split(header+test.updated, "\n")); diff != "" {
t.Errorf("'%v': updated was not as expected:\n%v", test.name, diff)
}
}
}

8
tools/src/cts/result/result.go
View File

@ -221,6 +221,14 @@ func (l List) FilterByTags(tags Tags) List {
})
}
// FilterByVariant returns the results that exactly match the given tags
func (l List) FilterByVariant(tags Tags) List {
str := TagsToString(tags)
return l.Filter(func(r Result) bool {
return len(r.Tags) == len(tags) && TagsToString(r.Tags) == str
})
}
// Statuses is a set of Status
type Statuses = container.Set[Status]

90
tools/src/cts/result/result_test.go
View File

@ -782,6 +782,96 @@ func TestFilterByTags(t *testing.T) {
}
}
func TestFilterByVariant(t *testing.T) {
type Test struct {
results result.List
tags result.Tags
expect result.List
}
for _, test := range []Test{
{ //////////////////////////////////////////////////////////////////////
results: result.List{
result.Result{
Query: Q(`a`),
Status: result.Pass,
Tags: result.NewTags("x"),
},
result.Result{
Query: Q(`b`),
Status: result.Failure,
Tags: result.NewTags("y"),
},
result.Result{
Query: Q(`c`),
Status: result.Pass,
Tags: result.NewTags("x", "y"),
},
},
tags: result.NewTags("x", "y"),
expect: result.List{
result.Result{
Query: Q(`c`),
Status: result.Pass,
Tags: result.NewTags("x", "y"),
},
},
},
{ //////////////////////////////////////////////////////////////////////
results: result.List{
result.Result{
Query: Q(`a`),
Status: result.Pass,
Tags: result.NewTags("x"),
},
result.Result{
Query: Q(`b`),
Status: result.Failure,
Tags: result.NewTags("y"),
},
result.Result{
Query: Q(`c`),
Status: result.Pass,
Tags: result.NewTags("x", "y"),
},
},
tags: result.NewTags("x"),
expect: result.List{
result.Result{
Query: Q(`a`),
Status: result.Pass,
Tags: result.NewTags("x"),
},
},
},
{ //////////////////////////////////////////////////////////////////////
results: result.List{
result.Result{
Query: Q(`a`),
Status: result.Pass,
Tags: result.NewTags("x"),
},
result.Result{
Query: Q(`b`),
Status: result.Failure,
Tags: result.NewTags("y"),
},
result.Result{
Query: Q(`c`),
Status: result.Pass,
Tags: result.NewTags("x", "y"),
},
},
tags: result.NewTags("q"),
expect: result.List{},
},
} {
got := test.results.FilterByVariant(test.tags)
if diff := cmp.Diff(got, test.expect); diff != "" {
t.Errorf("Results:\n%v\nFilterByVariant(%v) was not as expected:\n%v", test.results, test.tags, diff)
}
}
}
func TestStatuses(t *testing.T) {
type Test struct {
results result.List