tools/src/cts/result: Add more helpers

Add result.List.StatusTree() for building a query.Tree[Status].
Add helpers for serializing results.
Add helpers for merging and de-duplicating results.

Change the interface of result.List.ReplaceDuplicates() so that the
merging function takes a status set instead of a list of results.

Bug: dawn:1342
Change-Id: I77580ec5fd4c8f12109fb6e9e83afea8b740260c
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/87240
Reviewed-by: Corentin Wallez <cwallez@chromium.org>
Kokoro: Kokoro <noreply+kokoro@google.com>
This commit is contained in:
Ben Clayton 2022-04-20 19:10:42 +00:00
parent 1a10b73552
commit 2363ad16ea
4 changed files with 546 additions and 28 deletions

View File

@ -16,7 +16,11 @@
package result
import (
"bufio"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
@ -126,29 +130,41 @@ func (l List) TransformTags(f func(Tags) Tags) List {
// ReplaceDuplicates returns a new list with duplicate test results replaced.
// When a duplicate is found, the function f is called with the duplicate
// results. The returned status will be used as the replaced result.
func (l List) ReplaceDuplicates(f func(List) Status) List {
func (l List) ReplaceDuplicates(f func(Statuses) Status) List {
type key struct {
query query.Query
tags string
}
m := map[key]List{}
// Collect all duplicates
duplicates := map[key]Statuses{}
for _, r := range l {
k := key{r.Query, TagsToString(r.Tags)}
m[k] = append(m[k], r)
}
for key, results := range m {
if len(results) > 1 {
result := results[0]
result.Status = f(results)
m[key] = List{result}
if s, ok := duplicates[k]; ok {
s.Add(r.Status)
} else {
duplicates[k] = NewStatuses(r.Status)
}
}
out := make(List, 0, len(m))
// Resolve duplicates
merged := map[key]Status{}
for key, statuses := range duplicates {
if len(statuses) > 1 {
merged[key] = f(statuses)
} else {
merged[key] = statuses.One() // Only one status
}
}
// Rebuild list
out := make(List, 0, len(duplicates))
for _, r := range l {
k := key{r.Query, TagsToString(r.Tags)}
if unique, ok := m[k]; ok {
out = append(out, unique[0])
delete(m, k)
if status, ok := merged[k]; ok {
out = append(out, Result{
Query: r.Query,
Tags: r.Tags,
Status: status,
})
delete(merged, k) // Remove from map to prevent duplicates
}
}
return out
@ -201,11 +217,125 @@ func (l List) FilterByTags(tags Tags) List {
})
}
// Statuses is a set of Status
type Statuses = container.Set[Status]
// NewStatuses returns a new status set with the provided statuses
func NewStatuses(s ...Status) Statuses { return container.NewSet(s...) }
// Statuses returns a set of all the statuses in the list
func (l List) Statuses() container.Set[Status] {
set := container.NewSet[Status]()
func (l List) Statuses() Statuses {
set := NewStatuses()
for _, r := range l {
set.Add(r.Status)
}
return set
}
// StatusTree is a query tree of statuses
type StatusTree = query.Tree[Status]
// StatusTree returns a query.Tree from the List, with the Status as the tree
// node data.
func (l List) StatusTree() (StatusTree, error) {
tree := StatusTree{}
for _, r := range l {
if err := tree.Add(r.Query, r.Status); err != nil {
return StatusTree{}, err
}
}
return tree, nil
}
// Load loads the result list from the file with the given path
func Load(path string) (List, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
defer file.Close()
results, err := Read(file)
if err != nil {
return nil, fmt.Errorf("while reading '%v': %w", path, err)
}
return results, nil
}
// Save saves the result list to the file with the given path
func Save(path string, results List) error {
dir := filepath.Dir(path)
if err := os.MkdirAll(dir, 0777); err != nil {
return err
}
file, err := os.Create(path)
if err != nil {
return err
}
defer file.Close()
return Write(file, results)
}
// Read reads a result list from the given reader
func Read(r io.Reader) (List, error) {
scanner := bufio.NewScanner(r)
l := List{}
for scanner.Scan() {
r, err := Parse(scanner.Text())
if err != nil {
return nil, err
}
l = append(l, r)
}
return l, nil
}
// Write writes a result list to the given writer
func Write(w io.Writer, l List) error {
for _, r := range l {
if _, err := fmt.Fprintln(w, r); err != nil {
return err
}
}
return nil
}
// Merge merges and sorts two results lists.
// Duplicates are removed using the Deduplicate() function.
func Merge(a, b List) List {
merged := make(List, 0, len(a)+len(b))
merged = append(merged, a...)
merged = append(merged, b...)
out := merged.ReplaceDuplicates(Deduplicate)
out.Sort()
return out
}
// Deduplicate is the standard algorithm used to de-duplicating mixed results.
// This function is expected to be handed to List.ReplaceDuplicates().
func Deduplicate(s Statuses) Status {
// If all results have the same status, then use that
if len(s) == 1 {
return s.One()
}
// Mixed statuses. Replace with something appropriate.
switch {
// Crash + * = Crash
case s.Contains(Crash):
return Crash
// Abort + * = Abort
case s.Contains(Abort):
return Abort
// Unknown + * = Unknown
case s.Contains(Unknown):
return Unknown
// RetryOnFailure + ~(Crash | Abort | Unknown) = RetryOnFailure
case s.Contains(RetryOnFailure):
return RetryOnFailure
// Pass + ~(Crash | Abort | Unknown | RetryOnFailure | Slow) = RetryOnFailure
case s.Contains(Pass):
return RetryOnFailure
}
return Unknown
}

View File

@ -15,12 +15,14 @@
package result_test
import (
"bytes"
"fmt"
"testing"
"dawn.googlesource.com/dawn/tools/src/container"
"dawn.googlesource.com/dawn/tools/src/cts/query"
"dawn.googlesource.com/dawn/tools/src/cts/result"
"dawn.googlesource.com/dawn/tools/src/utils"
"github.com/google/go-cmp/cmp"
)
@ -304,16 +306,18 @@ func TestTransformTags(t *testing.T) {
func TestReplaceDuplicates(t *testing.T) {
type Test struct {
results result.List
fn func(result.List) result.Status
expect result.List
location string
results result.List
fn func(result.Statuses) result.Status
expect result.List
}
for _, test := range []Test{
{ //////////////////////////////////////////////////////////////////////
location: utils.ThisLine(),
results: result.List{
result.Result{Query: Q(`a`), Status: result.Pass},
},
fn: func(l result.List) result.Status {
fn: func(result.Statuses) result.Status {
return result.Abort
},
expect: result.List{
@ -321,23 +325,25 @@ func TestReplaceDuplicates(t *testing.T) {
},
},
{ //////////////////////////////////////////////////////////////////////
location: utils.ThisLine(),
results: result.List{
result.Result{Query: Q(`a`), Status: result.Pass},
result.Result{Query: Q(`a`), Status: result.Pass},
},
fn: func(l result.List) result.Status {
fn: func(result.Statuses) result.Status {
return result.Abort
},
expect: result.List{
result.Result{Query: Q(`a`), Status: result.Abort},
result.Result{Query: Q(`a`), Status: result.Pass},
},
},
{ //////////////////////////////////////////////////////////////////////
location: utils.ThisLine(),
results: result.List{
result.Result{Query: Q(`a`), Status: result.Pass},
result.Result{Query: Q(`b`), Status: result.Pass},
},
fn: func(l result.List) result.Status {
fn: func(result.Statuses) result.Status {
return result.Abort
},
expect: result.List{
@ -346,16 +352,14 @@ func TestReplaceDuplicates(t *testing.T) {
},
},
{ //////////////////////////////////////////////////////////////////////
location: utils.ThisLine(),
results: result.List{
result.Result{Query: Q(`a`), Status: result.Pass},
result.Result{Query: Q(`b`), Status: result.Pass},
result.Result{Query: Q(`a`), Status: result.Skip},
},
fn: func(got result.List) result.Status {
expect := result.List{
result.Result{Query: Q(`a`), Status: result.Pass},
result.Result{Query: Q(`a`), Status: result.Skip},
}
fn: func(got result.Statuses) result.Status {
expect := result.NewStatuses(result.Pass, result.Skip)
if diff := cmp.Diff(got, expect); diff != "" {
t.Errorf("function's parameter was not as expected:\n%v", diff)
}
@ -369,7 +373,7 @@ func TestReplaceDuplicates(t *testing.T) {
} {
got := test.results.ReplaceDuplicates(test.fn)
if diff := cmp.Diff(got, test.expect); diff != "" {
t.Errorf("Results:\n%v\nReplaceDuplicates() was not as expected:\n%v", test.results, diff)
t.Errorf("\n%v ReplaceDuplicates() was not as expected:\n%v", test.location, diff)
}
}
}
@ -847,3 +851,322 @@ func TestStatuses(t *testing.T) {
}
}
}
func TestStatusTree(t *testing.T) {
type Node = query.TreeNode[result.Status]
type Children = query.TreeNodeChildren[result.Status]
type ChildKey = query.TreeNodeChildKey
pass := result.Pass
type Test struct {
results result.List
expectErr error
expect result.StatusTree
}
for _, test := range []Test{
{ //////////////////////////////////////////////////////////////////////
results: result.List{},
expect: result.StatusTree{},
},
{ //////////////////////////////////////////////////////////////////////
results: result.List{
{Query: Q(`suite:a:*`), Status: result.Pass},
},
expect: result.StatusTree{
TreeNode: Node{
Children: Children{
ChildKey{Name: `suite`, Target: query.Suite}: &Node{
Query: Q(`suite`),
Children: Children{
ChildKey{Name: `a`, Target: query.Files}: &Node{
Query: Q(`suite:a`),
Children: Children{
ChildKey{Name: `*`, Target: query.Tests}: &Node{
Query: Q(`suite:a:*`),
Data: &pass,
},
},
},
},
},
},
},
},
},
{ //////////////////////////////////////////////////////////////////////
results: result.List{
{Query: Q(`suite:a:*`), Status: result.Pass},
{Query: Q(`suite:a:*`), Status: result.Failure},
},
expectErr: query.ErrDuplicateData{Query: Q(`suite:a:*`)},
},
} {
got, err := test.results.StatusTree()
if diff := cmp.Diff(err, test.expectErr); diff != "" {
t.Errorf("Results:\n%v\nStatusTree() error was not as expected:\n%v", test.results, diff)
continue
}
if diff := cmp.Diff(got, test.expect); diff != "" {
t.Errorf("Results:\n%v\nStatusTree() was not as expected:\n%v", test.results, diff)
}
}
}
func TestReadWrite(t *testing.T) {
in := result.List{
{Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Pass},
{Query: Q(`suite:b,*`), Tags: T(`y`), Status: result.Failure},
{Query: Q(`suite:a:b:*`), Tags: T(`x`, `y`), Status: result.Skip},
{Query: Q(`suite:a:c,*`), Tags: T(`y`, `x`), Status: result.Failure},
{Query: Q(`suite:a,b:c,*`), Tags: T(`y`, `x`), Status: result.Crash},
{Query: Q(`suite:a,b:c:*`), Status: result.Slow},
}
buf := &bytes.Buffer{}
if err := result.Write(buf, in); err != nil {
t.Fatalf("Write(): %v", err)
}
got, err := result.Read(buf)
if err != nil {
t.Fatalf("Read(): %v", err)
}
if diff := cmp.Diff(got, in); diff != "" {
t.Errorf("Read() was not as expected:\n%v", diff)
}
}
func TestMerge(t *testing.T) {
type Test struct {
location string
a, b result.List
expect result.List
}
for _, test := range []Test{
{ //////////////////////////////////////////////////////////////////////
location: utils.ThisLine(),
a: result.List{},
b: result.List{},
expect: result.List{},
},
{ //////////////////////////////////////////////////////////////////////
location: utils.ThisLine(),
a: result.List{
{Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Pass},
},
b: result.List{},
expect: result.List{
{Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Pass},
},
},
{ //////////////////////////////////////////////////////////////////////
location: utils.ThisLine(),
a: result.List{},
b: result.List{
{Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Pass},
},
expect: result.List{
{Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Pass},
},
},
{ //////////////////////////////////////////////////////////////////////
location: utils.ThisLine(),
a: result.List{
{Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Pass},
},
b: result.List{
{Query: Q(`suite:b:*`), Tags: T(`x`), Status: result.Pass},
},
expect: result.List{
{Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Pass},
{Query: Q(`suite:b:*`), Tags: T(`x`), Status: result.Pass},
},
},
{ //////////////////////////////////////////////////////////////////////
location: utils.ThisLine(),
a: result.List{
{Query: Q(`suite:b:*`), Tags: T(`x`), Status: result.Pass},
},
b: result.List{
{Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Pass},
},
expect: result.List{
{Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Pass},
{Query: Q(`suite:b:*`), Tags: T(`x`), Status: result.Pass},
},
},
{ //////////////////////////////////////////////////////////////////////
location: utils.ThisLine(),
a: result.List{
{Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Pass},
},
b: result.List{
{Query: Q(`suite:a:*`), Tags: T(`y`), Status: result.Pass},
},
expect: result.List{
{Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Pass},
{Query: Q(`suite:a:*`), Tags: T(`y`), Status: result.Pass},
},
},
{ //////////////////////////////////////////////////////////////////////
location: utils.ThisLine(),
a: result.List{
{Query: Q(`suite:a:*`), Status: result.Pass},
},
b: result.List{
{Query: Q(`suite:a:*`), Status: result.Pass},
},
expect: result.List{
{Query: Q(`suite:a:*`), Status: result.Pass},
},
},
{ //////////////////////////////////////////////////////////////////////
location: utils.ThisLine(),
a: result.List{
{Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Pass},
},
b: result.List{
{Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Pass},
},
expect: result.List{
{Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Pass},
},
},
{ //////////////////////////////////////////////////////////////////////
location: utils.ThisLine(),
a: result.List{
{Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Crash},
},
b: result.List{
{Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Crash},
},
expect: result.List{
{Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Crash},
},
},
{ //////////////////////////////////////////////////////////////////////
location: utils.ThisLine(),
a: result.List{
{Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Pass},
{Query: Q(`suite:b:*`), Tags: T(`x`), Status: result.Pass},
{Query: Q(`suite:c:*`), Tags: T(`x`), Status: result.Failure},
{Query: Q(`suite:d:*`), Tags: T(`x`), Status: result.Failure},
{Query: Q(`suite:e:*`), Tags: T(`x`), Status: result.Crash},
},
b: result.List{
{Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Failure},
{Query: Q(`suite:b:*`), Tags: T(`x`), Status: result.Pass},
{Query: Q(`suite:c:*`), Tags: T(`x`), Status: result.Pass},
{Query: Q(`suite:d:*`), Tags: T(`y`), Status: result.Pass},
{Query: Q(`suite:e:*`), Tags: T(`x`), Status: result.Pass},
},
expect: result.List{
{Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.RetryOnFailure},
{Query: Q(`suite:b:*`), Tags: T(`x`), Status: result.Pass},
{Query: Q(`suite:c:*`), Tags: T(`x`), Status: result.RetryOnFailure},
{Query: Q(`suite:d:*`), Tags: T(`x`), Status: result.Failure},
{Query: Q(`suite:d:*`), Tags: T(`y`), Status: result.Pass},
{Query: Q(`suite:e:*`), Tags: T(`x`), Status: result.Crash},
},
},
} {
got := result.Merge(test.a, test.b)
if diff := cmp.Diff(got, test.expect); diff != "" {
t.Errorf("%v\nStatusTree() was not as expected:\n%v", test.location, diff)
}
}
}
func TestDeduplicate(t *testing.T) {
type Test struct {
location string
statuses result.Statuses
expect result.Status
}
for _, test := range []Test{
{ //////////////////////////////////////////////////////////////////////
location: utils.ThisLine(),
statuses: result.NewStatuses(result.Pass),
expect: result.Pass,
},
{ //////////////////////////////////////////////////////////////////////
location: utils.ThisLine(),
statuses: result.NewStatuses(result.Abort),
expect: result.Abort,
},
{ //////////////////////////////////////////////////////////////////////
location: utils.ThisLine(),
statuses: result.NewStatuses(result.Failure),
expect: result.Failure,
},
{ //////////////////////////////////////////////////////////////////////
location: utils.ThisLine(),
statuses: result.NewStatuses(result.Skip),
expect: result.Skip,
},
{ //////////////////////////////////////////////////////////////////////
location: utils.ThisLine(),
statuses: result.NewStatuses(result.Crash),
expect: result.Crash,
},
{ //////////////////////////////////////////////////////////////////////
location: utils.ThisLine(),
statuses: result.NewStatuses(result.Slow),
expect: result.Slow,
},
{ //////////////////////////////////////////////////////////////////////
location: utils.ThisLine(),
statuses: result.NewStatuses(result.Unknown),
expect: result.Unknown,
},
{ //////////////////////////////////////////////////////////////////////
location: utils.ThisLine(),
statuses: result.NewStatuses(result.RetryOnFailure),
expect: result.RetryOnFailure,
},
{ //////////////////////////////////////////////////////////////////////
location: utils.ThisLine(),
statuses: result.NewStatuses(result.Pass, result.Failure),
expect: result.RetryOnFailure,
},
{ //////////////////////////////////////////////////////////////////////
location: utils.ThisLine(),
statuses: result.NewStatuses(result.Pass, result.Abort),
expect: result.Abort,
},
{ //////////////////////////////////////////////////////////////////////
location: utils.ThisLine(),
statuses: result.NewStatuses(result.Pass, result.Skip),
expect: result.RetryOnFailure,
},
{ //////////////////////////////////////////////////////////////////////
location: utils.ThisLine(),
statuses: result.NewStatuses(result.Pass, result.Crash),
expect: result.Crash,
},
{ //////////////////////////////////////////////////////////////////////
location: utils.ThisLine(),
statuses: result.NewStatuses(result.Pass, result.Slow),
expect: result.RetryOnFailure,
},
{ //////////////////////////////////////////////////////////////////////
location: utils.ThisLine(),
statuses: result.NewStatuses(result.Pass, result.Unknown),
expect: result.Unknown,
},
{ //////////////////////////////////////////////////////////////////////
location: utils.ThisLine(),
statuses: result.NewStatuses(result.Pass, result.RetryOnFailure),
expect: result.RetryOnFailure,
},
{ //////////////////////////////////////////////////////////////////////
location: utils.ThisLine(),
statuses: result.NewStatuses(result.Status("??"), result.Status("?!")),
expect: result.Unknown,
},
} {
got := result.Deduplicate(test.statuses)
if diff := cmp.Diff(got, test.expect); diff != "" {
t.Errorf("\n%v Deduplicate() was not as expected:\n%v", test.location, diff)
}
}
}

View File

@ -14,6 +14,8 @@
package result
import "dawn.googlesource.com/dawn/tools/src/container"
// Status is an enumerator of test results
type Status string
@ -28,3 +30,12 @@ const (
Slow = Status("Slow")
Unknown = Status("Unknown")
)
// CommonStatus is a function that can be used by StatusTree.Reduce() to reduce
// tree nodes with the same status
func CommonStatus(statuses []Status) *Status {
if set := container.NewSet(statuses...); len(set) == 1 {
return &statuses[0]
}
return nil
}

View File

@ -0,0 +1,54 @@
// Copyright 2022 The Dawn Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package result_test
import (
"testing"
"dawn.googlesource.com/dawn/tools/src/cts/result"
"github.com/google/go-cmp/cmp"
)
func TestCommonStatus(t *testing.T) {
pass := result.Pass
type Test struct {
in []result.Status
expect *result.Status
}
for _, test := range []Test{
{
in: nil,
expect: nil,
}, {
in: []result.Status{},
expect: nil,
}, {
in: []result.Status{result.Pass},
expect: &pass,
}, {
in: []result.Status{result.Pass, result.Pass, result.Pass},
expect: &pass,
}, {
in: []result.Status{result.Pass, result.Failure, result.Pass},
expect: nil,
},
} {
got := result.CommonStatus(test.in)
if diff := cmp.Diff(got, test.expect); diff != "" {
t.Errorf("%v.CommonStatus('%v') was not as expected:\n%v", test.in, test.expect, diff)
}
}
}