diff --git a/tools/src/cts/result/result.go b/tools/src/cts/result/result.go new file mode 100644 index 0000000000..2d4709458a --- /dev/null +++ b/tools/src/cts/result/result.go @@ -0,0 +1,211 @@ +// Copyright 2022 The Dawn Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package result holds types that describe CTS test results. +package result + +import ( + "fmt" + "sort" + "strings" + + "dawn.googlesource.com/dawn/tools/src/container" + "dawn.googlesource.com/dawn/tools/src/cts/query" +) + +// Result holds the result of a CTS test +type Result struct { + Query query.Query + Tags Tags + Status Status +} + +// Format writes the Result to the fmt.State +// The Result is printed as a single line, in the form: +// +// This matches the order in which results are sorted. +func (r Result) Format(f fmt.State, verb rune) { + if len(r.Tags) > 0 { + fmt.Fprintf(f, "%v %v %v", r.Query, TagsToString(r.Tags), r.Status) + } else { + fmt.Fprintf(f, "%v %v", r.Query, r.Status) + } +} + +// String returns the result as a string +func (r Result) String() string { + sb := strings.Builder{} + fmt.Fprint(&sb, r) + return sb.String() +} + +// Parse parses the result from a string of the form: +// +// may be omitted if there were no tags. +func Parse(in string) (Result, error) { + line := in + token := func() string { + for i, c := range line { + if c != ' ' { + line = line[i:] + break + } + } + for i, c := range line { + if c == ' ' { + tok := line[:i] + line = line[i:] + return tok + } + } + tok := line + line = "" + return tok + } + + a := token() + b := token() + c := token() + if a == "" || b == "" || token() != "" { + return Result{}, fmt.Errorf("unable to parse result '%v'", in) + } + q := query.Parse(a) + if c == "" { + status := Status(b) + return Result{q, nil, status}, nil + } + tags := StringToTags(b) + status := Status(c) + return Result{q, tags, status}, nil +} + +// List is a list of results +type List []Result + +// Returns the list of unique tags across all results. +func (l List) UniqueTags() []Tags { + tags := container.NewMap[string, Tags]() + for _, r := range l { + tags.Add(TagsToString(r.Tags), r.Tags) + } + return tags.Values() +} + +// TransformTags returns the list of results with the tags transformed using f. +// TransformTags assumes that f will return the same output for the same input. +func (l List) TransformTags(f func(Tags) Tags) List { + cache := map[string]Tags{} + out := List{} + for _, r := range l { + key := TagsToString(r.Tags) + tags, cached := cache[key] + if !cached { + tags = f(r.Tags.Clone()) + cache[key] = tags + } + out = append(out, Result{ + Query: r.Query, + Tags: tags, + Status: r.Status, + }) + } + return out +} + +// ReplaceDuplicates returns a new list with duplicate test results replaced. +// When a duplicate is found, the function f is called with the duplicate +// results. The returned status will be used as the replaced result. +func (l List) ReplaceDuplicates(f func(List) Status) List { + type key struct { + query query.Query + tags string + } + m := map[key]List{} + for _, r := range l { + k := key{r.Query, TagsToString(r.Tags)} + m[k] = append(m[k], r) + } + for key, results := range m { + if len(results) > 1 { + result := results[0] + result.Status = f(results) + m[key] = List{result} + } + } + out := make(List, 0, len(m)) + for _, r := range l { + k := key{r.Query, TagsToString(r.Tags)} + if unique, ok := m[k]; ok { + out = append(out, unique[0]) + delete(m, k) + } + } + return out +} + +// Sort sorts the list +func (l List) Sort() { + sort.Slice(l, func(i, j int) bool { + a, b := l[i], l[j] + switch a.Query.Compare(b.Query) { + case -1: + return true + case 1: + return false + } + ta := strings.Join(a.Tags.List(), TagDelimiter) + tb := strings.Join(b.Tags.List(), TagDelimiter) + switch { + case ta < tb: + return true + case ta > tb: + return false + } + return a.Status < b.Status + }) +} + +// Filter returns the results that match the given predicate +func (l List) Filter(f func(Result) bool) List { + out := make(List, 0, len(l)) + for _, r := range l { + if f(r) { + out = append(out, r) + } + } + return out +} + +// FilterByStatus returns the results that the given status +func (l List) FilterByStatus(status Status) List { + return l.Filter(func(r Result) bool { + return r.Status == status + }) +} + +// FilterByTags returns the results that have all the given tags +func (l List) FilterByTags(tags Tags) List { + return l.Filter(func(r Result) bool { + return r.Tags.ContainsAll(tags) + }) +} + +// Statuses returns a set of all the statuses in the list +func (l List) Statuses() container.Set[Status] { + set := container.NewSet[Status]() + for _, r := range l { + set.Add(r.Status) + } + return set +} diff --git a/tools/src/cts/result/result_test.go b/tools/src/cts/result/result_test.go new file mode 100644 index 0000000000..fd76ccc3cc --- /dev/null +++ b/tools/src/cts/result/result_test.go @@ -0,0 +1,849 @@ +// Copyright 2022 The Dawn Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package result_test + +import ( + "fmt" + "testing" + + "dawn.googlesource.com/dawn/tools/src/container" + "dawn.googlesource.com/dawn/tools/src/cts/query" + "dawn.googlesource.com/dawn/tools/src/cts/result" + "github.com/google/go-cmp/cmp" +) + +var Q = query.Parse + +func T(tags ...string) result.Tags { + return result.NewTags(tags...) +} + +func TestStringAndParse(t *testing.T) { + type Test struct { + result result.Result + expect string + } + for _, test := range []Test{ + { + result.Result{ + Query: Q(`a`), + Status: result.Failure, + }, + `a Failure`, + }, { + result.Result{ + Query: Q(`a:b,c,*`), + Tags: T("x"), + Status: result.Pass, + }, + `a:b,c,* x Pass`, + }, + { + result.Result{ + Query: Q(`a:b,c:d,*`), + Tags: T("zzz", "x", "yy"), + Status: result.Failure, + }, + `a:b,c:d,* x,yy,zzz Failure`, + }, + } { + if diff := cmp.Diff(test.result.String(), test.expect); diff != "" { + t.Errorf("'%v'.String() was not as expected:\n%v", test.result, diff) + continue + } + parsed, err := result.Parse(test.expect) + if err != nil { + t.Errorf("Parse('%v') returned %v", test.expect, err) + continue + } + if diff := cmp.Diff(parsed, test.result); diff != "" { + t.Errorf("Parse('%v') was not as expected:\n%v", test.expect, diff) + } + } +} + +func TestParseError(t *testing.T) { + for _, test := range []string{ + ``, + `a`, + `a b c d`, + } { + _, err := result.Parse(test) + expect := fmt.Sprintf(`unable to parse result '%v'`, test) + if err == nil || err.Error() != expect { + t.Errorf("Parse('%v') returned '%v'", test, err) + continue + } + } +} + +func TestUniqueTags(t *testing.T) { + type Test struct { + results result.List + expect []result.Tags + } + for _, test := range []Test{ + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{ + Query: Q(`a`), + Status: result.Pass, + Tags: result.NewTags(), + }, + }, + expect: []result.Tags{ + result.NewTags(), + }, + }, + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{ + Query: Q(`a`), + Status: result.Pass, + Tags: result.NewTags("x"), + }, + }, + expect: []result.Tags{ + result.NewTags("x"), + }, + }, + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{ + Query: Q(`a`), + Status: result.Pass, + Tags: result.NewTags("x", "y"), + }, + }, + expect: []result.Tags{ + result.NewTags("x", "y"), + }, + }, + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{ + Query: Q(`a`), + Status: result.Pass, + Tags: result.NewTags("x", "y"), + }, + result.Result{ + Query: Q(`b`), + Status: result.Pass, + Tags: result.NewTags("x", "y"), + }, + }, + expect: []result.Tags{ + result.NewTags("x", "y"), + }, + }, + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{ + Query: Q(`a`), + Status: result.Pass, + Tags: result.NewTags("x", "y"), + }, + result.Result{ + Query: Q(`b`), + Status: result.Pass, + Tags: result.NewTags("y", "z"), + }, + }, + expect: []result.Tags{ + result.NewTags("x", "y"), + result.NewTags("y", "z"), + }, + }, + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{ + Query: Q(`a`), + Status: result.Pass, + Tags: result.NewTags("x", "y"), + }, + result.Result{ + Query: Q(`b`), + Status: result.Pass, + Tags: result.NewTags("y", "z"), + }, + result.Result{ + Query: Q(`c`), + Status: result.Pass, + Tags: result.NewTags("z", "x"), + }, + result.Result{ + Query: Q(`d`), + Status: result.Pass, + Tags: result.NewTags("y", "z"), + }, + }, + expect: []result.Tags{ + result.NewTags("x", "y"), + result.NewTags("x", "z"), + result.NewTags("y", "z"), + }, + }, + } { + got := test.results.UniqueTags() + if diff := cmp.Diff(got, test.expect); diff != "" { + t.Errorf("Results:\n%v\nUniqueTags() was not as expected:\n%v", test.results, diff) + } + } +} + +func TestTransformTags(t *testing.T) { + type Test struct { + results result.List + transform func(result.Tags) result.Tags + expect result.List + } + for _, test := range []Test{ + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{ + Query: Q(`a`), + Status: result.Pass, + Tags: result.NewTags(), + }, + }, + transform: func(t result.Tags) result.Tags { return t }, + expect: result.List{ + result.Result{ + Query: Q(`a`), + Status: result.Pass, + Tags: result.NewTags(), + }, + }, + }, + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{ + Query: Q(`a`), + Status: result.Pass, + Tags: result.NewTags("x"), + }, + }, + transform: func(got result.Tags) result.Tags { + expect := result.NewTags("x") + if diff := cmp.Diff(got, expect); diff != "" { + t.Errorf("transform function's parameter was not as expected:\n%v", diff) + } + return got + }, + expect: result.List{ + result.Result{ + Query: Q(`a`), + Status: result.Pass, + Tags: result.NewTags("x"), + }, + }, + }, + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{ + Query: Q(`a`), + Status: result.Pass, + Tags: result.NewTags("x", "y"), + }, + result.Result{ + Query: Q(`b`), + Status: result.Pass, + Tags: result.NewTags("y", "z"), + }, + result.Result{ + Query: Q(`c`), + Status: result.Pass, + Tags: result.NewTags("z", "x"), + }, + }, + transform: func(l result.Tags) result.Tags { + l = l.Clone() + if l.Contains("x") { + l.Remove("x") + l.Add("X") + } + return l + }, + expect: result.List{ + result.Result{ + Query: Q(`a`), + Status: result.Pass, + Tags: result.NewTags("X", "y"), + }, + result.Result{ + Query: Q(`b`), + Status: result.Pass, + Tags: result.NewTags("y", "z"), + }, + result.Result{ + Query: Q(`c`), + Status: result.Pass, + Tags: result.NewTags("z", "X"), + }, + }, + }, + } { + got := test.results.TransformTags(test.transform) + if diff := cmp.Diff(got, test.expect); diff != "" { + t.Errorf("Results:\n%v\nTransformTags() was not as expected:\n%v", test.results, diff) + } + } +} + +func TestReplaceDuplicates(t *testing.T) { + type Test struct { + results result.List + fn func(result.List) result.Status + expect result.List + } + for _, test := range []Test{ + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{Query: Q(`a`), Status: result.Pass}, + }, + fn: func(l result.List) result.Status { + return result.Abort + }, + expect: result.List{ + result.Result{Query: Q(`a`), Status: result.Pass}, + }, + }, + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{Query: Q(`a`), Status: result.Pass}, + result.Result{Query: Q(`a`), Status: result.Pass}, + }, + fn: func(l result.List) result.Status { + return result.Abort + }, + expect: result.List{ + result.Result{Query: Q(`a`), Status: result.Abort}, + }, + }, + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{Query: Q(`a`), Status: result.Pass}, + result.Result{Query: Q(`b`), Status: result.Pass}, + }, + fn: func(l result.List) result.Status { + return result.Abort + }, + expect: result.List{ + result.Result{Query: Q(`a`), Status: result.Pass}, + result.Result{Query: Q(`b`), Status: result.Pass}, + }, + }, + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{Query: Q(`a`), Status: result.Pass}, + result.Result{Query: Q(`b`), Status: result.Pass}, + result.Result{Query: Q(`a`), Status: result.Skip}, + }, + fn: func(got result.List) result.Status { + expect := result.List{ + result.Result{Query: Q(`a`), Status: result.Pass}, + result.Result{Query: Q(`a`), Status: result.Skip}, + } + if diff := cmp.Diff(got, expect); diff != "" { + t.Errorf("function's parameter was not as expected:\n%v", diff) + } + return result.Abort + }, + expect: result.List{ + result.Result{Query: Q(`a`), Status: result.Abort}, + result.Result{Query: Q(`b`), Status: result.Pass}, + }, + }, + } { + got := test.results.ReplaceDuplicates(test.fn) + if diff := cmp.Diff(got, test.expect); diff != "" { + t.Errorf("Results:\n%v\nReplaceDuplicates() was not as expected:\n%v", test.results, diff) + } + } +} + +func TestSort(t *testing.T) { + type Test struct { + results result.List + expect result.List + } + for _, test := range []Test{ + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{Query: Q(`a`), Status: result.Pass}, + }, + expect: result.List{ + result.Result{Query: Q(`a`), Status: result.Pass}, + }, + }, + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{Query: Q(`a`), Status: result.Pass}, + result.Result{Query: Q(`b`), Status: result.Pass}, + }, + expect: result.List{ + result.Result{Query: Q(`a`), Status: result.Pass}, + result.Result{Query: Q(`b`), Status: result.Pass}, + }, + }, + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{Query: Q(`b`), Status: result.Pass}, + result.Result{Query: Q(`a`), Status: result.Pass}, + }, + expect: result.List{ + result.Result{Query: Q(`a`), Status: result.Pass}, + result.Result{Query: Q(`b`), Status: result.Pass}, + }, + }, + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{Query: Q(`a`), Status: result.Pass}, + result.Result{Query: Q(`a`), Status: result.Skip}, + }, + expect: result.List{ + result.Result{Query: Q(`a`), Status: result.Pass}, + result.Result{Query: Q(`a`), Status: result.Skip}, + }, + }, + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{ + Query: Q(`a`), + Status: result.Skip, + Tags: result.NewTags(), + }, + result.Result{ + Query: Q(`a`), + Status: result.Pass, + Tags: result.NewTags(), + }, + }, + expect: result.List{ + result.Result{ + Query: Q(`a`), + Status: result.Pass, + Tags: result.NewTags(), + }, + result.Result{ + Query: Q(`a`), + Status: result.Skip, + Tags: result.NewTags(), + }, + }, + }, + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{ + Query: Q(`a`), + Status: result.Pass, + Tags: result.NewTags("a"), + }, + result.Result{ + Query: Q(`a`), + Status: result.Pass, + Tags: result.NewTags("b"), + }, + }, + expect: result.List{ + result.Result{ + Query: Q(`a`), + Status: result.Pass, + Tags: result.NewTags("a"), + }, + result.Result{ + Query: Q(`a`), + Status: result.Pass, + Tags: result.NewTags("b"), + }, + }, + }, + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{ + Query: Q(`a`), + Status: result.Pass, + Tags: result.NewTags("b"), + }, + result.Result{ + Query: Q(`a`), + Status: result.Pass, + Tags: result.NewTags("a"), + }, + }, + expect: result.List{ + result.Result{ + Query: Q(`a`), + Status: result.Pass, + Tags: result.NewTags("a"), + }, + result.Result{ + Query: Q(`a`), + Status: result.Pass, + Tags: result.NewTags("b"), + }, + }, + }, + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{ + Query: Q(`c`), + Status: result.RetryOnFailure, + Tags: result.NewTags("z"), + }, + result.Result{ + Query: Q(`a`), + Status: result.Crash, + Tags: result.NewTags("y"), + }, + result.Result{ + Query: Q(`b`), + Status: result.Slow, + Tags: result.NewTags("y"), + }, + result.Result{ + Query: Q(`b`), + Status: result.Skip, + Tags: result.NewTags("y"), + }, + result.Result{ + Query: Q(`a`), + Status: result.Crash, + Tags: result.NewTags("x"), + }, + result.Result{ + Query: Q(`aa`), + Status: result.Crash, + Tags: result.NewTags("x"), + }, + result.Result{ + Query: Q(`c`), + Status: result.Abort, + Tags: result.NewTags("z"), + }, + result.Result{ + Query: Q(`a`), + Status: result.Pass, + Tags: result.NewTags("x"), + }, + }, + expect: result.List{ + result.Result{ + Query: Q(`a`), + Status: result.Crash, + Tags: result.NewTags("x"), + }, + result.Result{ + Query: Q(`a`), + Status: result.Pass, + Tags: result.NewTags("x"), + }, + result.Result{ + Query: Q(`a`), + Status: result.Crash, + Tags: result.NewTags("y"), + }, + result.Result{ + Query: Q(`aa`), + Status: result.Crash, + Tags: result.NewTags("x"), + }, + result.Result{ + Query: Q(`b`), + Status: result.Skip, + Tags: result.NewTags("y"), + }, + result.Result{ + Query: Q(`b`), + Status: result.Slow, + Tags: result.NewTags("y"), + }, + result.Result{ + Query: Q(`c`), + Status: result.Abort, + Tags: result.NewTags("z"), + }, + result.Result{ + Query: Q(`c`), + Status: result.RetryOnFailure, + Tags: result.NewTags("z"), + }, + }, + }, + } { + got := make(result.List, len(test.results)) + copy(got, test.results) + got.Sort() + if diff := cmp.Diff(got, test.expect); diff != "" { + t.Errorf("Results:\n%v\nSort() was not as expected:\n%v", test.results, diff) + } + } +} + +func TestFilter(t *testing.T) { + type Test struct { + results result.List + f func(result.Result) bool + expect result.List + } + for _, test := range []Test{ + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{Query: Q(`a`), Status: result.Pass}, + }, + f: func(result.Result) bool { return true }, + expect: result.List{ + result.Result{Query: Q(`a`), Status: result.Pass}, + }, + }, + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{Query: Q(`a`), Status: result.Pass}, + result.Result{Query: Q(`b`), Status: result.Failure}, + }, + f: func(r result.Result) bool { return r.Query == Q("b") }, + expect: result.List{ + result.Result{Query: Q(`b`), Status: result.Failure}, + }, + }, + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{Query: Q(`a`), Status: result.Pass}, + result.Result{Query: Q(`b`), Status: result.Skip}, + result.Result{Query: Q(`c`), Status: result.Pass}, + }, + f: func(r result.Result) bool { return r.Status == result.Pass }, + expect: result.List{ + result.Result{Query: Q(`a`), Status: result.Pass}, + result.Result{Query: Q(`c`), Status: result.Pass}, + }, + }, + } { + got := test.results.Filter(test.f) + if diff := cmp.Diff(got, test.expect); diff != "" { + t.Errorf("Results:\n%v\nFilter() was not as expected:\n%v", test.results, diff) + } + } +} + +func TestFilterByStatus(t *testing.T) { + type Test struct { + results result.List + status result.Status + expect result.List + } + for _, test := range []Test{ + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{Query: Q(`a`), Status: result.Pass}, + result.Result{Query: Q(`b`), Status: result.Failure}, + result.Result{Query: Q(`c`), Status: result.Pass}, + }, + status: result.Pass, + expect: result.List{ + result.Result{Query: Q(`a`), Status: result.Pass}, + result.Result{Query: Q(`c`), Status: result.Pass}, + }, + }, + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{Query: Q(`a`), Status: result.Pass}, + result.Result{Query: Q(`b`), Status: result.Failure}, + }, + status: result.Failure, + expect: result.List{ + result.Result{Query: Q(`b`), Status: result.Failure}, + }, + }, + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{Query: Q(`a`), Status: result.Pass}, + result.Result{Query: Q(`b`), Status: result.Failure}, + }, + status: result.RetryOnFailure, + expect: result.List{}, + }, + } { + got := test.results.FilterByStatus(test.status) + if diff := cmp.Diff(got, test.expect); diff != "" { + t.Errorf("Results:\n%v\nFilterByStatus(%v) was not as expected:\n%v", test.results, test.status, diff) + } + } +} + +func TestFilterByTags(t *testing.T) { + type Test struct { + results result.List + tags result.Tags + expect result.List + } + for _, test := range []Test{ + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{ + Query: Q(`a`), + Status: result.Pass, + Tags: result.NewTags("x"), + }, + result.Result{ + Query: Q(`b`), + Status: result.Failure, + Tags: result.NewTags("y"), + }, + result.Result{ + Query: Q(`c`), + Status: result.Pass, + Tags: result.NewTags("x", "y"), + }, + }, + tags: result.NewTags("x", "y"), + expect: result.List{ + result.Result{ + Query: Q(`c`), + Status: result.Pass, + Tags: result.NewTags("x", "y"), + }, + }, + }, + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{ + Query: Q(`a`), + Status: result.Pass, + Tags: result.NewTags("x"), + }, + result.Result{ + Query: Q(`b`), + Status: result.Failure, + Tags: result.NewTags("y"), + }, + result.Result{ + Query: Q(`c`), + Status: result.Pass, + Tags: result.NewTags("x", "y"), + }, + }, + tags: result.NewTags("x"), + expect: result.List{ + result.Result{ + Query: Q(`a`), + Status: result.Pass, + Tags: result.NewTags("x"), + }, + result.Result{ + Query: Q(`c`), + Status: result.Pass, + Tags: result.NewTags("x", "y"), + }, + }, + }, + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{ + Query: Q(`a`), + Status: result.Pass, + Tags: result.NewTags("x"), + }, + result.Result{ + Query: Q(`b`), + Status: result.Failure, + Tags: result.NewTags("y"), + }, + result.Result{ + Query: Q(`c`), + Status: result.Pass, + Tags: result.NewTags("x", "y"), + }, + }, + tags: result.NewTags("q"), + expect: result.List{}, + }, + } { + got := test.results.FilterByTags(test.tags) + if diff := cmp.Diff(got, test.expect); diff != "" { + t.Errorf("Results:\n%v\nFilterByTags(%v) was not as expected:\n%v", test.results, test.tags, diff) + } + } +} + +func TestStatuses(t *testing.T) { + type Test struct { + results result.List + expect container.Set[result.Status] + } + for _, test := range []Test{ + { ////////////////////////////////////////////////////////////////////// + results: result.List{}, + expect: container.NewSet[result.Status](), + }, + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{ + Query: Q(`a`), + Status: result.Pass, + }, + }, + expect: container.NewSet(result.Pass), + }, + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{ + Query: Q(`a`), + Status: result.Pass, + }, + result.Result{ + Query: Q(`b`), + Status: result.Pass, + }, + }, + expect: container.NewSet(result.Pass), + }, + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{ + Query: Q(`a`), + Status: result.Pass, + }, + result.Result{ + Query: Q(`b`), + Status: result.Skip, + }, + }, + expect: container.NewSet(result.Pass, result.Skip), + }, + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + result.Result{ + Query: Q(`a`), + Status: result.Pass, + }, + result.Result{ + Query: Q(`b`), + Status: result.Skip, + }, + result.Result{ + Query: Q(`c`), + Status: result.Failure, + }, + }, + expect: container.NewSet(result.Pass, result.Skip, result.Failure), + }, + } { + got := test.results.Statuses() + if diff := cmp.Diff(got, test.expect); diff != "" { + t.Errorf("Results:\n%v\nStatuses() was not as expected:\n%v", test.results, diff) + } + } +} diff --git a/tools/src/cts/result/status.go b/tools/src/cts/result/status.go new file mode 100644 index 0000000000..5e5fd157be --- /dev/null +++ b/tools/src/cts/result/status.go @@ -0,0 +1,30 @@ +// Copyright 2022 The Dawn Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package result + +// Status is an enumerator of test results +type Status string + +// Enumerator values for Status +const ( + Abort = Status("Abort") + Crash = Status("Crash") + Failure = Status("Failure") + Pass = Status("Pass") + RetryOnFailure = Status("RetryOnFailure") + Skip = Status("Skip") + Slow = Status("Slow") + Unknown = Status("Unknown") +) diff --git a/tools/src/cts/result/tags.go b/tools/src/cts/result/tags.go new file mode 100644 index 0000000000..ec86d8d261 --- /dev/null +++ b/tools/src/cts/result/tags.go @@ -0,0 +1,43 @@ +// Copyright 2022 The Dawn Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package result + +import ( + "strings" + + "dawn.googlesource.com/dawn/tools/src/container" +) + +// Tags is a collection of strings used to annotate test results with the test +// configuration. +type Tags = container.Set[string] + +// Returns a new tag set with the given tags +func NewTags(tags ...string) Tags { + return Tags(container.NewSet(tags...)) +} + +// TagsToString returns the tags sorted and joined using the TagDelimiter +func TagsToString(t Tags) string { + return strings.Join(t.List(), TagDelimiter) +} + +// StringToTags returns the tags sorted and joined using the TagDelimiter +func StringToTags(s string) Tags { + return NewTags(strings.Split(s, TagDelimiter)...) +} + +// The delimiter used to separate tags when stored as a string +const TagDelimiter = ","