CTS: Handle may_exonerate tag

may_exonerate indicates that a test failed for a known issue that
we could exonerate. Merging of test results now removes results
with may_exonerate unless all of them were tagged as such. So, if
for example, a test fails for a known timeout issue, but has a
subsequent pass, the timeout will be ignored.

This serves to reduce the impact of known, hard-to-fix issues and
allow the CTS roller to make progress with less noise.

Change-Id: I5103a666496398a17b3aa6ccf3f267421e40ba97
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/101804
Reviewed-by: Ben Clayton <bclayton@google.com>
Kokoro: Kokoro <noreply+kokoro@google.com>
Commit-Queue: Austin Eng <enga@chromium.org>

tools/src/cmd/cts/common/results.go

@@ -219,6 +219,7 @@ func GetResults(
 		tags := result.NewTags()
 
 		duration := rpb.GetDuration().AsDuration()
+		mayExonerate := false
 
 		for _, sp := range rpb.Tags {
 			if sp.Key == "typ_tag" {
@@ -230,6 +231,12 @@ func GetResults(
 					return err
 				}
 			}
+			if sp.Key == "may_exonerate" {
+				var err error
+				if mayExonerate, err = strconv.ParseBool(sp.Value); err != nil {
+					return err
+				}
+			}
 		}
 
 		if status == result.Pass && duration > cfg.Test.SlowThreshold {
@@ -237,10 +244,11 @@ func GetResults(
 		}
 
 		results = append(results, result.Result{
-			Query:    query.Parse(testName),
-			Status:   status,
-			Tags:     tags,
-			Duration: duration,
+			Query:        query.Parse(testName),
+			Status:       status,
+			Tags:         tags,
+			Duration:     duration,
+			MayExonerate: mayExonerate,
 		})
 
 		return nil

tools/src/cmd/cts/merge/merge.go

@@ -53,11 +53,7 @@ func (c *cmd) Run(ctx context.Context, cfg common.Config) error {
 			return fmt.Errorf("while reading '%v': %w", path, err)
 		}
 		// Combine and merge
-		if len(results) > 0 {
-			results = result.Merge(results, r)
-		} else {
-			results = r
-		}
+		results = result.Merge(results, r)
 	}
 
 	// Open output file

tools/src/cmd/cts/update/update.go

@@ -21,6 +21,7 @@ import (
 
 	"dawn.googlesource.com/dawn/tools/src/cmd/cts/common"
 	"dawn.googlesource.com/dawn/tools/src/cts/expectations"
+	"dawn.googlesource.com/dawn/tools/src/cts/result"
 	"go.chromium.org/luci/auth/client/authcli"
 )
 
@@ -65,6 +66,9 @@ func (c *cmd) Run(ctx context.Context, cfg common.Config) error {
 		return err
 	}
 
+	// Merge to remove duplicates
+	results = result.Merge(results)
+
 	// Load the expectations file
 	ex, err := expectations.Load(c.flags.expectations)
 	if err != nil {