Add a slow_tests.txt file

This file should hold all Slow expectations.
Overlaps here are allowed.

Bug: dawn:1832
Change-Id: I69b4e53614d9eeed68315b127f2d37ed76b00026
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/133565
Reviewed-by: Ben Clayton <bclayton@google.com>
Kokoro: Kokoro <noreply+kokoro@google.com>
Commit-Queue: Austin Eng <enga@chromium.org>

tools/src/cmd/cts/common/paths.go

@@ -26,6 +26,10 @@ const (
 	// expectations.txt file.
 	RelativeExpectationsPath = "webgpu-cts/expectations.txt"
 
+	// RelativeSlowExpectationsPath is the dawn-root relative path to the
+	// slow_tests.txt file.
+	RelativeSlowExpectationsPath = "webgpu-cts/slow_tests.txt"
+
 	// RelativeTestListPath is the dawn-root relative path to the
 	// test_list.txt file.
 	RelativeTestListPath = "third_party/gn/webgpu-cts/test_list.txt"
@@ -41,6 +45,16 @@ func DefaultExpectationsPath() string {
 	return path
 }
 
+// DefaultSlowExpectationsPath returns the default path to the slow_tests.txt
+// file. Returns an empty string if the file cannot be found.
+func DefaultSlowExpectationsPath() string {
+	path := filepath.Join(fileutils.DawnRoot(), RelativeSlowExpectationsPath)
+	if _, err := os.Stat(path); err != nil {
+		return ""
+	}
+	return path
+}
+
 // DefaultTestListPath returns the default path to the test_list.txt
 // file. Returns an empty string if the file cannot be found.
 func DefaultTestListPath() string {

tools/src/cmd/cts/validate/validate.go

@@ -31,6 +31,7 @@ func init() {
 type cmd struct {
 	flags struct {
 		expectations string // expectations file path
+		slow string         // slow test expectations file path
 	}
 }
 
@@ -44,16 +45,19 @@ func (cmd) Desc() string {
 
 func (c *cmd) RegisterFlags(ctx context.Context, cfg common.Config) ([]string, error) {
 	defaultExpectations := common.DefaultExpectationsPath()
-	flag.StringVar(&c.flags.expectations, "expectations", defaultExpectations, "path to CTS expectations file to update")
+	slowExpectations := common.DefaultSlowExpectationsPath()
+	flag.StringVar(&c.flags.expectations, "expectations", defaultExpectations, "path to CTS expectations file to validate")
+	flag.StringVar(&c.flags.slow, "slow", slowExpectations, "path to CTS slow expectations file to validate")
 	return nil, nil
 }
 
 func (c *cmd) Run(ctx context.Context, cfg common.Config) error {
-	ex, err := expectations.Load(c.flags.expectations)
+	// Load expectations.txt
+	content, err := expectations.Load(c.flags.expectations)
 	if err != nil {
 		return err
 	}
-	diags := ex.Validate()
+	diags := content.Validate()
 
 	// Print any diagnostics
 	diags.Print(os.Stdout, c.flags.expectations)
@@ -61,6 +65,19 @@ func (c *cmd) Run(ctx context.Context, cfg common.Config) error {
 		return fmt.Errorf("%v errors found", numErrs)
 	}
 
+	// Load slow_tests.txt
+	content, err = expectations.Load(c.flags.slow)
+	if err != nil {
+		return err
+	}
+
+	diags = content.ValidateSlowTests()
+	// Print any diagnostics
+	diags.Print(os.Stdout, c.flags.expectations)
+	if numErrs := diags.NumErrors(); numErrs > 0 {
+		return fmt.Errorf("%v errors found", numErrs)
+	}
+
 	fmt.Println("no issues found")
 	return nil
 }

tools/src/cts/expectations/validate.go

@@ -23,6 +23,7 @@ import (
 
 	"dawn.googlesource.com/dawn/tools/src/container"
 	"dawn.googlesource.com/dawn/tools/src/cts/query"
+	"github.com/google/go-cmp/cmp"
 )
 
 func (c Content) tagsCollide(a, b container.Set[string]) bool {
@@ -75,3 +76,20 @@ func (c Content) Validate() Diagnostics {
 	}
 	return out
 }
+
+// ValidateSlowTests checks that the expectations are only [ Slow ]
+func (c Content) ValidateSlowTests() Diagnostics {
+	var out Diagnostics
+	for _, chunk := range c.Chunks {
+		for _, ex := range chunk.Expectations {
+			if !cmp.Equal(ex.Status, []string{"Slow"}) {
+				out = append(out, Diagnostic{
+					Severity: Error,
+					Line:     ex.Line,
+					Message:  fmt.Sprintf("slow test expectation for %v must be %v but was %v", ex.Query, []string{"Slow"}, ex.Status),
+				})
+			}
+		}
+	}
+	return out
+}

webgpu-cts/slow_tests.txt

@@ -0,0 +1,82 @@
+# This file is a manually maintained list of slow tests. It has the same format as
+# `expectations.txt`, but the only result tag that should be used is [ Slow ]
+
+# BEGIN TAG HEADER (autogenerated, see validate_tag_consistency.py)
+# OS
+# tags: [ android android-lollipop android-marshmallow android-nougat
+#             android-pie android-r android-s android-t
+#         chromeos
+#         fuchsia
+#         linux ubuntu
+#         mac highsierra mojave catalina bigsur monterey ventura
+#         win win8 win10 ]
+# Devices
+# tags: [ android-nexus-5x android-pixel-2 android-pixel-4
+#             android-pixel-6 android-shield-android-tv android-sm-a135m
+#             android-sm-a235m
+#         chromeos-board-amd64-generic chromeos-board-kevin chromeos-board-eve
+#         fuchsia-board-astro fuchsia-board-sherlock fuchsia-board-qemu-x64 ]
+# Platform
+# tags: [ desktop
+#         mobile ]
+# Browser
+# tags: [ android-chromium android-webview-instrumentation
+#         debug debug-x64
+#         release release-x64
+#         fuchsia-chrome web-engine-shell ]
+# GPU
+# tags: [ amd amd-0x6613 amd-0x679e amd-0x67ef amd-0x6821 amd-0x7340
+#         apple apple-apple-m1 apple-angle-metal-renderer:-apple-m1
+#         arm
+#         google google-0xffff google-0xc0de
+#         intel intel-gen-9 intel-gen-12 intel-0xa2e intel-0xd26 intel-0xa011
+#               intel-0x3e92 intel-0x3e9b intel-0x5912 intel-0x9bc5
+#         nvidia nvidia-0xfe9 nvidia-0x1cb3 nvidia-0x2184
+#         qualcomm ]
+# Architecture
+# tags: [ mac-arm64 mac-x86_64 ]
+# Decoder
+# tags: [ passthrough no-passthrough ]
+# Browser Target CPU
+# tags: [ target-cpu-64 target-cpu-32 target-cpu-31 ]
+# ANGLE Backend
+# tags: [ angle-disabled
+#         angle-d3d9 angle-d3d11
+#         angle-metal
+#         angle-opengl angle-opengles
+#         angle-swiftshader
+#         angle-vulkan ]
+# Skia Renderer
+# tags: [ renderer-skia-dawn
+#         renderer-skia-gl
+#         renderer-skia-vulkan
+#         renderer-software ]
+# Driver
+# tags: [ mesa_lt_19.1
+#         mesa_ge_21.0 ]
+# ASan
+# tags: [ asan no-asan ]
+# Display Server
+# tags: [ display-server-wayland display-server-x ]
+# OOP-Canvas
+# tags: [ oop-c no-oop-c ]
+# WebGPU Backend Validation
+# tags: [ dawn-backend-validation dawn-no-backend-validation ]
+# WebGPU Adapter
+# tags: [ webgpu-adapter-default webgpu-adapter-swiftshader ]
+# Clang coverage
+# tags: [ clang-coverage no-clang-coverage ]
+# results: [ Failure RetryOnFailure Skip Slow ]
+# END TAG HEADER
+
+################################################################################
+# Large and slow tests on intel-gen-9 win10
+################################################################################
+crbug.com/1406064 [ intel-gen-9 win10 ] webgpu:api,validation,error_scope:current_scope:errorFilter="out-of-memory";stackDepth=100000 [ Slow ]
+crbug.com/1406064 [ intel-gen-9 win10 ] webgpu:api,validation,error_scope:current_scope:errorFilter="validation";stackDepth=100000 [ Slow ]
+
+################################################################################
+# These tests are slow enough that they fail often on a lot of configurations,
+# especially code-coverage builders.
+################################################################################
+crbug.com/dawn/1783 webgpu:shader,execution,expression,binary,* [ Slow ]