Update cts expectations to be sorted by bug first.

- Run format on the current expectations file to make it up to date.
- It's a lot easier to read the failures when they are grouped by the bug.

Change-Id: I333a0b237ccd405d204e47809eb7ab723824fddd
No-Try: True
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/98241
Auto-Submit: Loko Kung <lokokung@google.com>
Reviewed-by: Corentin Wallez <cwallez@chromium.org>
Commit-Queue: Loko Kung <lokokung@google.com>

tools/src/cts/expectations/expectations.go

@@ -23,6 +23,7 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
+	"sort"
 	"strings"
 
 	"dawn.googlesource.com/dawn/tools/src/cts/result"
@@ -39,7 +40,7 @@ type Content struct {
 // expectation to a line-comment.
 type Chunk struct {
 	Comments     []string      // Line comments at the top of the chunk
-	Expectations []Expectation // Expectations for the chunk
+	Expectations Expectations  // Expectations for the chunk
 }
 
 // Tags holds the tag information parsed in the comments between the
@@ -80,6 +81,8 @@ type Expectation struct {
 	Comment string      // Optional comment at end of line
 }
 
+type Expectations []Expectation
+
 // Load loads the expectation file at 'path', returning a Content.
 func Load(path string) (Content, error) {
 	content, err := ioutil.ReadFile(path)
@@ -228,3 +231,36 @@ func (e Expectation) Clone() Expectation {
 	}
 	return out
 }
+
+// Compare compares the relative order of a and b, returning:
+//  -1 if a should come before b
+//   1 if a should come after b
+//   0 if a and b are identical
+// Note: Only comparing bug, query, and tags (in that order).
+func (a Expectation) Compare(b Expectation) int {
+	switch strings.Compare(a.Bug, b.Bug) {
+	case -1:
+		return -1
+	case 1:
+		return 1
+	}
+	switch strings.Compare(a.Query, b.Query) {
+	case -1:
+		return -1
+	case 1:
+		return 1
+	}
+	aTag := result.TagsToString(a.Tags)
+	bTag := result.TagsToString(b.Tags)
+	switch strings.Compare(aTag, bTag) {
+	case -1:
+		return -1
+	case 1:
+		return 1
+	}
+	return 0
+}
+
+func (l Expectations) Sort() {
+	sort.Slice(l, func(i, j int) bool { return l[i].Compare(l[j]) < 0 })
+}

tools/src/cts/expectations/parse.go

@@ -55,6 +55,7 @@ func Parse(body string) (Content, error) {
 	// flush completes the current chunk, appending it to 'content'
 	flush := func() {
 		parseTags(&content.Tags, pending.Comments)
+		pending.Expectations.Sort()
 		content.Chunks = append(content.Chunks, pending)
 		pending = Chunk{}
 	}

tools/src/cts/expectations/update.go

@@ -17,7 +17,6 @@ package expectations
 import (
 	"errors"
 	"fmt"
-	"sort"
 	"strings"
 	"time"
 
@@ -271,24 +270,7 @@ func (u *updater) chunk(in Chunk) Chunk {
 	}
 
 	// Sort the expectations to keep things clean and tidy.
-	sort.Slice(out.Expectations, func(i, j int) bool {
+	out.Expectations.Sort()
-		switch {
-		case out.Expectations[i].Query < out.Expectations[j].Query:
-			return true
-		case out.Expectations[i].Query > out.Expectations[j].Query:
-			return false
-		}
-		a := result.TagsToString(out.Expectations[i].Tags)
-		b := result.TagsToString(out.Expectations[j].Tags)
-		switch {
-		case a < b:
-			return true
-		case a > b:
-			return false
-		}
-		return false
-	})
-
 	return out
 }
 

webgpu-cts/expectations.txt

@@ -76,16 +76,16 @@ crbug.com/dawn/1487 [ intel-gen-9 win10 ] webgpu:api,operation,resource_init,tex
 ################################################################################
 crbug.com/1343969 [ intel-gen-9 win10 ] webgpu:web_platform,copyToTexture,ImageBitmap:from_ImageData:alpha="premultiply";orientation="flipY";srcDoFlipYDuringCopy=false;dstColorFormat="rgba32float";* [ RetryOnFailure ]
 crbug.com/1343969 [ intel-gen-9 win10 ] webgpu:web_platform,copyToTexture,ImageBitmap:from_ImageData:alpha="premultiply";orientation="flipY";srcDoFlipYDuringCopy=true;dstColorFormat="rgba32float";* [ RetryOnFailure ]
-crbug.com/1343969 [ intel-gen-9 win10 ] webgpu:web_platform,copyToTexture,ImageBitmap:from_ImageData:alpha="premultiply";orientation="none";srcDoFlipYDuringCopy=true;dstColorFormat="rg32float";* [ RetryOnFailure ]
 crbug.com/1343969 [ intel-gen-9 win10 ] webgpu:web_platform,copyToTexture,ImageBitmap:from_ImageData:alpha="premultiply";orientation="none";srcDoFlipYDuringCopy=false;dstColorFormat="rg32float";* [ RetryOnFailure ]
-crbug.com/1343969 [ intel-gen-9 win10 webgpu-adapter-default ] webgpu:web_platform,copyToTexture,ImageBitmap:from_canvas:orientation="flipY";srcDoFlipYDuringCopy=false;dstColorFormat="rgba32float";* [ RetryOnFailure ]
+crbug.com/1343969 [ intel-gen-9 win10 ] webgpu:web_platform,copyToTexture,ImageBitmap:from_ImageData:alpha="premultiply";orientation="none";srcDoFlipYDuringCopy=true;dstColorFormat="rg32float";* [ RetryOnFailure ]
-crbug.com/1343969 [ intel-gen-9 win10 webgpu-adapter-default ] webgpu:web_platform,copyToTexture,ImageBitmap:from_canvas:orientation="flipY";srcDoFlipYDuringCopy=true;dstColorFormat="rgba32float";* [ RetryOnFailure ]
+crbug.com/1343969 [ intel-gen-9 webgpu-adapter-default win10 ] webgpu:web_platform,copyToTexture,ImageBitmap:from_canvas:orientation="flipY";srcDoFlipYDuringCopy=false;dstColorFormat="rgba32float";* [ RetryOnFailure ]
-crbug.com/1343969 [ intel-gen-9 win10 webgpu-adapter-default ] webgpu:web_platform,copyToTexture,ImageBitmap:from_canvas:orientation="none";srcDoFlipYDuringCopy=true;dstColorFormat="rg32float";* [ RetryOnFailure ]
+crbug.com/1343969 [ intel-gen-9 webgpu-adapter-swiftshader win10 ] webgpu:web_platform,copyToTexture,ImageBitmap:from_canvas:orientation="flipY";srcDoFlipYDuringCopy=false;dstColorFormat="rgba32float";* [ Failure ]
-crbug.com/1343969 [ intel-gen-9 win10 webgpu-adapter-default ] webgpu:web_platform,copyToTexture,ImageBitmap:from_canvas:orientation="none";srcDoFlipYDuringCopy=false;dstColorFormat="rg32float";* [ RetryOnFailure ]
+crbug.com/1343969 [ intel-gen-9 webgpu-adapter-default win10 ] webgpu:web_platform,copyToTexture,ImageBitmap:from_canvas:orientation="flipY";srcDoFlipYDuringCopy=true;dstColorFormat="rgba32float";* [ RetryOnFailure ]
-crbug.com/1343969 [ intel-gen-9 win10 webgpu-adapter-swiftshader ] webgpu:web_platform,copyToTexture,ImageBitmap:from_canvas:orientation="flipY";srcDoFlipYDuringCopy=false;dstColorFormat="rgba32float";* [ Failure ]
+crbug.com/1343969 [ intel-gen-9 webgpu-adapter-swiftshader win10 ] webgpu:web_platform,copyToTexture,ImageBitmap:from_canvas:orientation="flipY";srcDoFlipYDuringCopy=true;dstColorFormat="rgba32float";* [ Failure ]
-crbug.com/1343969 [ intel-gen-9 win10 webgpu-adapter-swiftshader ] webgpu:web_platform,copyToTexture,ImageBitmap:from_canvas:orientation="flipY";srcDoFlipYDuringCopy=true;dstColorFormat="rgba32float";* [ Failure ]
+crbug.com/1343969 [ intel-gen-9 webgpu-adapter-default win10 ] webgpu:web_platform,copyToTexture,ImageBitmap:from_canvas:orientation="none";srcDoFlipYDuringCopy=false;dstColorFormat="rg32float";* [ RetryOnFailure ]
-crbug.com/1343969 [ intel-gen-9 win10 webgpu-adapter-swiftshader ] webgpu:web_platform,copyToTexture,ImageBitmap:from_canvas:orientation="none";srcDoFlipYDuringCopy=true;dstColorFormat="rg32float";* [ Failure ]
+crbug.com/1343969 [ intel-gen-9 webgpu-adapter-swiftshader win10 ] webgpu:web_platform,copyToTexture,ImageBitmap:from_canvas:orientation="none";srcDoFlipYDuringCopy=false;dstColorFormat="rg32float";* [ Failure ]
-crbug.com/1343969 [ intel-gen-9 win10 webgpu-adapter-swiftshader ] webgpu:web_platform,copyToTexture,ImageBitmap:from_canvas:orientation="none";srcDoFlipYDuringCopy=false;dstColorFormat="rg32float";* [ Failure ]
+crbug.com/1343969 [ intel-gen-9 webgpu-adapter-default win10 ] webgpu:web_platform,copyToTexture,ImageBitmap:from_canvas:orientation="none";srcDoFlipYDuringCopy=true;dstColorFormat="rg32float";* [ RetryOnFailure ]
+crbug.com/1343969 [ intel-gen-9 webgpu-adapter-swiftshader win10 ] webgpu:web_platform,copyToTexture,ImageBitmap:from_canvas:orientation="none";srcDoFlipYDuringCopy=true;dstColorFormat="rg32float";* [ Failure ]
 
 ################################################################################
 # copyToTexture failures on Linux
@@ -116,6 +116,12 @@ crbug.com/1344876 [ webgpu-adapter-swiftshader win ] webgpu:web_platform,copyToT
 # copyToTexture failures with incorrect values on Mac Intel
 # KEEP
 ################################################################################
+crbug.com/dawn/1047 [ intel mac ] webgpu:api,operation,command_buffer,copyTextureToTexture:color_textures,non_compressed,array:srcFormat="r8snorm";* [ Failure ]
+crbug.com/dawn/1047 [ intel mac ] webgpu:api,operation,command_buffer,copyTextureToTexture:color_textures,non_compressed,array:srcFormat="rg8snorm";* [ Failure ]
+crbug.com/dawn/1047 [ intel mac ] webgpu:api,operation,command_buffer,copyTextureToTexture:color_textures,non_compressed,array:srcFormat="rgba8snorm";* [ Failure ]
+crbug.com/dawn/1047 [ intel mac ] webgpu:api,operation,command_buffer,copyTextureToTexture:color_textures,non_compressed,non_array:srcFormat="r8snorm";* [ Failure ]
+crbug.com/dawn/1047 [ intel mac ] webgpu:api,operation,command_buffer,copyTextureToTexture:color_textures,non_compressed,non_array:srcFormat="rg8snorm";* [ Failure ]
+crbug.com/dawn/1047 [ intel mac ] webgpu:api,operation,command_buffer,copyTextureToTexture:color_textures,non_compressed,non_array:srcFormat="rgba8snorm";* [ Failure ]
 crbug.com/dawn/1107 [ intel mac ] webgpu:api,operation,command_buffer,copyTextureToTexture:color_textures,non_compressed,array:srcFormat="r32float";dstFormat="r32float";dimension="2d" [ Failure ]
 crbug.com/dawn/1107 [ intel mac ] webgpu:api,operation,command_buffer,copyTextureToTexture:color_textures,non_compressed,array:srcFormat="rg11b10ufloat";dstFormat="rg11b10ufloat";dimension="2d" [ Failure ]
 crbug.com/dawn/1107 [ intel mac ] webgpu:api,operation,command_buffer,copyTextureToTexture:color_textures,non_compressed,array:srcFormat="rg16float";dstFormat="rg16float";dimension="2d" [ Failure ]
@@ -128,12 +134,6 @@ crbug.com/dawn/1107 [ intel mac ] webgpu:api,operation,command_buffer,copyTextur
 crbug.com/dawn/1107 [ intel mac ] webgpu:api,operation,command_buffer,copyTextureToTexture:color_textures,non_compressed,non_array:srcFormat="rg32float";dstFormat="rg32float";dimension="2d" [ Failure ]
 crbug.com/dawn/1107 [ intel mac ] webgpu:api,operation,command_buffer,copyTextureToTexture:color_textures,non_compressed,non_array:srcFormat="rgba16float";dstFormat="rgba16float";dimension="2d" [ Failure ]
 crbug.com/dawn/1107 [ intel mac ] webgpu:api,operation,command_buffer,copyTextureToTexture:color_textures,non_compressed,non_array:srcFormat="rgba32float";dstFormat="rgba32float";dimension="2d" [ Failure ]
-crbug.com/dawn/1047 [ intel mac ] webgpu:api,operation,command_buffer,copyTextureToTexture:color_textures,non_compressed,array:srcFormat="r8snorm";* [ Failure ]
-crbug.com/dawn/1047 [ intel mac ] webgpu:api,operation,command_buffer,copyTextureToTexture:color_textures,non_compressed,array:srcFormat="rg8snorm";* [ Failure ]
-crbug.com/dawn/1047 [ intel mac ] webgpu:api,operation,command_buffer,copyTextureToTexture:color_textures,non_compressed,array:srcFormat="rgba8snorm";* [ Failure ]
-crbug.com/dawn/1047 [ intel mac ] webgpu:api,operation,command_buffer,copyTextureToTexture:color_textures,non_compressed,non_array:srcFormat="r8snorm";* [ Failure ]
-crbug.com/dawn/1047 [ intel mac ] webgpu:api,operation,command_buffer,copyTextureToTexture:color_textures,non_compressed,non_array:srcFormat="rg8snorm";* [ Failure ]
-crbug.com/dawn/1047 [ intel mac ] webgpu:api,operation,command_buffer,copyTextureToTexture:color_textures,non_compressed,non_array:srcFormat="rgba8snorm";* [ Failure ]
 
 ################################################################################
 # copyToTexture failures. Needs investigation
@@ -499,8 +499,6 @@ crbug.com/dawn/0000 [ dawn-no-backend-validation nvidia-0x2184 win10 ] webgpu:sh
 crbug.com/dawn/0000 [ amd-0x6821 ] webgpu:shader,execution,expression,call,builtin,min:f32:inputSource="uniform";vectorize=2 [ Failure ]
 crbug.com/dawn/0000 [ amd-0x6821 ] webgpu:shader,execution,expression,call,builtin,min:f32:inputSource="uniform";vectorize=3 [ Failure ]
 crbug.com/dawn/0000 [ amd-0x6821 ] webgpu:shader,execution,expression,call,builtin,min:f32:inputSource="uniform";vectorize=4 [ Failure ]
-crbug.com/tint/1632 [ linux ] webgpu:shader,execution,robust_access:linear_memory:storageClass="uniform";access="read";dynamicOffset=false;containerType="matrix";isAtomic=false [ Failure ]
-crbug.com/tint/1632 [ linux ] webgpu:shader,execution,robust_access:linear_memory:storageClass="uniform";access="read";dynamicOffset=true;containerType="matrix";isAtomic=false [ Failure ]
 crbug.com/tint/1500 [ nvidia-0x2184 win10 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="mixed";dispatch="indirect";groupSize={"x":1,"y":1,"z":1};numGroups={"x":1,"y":1,"z":1} [ Failure ]
 crbug.com/tint/1500 [ nvidia-0x2184 win10 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="mixed";dispatch="indirect";groupSize={"x":1,"y":1,"z":1};numGroups={"x":8,"y":4,"z":2} [ Failure ]
 crbug.com/tint/1500 [ nvidia-0x2184 win10 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="mixed";dispatch="indirect";groupSize={"x":3,"y":7,"z":5};numGroups={"x":13,"y":9,"z":11} [ Failure ]
@@ -513,6 +511,8 @@ crbug.com/tint/1500 [ nvidia-0x2184 win10 ] webgpu:shader,execution,shader_io,co
 crbug.com/tint/1500 [ nvidia-0x2184 win10 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="struct";dispatch="indirect";groupSize={"x":1,"y":1,"z":1};numGroups={"x":8,"y":4,"z":2} [ Failure ]
 crbug.com/tint/1500 [ nvidia-0x2184 win10 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="struct";dispatch="indirect";groupSize={"x":3,"y":7,"z":5};numGroups={"x":13,"y":9,"z":11} [ Failure ]
 crbug.com/tint/1500 [ nvidia-0x2184 win10 ] webgpu:shader,execution,shader_io,compute_builtins:inputs:method="struct";dispatch="indirect";groupSize={"x":8,"y":4,"z":2};numGroups={"x":1,"y":1,"z":1} [ Failure ]
+crbug.com/tint/1632 [ linux ] webgpu:shader,execution,robust_access:linear_memory:storageClass="uniform";access="read";dynamicOffset=false;containerType="matrix";isAtomic=false [ Failure ]
+crbug.com/tint/1632 [ linux ] webgpu:shader,execution,robust_access:linear_memory:storageClass="uniform";access="read";dynamicOffset=true;containerType="matrix";isAtomic=false [ Failure ]
 
 ################################################################################
 # webgpu:shader,execution,memory_model
@@ -526,6 +526,7 @@ crbug.com/dawn/0000 [ win ] webgpu:shader,execution,memory_model,* [ Failure ]
 ################################################################################
 # shader validation failures
 ################################################################################
+crbug.com/dawn/0000 [ intel-gen-9 win10 ] webgpu:shader,validation,parse,literal:abstract_float:val=".0" [ Failure ]
 crbug.com/tint/0000 webgpu:shader,validation,parse,blankspace:null_characters:contains_null=true;placement="comment" [ Failure ]
 crbug.com/tint/0000 webgpu:shader,validation,parse,blankspace:null_characters:contains_null=true;placement="eol" [ Failure ]
 crbug.com/tint/0000 [ amd-0x6821 monterey ] webgpu:shader,validation,parse,identifiers:identifiers:* [ Failure ]
@@ -534,7 +535,6 @@ crbug.com/tint/0000 [ intel-gen-9 ubuntu ] webgpu:shader,validation,parse,identi
 crbug.com/tint/0000 [ intel-gen-9 win10 ] webgpu:shader,validation,parse,identifiers:identifiers:* [ Failure ]
 crbug.com/tint/0000 [ nvidia-0x2184 ubuntu ] webgpu:shader,validation,parse,identifiers:identifiers:* [ Failure ]
 crbug.com/tint/0000 [ nvidia-0x2184 win10 ] webgpu:shader,validation,parse,identifiers:identifiers:* [ Failure ]
-crbug.com/dawn/0000 [ intel-gen-9 win10 ] webgpu:shader,validation,parse,literal:abstract_float:val=".0" [ Failure ]
 crbug.com/tint/1502 webgpu:shader,validation,parse,literal:abstract_float:val=".1h" [ Failure ]
 crbug.com/tint/1502 webgpu:shader,validation,parse,literal:abstract_float:val="0X3h" [ Failure ]
 crbug.com/tint/1502 webgpu:shader,validation,parse,literal:abstract_float:val="0h" [ Failure ]
@@ -592,12 +592,12 @@ crbug.com/dawn/0000 [ nvidia-0x2184 ubuntu ] webgpu:api,operation,render_pass,cl
 # Failures with stencil textures on AMD Mac. Needs triage.
 # KEEP
 ################################################################################
+crbug.com/dawn/0000 [ amd monterey ] webgpu:api,operation,command_buffer,copyTextureToTexture:copy_depth_stencil:format="stencil8" [ Failure ]
+crbug.com/dawn/0000 [ amd monterey ] webgpu:api,operation,command_buffer,image_copy:offsets_and_sizes_copy_depth_stencil:format="stencil8";* [ Failure ]
 crbug.com/dawn/0000 [ amd monterey ] webgpu:api,operation,command_buffer,image_copy:rowsPerImage_and_bytesPerRow_depth_stencil:format="stencil8";* [ Failure ]
 crbug.com/dawn/0000 [ amd monterey ] webgpu:api,operation,resource_init,texture_zero:uninitialized_texture_is_zero:dimension="2d";readMethod="CopyToBuffer";format="stencil8" [ Failure ]
 crbug.com/dawn/0000 [ amd monterey ] webgpu:api,operation,resource_init,texture_zero:uninitialized_texture_is_zero:dimension="2d";readMethod="CopyToTexture";format="stencil8" [ Failure ]
 crbug.com/dawn/0000 [ amd monterey ] webgpu:api,operation,resource_init,texture_zero:uninitialized_texture_is_zero:dimension="2d";readMethod="StencilTest";format="stencil8" [ Failure ]
-crbug.com/dawn/0000 [ amd monterey ] webgpu:api,operation,command_buffer,image_copy:offsets_and_sizes_copy_depth_stencil:format="stencil8";* [ Failure ]
-crbug.com/dawn/0000 [ amd monterey ] webgpu:api,operation,command_buffer,copyTextureToTexture:copy_depth_stencil:format="stencil8" [ Failure ]
 
 ################################################################################
 # Failures with depth/stencil textures on Intel Mac
@@ -605,12 +605,12 @@ crbug.com/dawn/0000 [ amd monterey ] webgpu:api,operation,command_buffer,copyTex
 ################################################################################
 crbug.com/dawn/1083 [ intel monterey ] webgpu:api,operation,command_buffer,copyTextureToTexture:copy_depth_stencil:format="depth16unorm" [ Failure ]
 crbug.com/dawn/1083 [ intel monterey ] webgpu:api,operation,command_buffer,copyTextureToTexture:copy_depth_stencil:format="depth24plus" [ Failure ]
-crbug.com/dawn/1083 [ intel monterey ] webgpu:api,operation,command_buffer,copyTextureToTexture:copy_depth_stencil:format="depth32float" [ Failure ]
 crbug.com/dawn/1083 [ intel monterey ] webgpu:api,operation,command_buffer,copyTextureToTexture:copy_depth_stencil:format="depth24plus-stencil8" [ Failure ]
+crbug.com/dawn/1083 [ intel monterey ] webgpu:api,operation,command_buffer,copyTextureToTexture:copy_depth_stencil:format="depth32float" [ Failure ]
 crbug.com/dawn/1083 [ intel monterey ] webgpu:api,operation,command_buffer,copyTextureToTexture:copy_depth_stencil:format="depth32float-stencil8" [ Failure ]
 crbug.com/dawn/1083 [ intel monterey ] webgpu:api,operation,command_buffer,image_copy:offsets_and_sizes_copy_depth_stencil:format="stencil8";* [ Failure ]
-crbug.com/dawn/1083 [ intel monterey ] webgpu:api,operation,render_pass,storeOp:render_pass_store_op,depth_stencil_attachment_only:depthStencilFormat="stencil8";* [ Failure ]
 crbug.com/dawn/1083 [ intel monterey ] webgpu:api,operation,command_buffer,image_copy:rowsPerImage_and_bytesPerRow_depth_stencil:format="stencil8";* [ Failure ]
+crbug.com/dawn/1083 [ intel monterey ] webgpu:api,operation,render_pass,storeOp:render_pass_store_op,depth_stencil_attachment_only:depthStencilFormat="stencil8";* [ Failure ]
 
 ################################################################################
 # webgpu:api,operation,resource_init,texture_zero:uninitialized_texture_is_zero