tools: Add src/cts/expectations

Implement an expectation parser, data structures and writer.

Bug: dawn:1342
Change-Id: I53587a9b55346ccf1543e15c9cec5ff68c6849ad
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/87641
Reviewed-by: Dan Sinclair <dsinclair@chromium.org>
Kokoro: Kokoro <noreply+kokoro@google.com>
Commit-Queue: Ben Clayton <bclayton@google.com>

tools/src/cts/expectations/expectations.go

@@ -0,0 +1,230 @@
+// Copyright 2022 The Dawn Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package expectations provides types and helpers for parsing, updating and
+// writing WebGPU expectations files.
+//
+// See <dawn>/webgpu-cts/expectations.txt for more information.
+package expectations
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"strings"
+
+	"dawn.googlesource.com/dawn/tools/src/cts/result"
+)
+
+// Content holds the full content of an expectations file.
+type Content struct {
+	Chunks []Chunk
+	Tags   Tags
+}
+
+// Chunk is an optional comment followed by a run of expectations.
+// A chunk ends at the first blank line, or at the transition from an
+// expectation to a line-comment.
+type Chunk struct {
+	Comments     []string      // Line comments at the top of the chunk
+	Expectations []Expectation // Expectations for the chunk
+}
+
+// Tags holds the tag information parsed in the comments between the
+// 'BEGIN TAG HEADER' and 'END TAG HEADER' markers.
+// Tags are grouped in tag-sets.
+type Tags struct {
+	// Map of tag-set name to tags
+	Sets []TagSet
+	// Map of tag name to tag-set and priority
+	ByName map[string]TagSetAndPriority
+}
+
+// TagSet is a named collection of tags, parsed from the 'TAG HEADER'
+type TagSet struct {
+	Name string      // Name of the tag-set
+	Tags result.Tags // Tags belonging to the tag-set
+}
+
+// TagSetAndPriority is used by the Tags.ByName map to identify which tag-set
+// a tag belongs to.
+type TagSetAndPriority struct {
+	// The tag-set that the tag belongs to.
+	Set string
+	// The declared order of tag in the set.
+	// An expectation may only list a single tag from any set. This priority
+	// is used to decide which tag(s) should be dropped when multiple tags are
+	// found in the same set.
+	Priority int
+}
+
+// Expectation holds a single expectation line
+type Expectation struct {
+	Line    int         // The 1-based line number of the expectation
+	Bug     string      // The associated bug URL for this expectation
+	Tags    result.Tags // Tags used to filter the expectation
+	Query   string      // The CTS query
+	Status  []string    // The expected result status
+	Comment string      // Optional comment at end of line
+}
+
+// Load loads the expectation file at 'path', returning a Content.
+func Load(path string) (Content, error) {
+	content, err := ioutil.ReadFile(path)
+	if err != nil {
+		return Content{}, err
+	}
+	ex, err := Parse(string(content))
+	if err != nil {
+		return Content{}, err
+	}
+	return ex, nil
+}
+
+// Save saves the Content file to 'path'.
+func (c Content) Save(path string) error {
+	f, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	return c.Write(f)
+}
+
+// Clone makes a deep-copy of the Content.
+func (c Content) Clone() Content {
+	chunks := make([]Chunk, len(c.Chunks))
+	for i, c := range c.Chunks {
+		chunks[i] = c.Clone()
+	}
+	return Content{chunks, c.Tags.Clone()}
+}
+
+// Empty returns true if the Content has no chunks.
+func (c Content) Empty() bool {
+	return len(c.Chunks) == 0
+}
+
+// EndsInBlankLine returns true if the Content ends with a blank line
+func (c Content) EndsInBlankLine() bool {
+	return !c.Empty() && c.Chunks[len(c.Chunks)-1].IsBlankLine()
+}
+
+// MaybeAddBlankLine appends a new blank line to the content, if the content
+// does not already end in a blank line.
+func (c *Content) MaybeAddBlankLine() {
+	if !c.Empty() && !c.EndsInBlankLine() {
+		c.Chunks = append(c.Chunks, Chunk{})
+	}
+}
+
+// Write writes the Content, in textual form, to the writer w.
+func (c Content) Write(w io.Writer) error {
+	for _, chunk := range c.Chunks {
+		if len(chunk.Comments) == 0 && len(chunk.Expectations) == 0 {
+			if _, err := fmt.Fprintln(w); err != nil {
+				return err
+			}
+			continue
+		}
+		for _, comment := range chunk.Comments {
+			if _, err := fmt.Fprintln(w, comment); err != nil {
+				return err
+			}
+		}
+		for _, expectation := range chunk.Expectations {
+			parts := []string{}
+			if expectation.Bug != "" {
+				parts = append(parts, expectation.Bug)
+			}
+			if len(expectation.Tags) > 0 {
+				parts = append(parts, fmt.Sprintf("[ %v ]", strings.Join(expectation.Tags.List(), " ")))
+			}
+			parts = append(parts, expectation.Query)
+			parts = append(parts, fmt.Sprintf("[ %v ]", strings.Join(expectation.Status, " ")))
+			if expectation.Comment != "" {
+				parts = append(parts, expectation.Comment)
+			}
+			if _, err := fmt.Fprintln(w, strings.Join(parts, " ")); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// String returns the Content as a string.
+func (c Content) String() string {
+	sb := strings.Builder{}
+	c.Write(&sb)
+	return sb.String()
+}
+
+// IsCommentOnly returns true if the Chunk contains comments and no expectations.
+func (c Chunk) IsCommentOnly() bool {
+	return len(c.Comments) > 0 && len(c.Expectations) == 0
+}
+
+// IsBlankLine returns true if the Chunk has no comments or expectations.
+func (c Chunk) IsBlankLine() bool {
+	return len(c.Comments) == 0 && len(c.Expectations) == 0
+}
+
+// Clone returns a deep-copy of the Chunk
+func (c Chunk) Clone() Chunk {
+	comments := make([]string, len(c.Comments))
+	for i, c := range c.Comments {
+		comments[i] = c
+	}
+	expectations := make([]Expectation, len(c.Expectations))
+	for i, e := range c.Expectations {
+		expectations[i] = e.Clone()
+	}
+	return Chunk{comments, expectations}
+}
+
+// Clone returns a deep-copy of the Tags
+func (t Tags) Clone() Tags {
+	out := Tags{}
+	if t.ByName != nil {
+		out.ByName = make(map[string]TagSetAndPriority, len(t.ByName))
+		for n, t := range t.ByName {
+			out.ByName[n] = t
+		}
+	}
+	if t.Sets != nil {
+		out.Sets = make([]TagSet, len(t.Sets))
+		copy(out.Sets, t.Sets)
+	}
+	return out
+}
+
+// Clone makes a deep-copy of the Expectation.
+func (e Expectation) Clone() Expectation {
+	out := Expectation{
+		Line:    e.Line,
+		Bug:     e.Bug,
+		Query:   e.Query,
+		Comment: e.Comment,
+	}
+	if e.Tags != nil {
+		out.Tags = e.Tags.Clone()
+	}
+	if e.Status != nil {
+		out.Status = append([]string{}, e.Status...)
+	}
+	return out
+}

tools/src/cts/expectations/parse.go

@@ -0,0 +1,312 @@
+// Copyright 2022 The Dawn Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package expectations
+
+import (
+	"fmt"
+	"strings"
+
+	"dawn.googlesource.com/dawn/tools/src/cts/result"
+)
+
+const (
+	tagHeaderStart = `BEGIN TAG HEADER`
+	tagHeaderEnd   = `END TAG HEADER`
+)
+
+// SyntaxError is the error type returned by Parse() when a syntax error is
+// encountered.
+type SyntaxError struct {
+	Line    int // 1-based
+	Column  int // 1-based
+	Message string
+}
+
+// Error implements the 'error' interface.
+func (e SyntaxError) Error() string {
+	return fmt.Sprintf("%v:%v: %v", e.Line, e.Column, e.Message)
+}
+
+// Parse parses an expectations file, returning the Content
+func Parse(body string) (Content, error) {
+	// LineType is an enumerator classifying the 'type' of the line.
+	type LineType int
+	const (
+		comment     LineType = iota // The line starts with the '#'
+		expectation                 // The line declares an expectation
+		blank                       // The line is blank
+	)
+
+	// classifyLine returns the LineType for the given line
+	classifyLine := func(line string) LineType {
+		line = strings.TrimSpace(line)
+		switch {
+		case line == "":
+			return blank
+		case strings.HasPrefix(line, "#"):
+			return comment
+		default:
+			return expectation
+		}
+	}
+
+	content := Content{} // The output content
+
+	var pending Chunk // The current Chunk being parsed
+
+	// flush completes the current chunk, appending it to 'content'
+	flush := func() {
+		parseTags(&content.Tags, pending.Comments)
+		content.Chunks = append(content.Chunks, pending)
+		pending = Chunk{}
+	}
+
+	lastLineType := blank                         // The type of the last parsed line
+	for i, l := range strings.Split(body, "\n") { // For each line...
+		lineIdx := i + 1 // line index
+		lineType := classifyLine(l)
+
+		// Compare the new line type to the last.
+		// Flush the pending chunk if needed.
+		if i > 0 {
+			switch {
+			case
+				lastLineType == blank && lineType != blank,             // blank -> !blank
+				lastLineType != blank && lineType == blank,             // !blank -> blank
+				lastLineType == expectation && lineType != expectation: // expectation -> comment
+				flush()
+			}
+		}
+
+		lastLineType = lineType
+
+		// Handle blank lines and comments.
+		switch lineType {
+		case blank:
+			continue
+		case comment:
+			pending.Comments = append(pending.Comments, l)
+			continue
+		}
+
+		// Below this point, we're dealing with an expectation
+
+		// Split the line by whitespace to form a list of tokens
+		type Token struct {
+			str        string
+			start, end int // line offsets (0-based)
+		}
+		tokens := []Token{}
+		if len(l) > 0 { // Parse the tokens
+			inToken, s := false, 0
+			for i, c := range l {
+				if c == ' ' {
+					if inToken {
+						tokens = append(tokens, Token{l[s:i], s, i})
+						inToken = false
+					}
+				} else if !inToken {
+					s = i
+					inToken = true
+				}
+			}
+			if inToken {
+				tokens = append(tokens, Token{l[s:], s, len(l)})
+			}
+		}
+
+		// syntaxErr is a helper for returning a SyntaxError with the current
+		// line and column index.
+		syntaxErr := func(at Token, msg string) error {
+			column := at.start + 1
+			if column == 1 {
+				column = len(l) + 1
+			}
+			return SyntaxError{lineIdx, column, msg}
+		}
+
+		// peek returns the next token without consuming it.
+		// If there are no more tokens then an empty Token is returned.
+		peek := func() Token {
+			if len(tokens) > 0 {
+				return tokens[0]
+			}
+			return Token{}
+		}
+
+		// next returns the next token, consuming it and incrementing the
+		// column index.
+		// If there are no more tokens then an empty Token is returned.
+		next := func() Token {
+			if len(tokens) > 0 {
+				tok := tokens[0]
+				tokens = tokens[1:]
+				return tok
+			}
+			return Token{}
+		}
+
+		match := func(str string) bool {
+			if peek().str != str {
+				return false
+			}
+			next()
+			return true
+		}
+
+		// tags parses a [ tag ] block.
+		tags := func(use string) (result.Tags, error) {
+			if !match("[") {
+				return result.Tags{}, nil
+			}
+			out := result.NewTags()
+			for {
+				t := next()
+				switch t.str {
+				case "]":
+					return out, nil
+				case "":
+					return result.Tags{}, syntaxErr(t, "expected ']' for "+use)
+				default:
+					out.Add(t.str)
+				}
+			}
+		}
+
+		// Parse the optional bug
+		var bug string
+		if strings.HasPrefix(peek().str, "crbug.com") {
+			bug = next().str
+		}
+
+		// Parse the optional test tags
+		testTags, err := tags("tags")
+		if err != nil {
+			return Content{}, err
+		}
+
+		// Parse the query
+		if t := peek(); t.str == "" || t.str[0] == '#' || t.str[0] == '[' {
+			return Content{}, syntaxErr(t, "expected test query")
+		}
+		query := next().str
+
+		// Parse the expected status
+		if t := peek(); !strings.HasPrefix(t.str, "[") {
+			return Content{}, syntaxErr(t, "expected status")
+		}
+		status, err := tags("status")
+		if err != nil {
+			return Content{}, err
+		}
+
+		// Parse any optional trailing comment
+		comment := ""
+		if t := peek(); strings.HasPrefix(t.str, "#") {
+			comment = l[t.start:]
+		}
+
+		// Append the expectation to the list.
+		pending.Expectations = append(pending.Expectations, Expectation{
+			Line:    lineIdx,
+			Bug:     bug,
+			Tags:    testTags,
+			Query:   query,
+			Status:  status.List(),
+			Comment: comment,
+		})
+	}
+
+	if lastLineType != blank {
+		flush()
+	}
+
+	return content, nil
+}
+
+// parseTags parses the tag information found between tagHeaderStart and
+// tagHeaderEnd comments.
+func parseTags(tags *Tags, lines []string) {
+	// Flags for whether we're currently parsing a TAG HEADER and whether we're
+	// also within a tag-set.
+	inTagsHeader, inTagSet := false, false
+	tagSet := TagSet{} // The currently parsed tag-set
+	for _, line := range lines {
+		line = strings.TrimSpace(strings.TrimLeft(strings.TrimSpace(line), "#"))
+		if strings.Contains(line, tagHeaderStart) {
+			if tags.ByName == nil {
+				*tags = Tags{
+					ByName: map[string]TagSetAndPriority{},
+					Sets:   []TagSet{},
+				}
+			}
+			inTagsHeader = true
+			continue
+		}
+		if strings.Contains(line, tagHeaderEnd) {
+			return // Reached the end of the TAG HEADER
+		}
+		if !inTagsHeader {
+			continue // Still looking for a tagHeaderStart
+		}
+
+		// Below this point, we're in a TAG HEADER.
+		tokens := removeEmpty(strings.Split(line, " "))
+		for len(tokens) > 0 {
+			if inTagSet {
+				// Parsing tags in a tag-set (between the '[' and ']')
+				if tokens[0] == "]" {
+					// End of the tag-set.
+					tags.Sets = append(tags.Sets, tagSet)
+					inTagSet = false
+					break
+				} else {
+					// Still inside the tag-set. Consume the tag.
+					tag := tokens[0]
+					tags.ByName[tag] = TagSetAndPriority{
+						Set:      tagSet.Name,
+						Priority: len(tagSet.Tags),
+					}
+					tagSet.Tags.Add(tag)
+				}
+				tokens = tokens[1:]
+			} else {
+				// Outside of tag-set. Scan for 'tags: ['
+				if len(tokens) > 2 && tokens[0] == "tags:" && tokens[1] == "[" {
+					inTagSet = true
+					tagSet.Tags = result.NewTags()
+					tokens = tokens[2:] // Skip 'tags:' and '['
+				} else {
+					// Tag set names are on their own line.
+					// Remember the content of the line, in case the next line
+					// starts a tag-set.
+					tagSet.Name = strings.Join(tokens, " ")
+					break
+				}
+			}
+		}
+	}
+}
+
+// removeEmpty returns the list of strings with all empty strings removed.
+func removeEmpty(in []string) []string {
+	out := make([]string, 0, len(in))
+	for _, s := range in {
+		if s != "" {
+			out = append(out, s)
+		}
+	}
+	return out
+}

tools/src/cts/expectations/parse_test.go

@@ -0,0 +1,472 @@
+// Copyright 2022 The Dawn Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package expectations_test
+
+import (
+	"testing"
+
+	"dawn.googlesource.com/dawn/tools/src/cts/expectations"
+	"dawn.googlesource.com/dawn/tools/src/cts/result"
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestParse(t *testing.T) {
+	type Test struct {
+		name      string
+		in        string
+		expect    expectations.Content
+		expectErr string
+	}
+	for _, test := range []Test{
+		{
+			name:   "empty",
+			in:     ``,
+			expect: expectations.Content{},
+		}, /////////////////////////////////////////////////////////////////////
+		{
+			name: "single line comment",
+			in:   `# a comment`,
+			expect: expectations.Content{
+				Chunks: []expectations.Chunk{
+					{Comments: []string{`# a comment`}},
+				},
+			},
+		}, /////////////////////////////////////////////////////////////////////
+		{
+			name: "single line comment, followed by newline",
+			in: `# a comment
+`,
+			expect: expectations.Content{
+				Chunks: []expectations.Chunk{
+					{Comments: []string{`# a comment`}},
+				},
+			},
+		}, /////////////////////////////////////////////////////////////////////
+		{
+			name: "newline, followed by single line comment",
+			in: `
+# a comment`,
+			expect: expectations.Content{
+				Chunks: []expectations.Chunk{
+					{},
+					{Comments: []string{`# a comment`}},
+				},
+			},
+		}, /////////////////////////////////////////////////////////////////////
+		{
+			name: "comments separated by single newline",
+			in: `# comment 1
+# comment 2`,
+			expect: expectations.Content{
+				Chunks: []expectations.Chunk{
+					{
+						Comments: []string{
+							`# comment 1`,
+							`# comment 2`,
+						},
+					},
+				},
+			},
+		}, /////////////////////////////////////////////////////////////////////
+		{
+			name: "comments separated by two newlines",
+			in: `# comment 1
+
+# comment 2`,
+			expect: expectations.Content{
+				Chunks: []expectations.Chunk{
+					{Comments: []string{`# comment 1`}},
+					{},
+					{Comments: []string{`# comment 2`}},
+				},
+			},
+		}, /////////////////////////////////////////////////////////////////////
+		{
+			name: "comments separated by multiple newlines",
+			in: `# comment 1
+
+
+
+# comment 2`,
+			expect: expectations.Content{
+				Chunks: []expectations.Chunk{
+					{Comments: []string{`# comment 1`}},
+					{},
+					{Comments: []string{`# comment 2`}},
+				},
+			},
+		}, /////////////////////////////////////////////////////////////////////
+		{
+			name: "expectation, single result",
+			in:   `abc,def [ FAIL ]`,
+			expect: expectations.Content{
+				Chunks: []expectations.Chunk{
+					{
+						Expectations: []expectations.Expectation{
+							{
+								Line:   1,
+								Tags:   result.NewTags(),
+								Query:  "abc,def",
+								Status: []string{"FAIL"},
+							},
+						},
+					},
+				},
+			},
+		}, /////////////////////////////////////////////////////////////////////
+		{
+			name: "expectation, with comment",
+			in:   `abc,def [ FAIL ] # this is a comment`,
+			expect: expectations.Content{
+				Chunks: []expectations.Chunk{
+					{
+						Expectations: []expectations.Expectation{
+							{
+								Line:    1,
+								Tags:    result.NewTags(),
+								Query:   "abc,def",
+								Status:  []string{"FAIL"},
+								Comment: "# this is a comment",
+							},
+						},
+					},
+				},
+			},
+		}, /////////////////////////////////////////////////////////////////////
+		{
+			name: "expectation, multiple results",
+			in:   `abc,def [ FAIL SLOW ]`,
+			expect: expectations.Content{
+				Chunks: []expectations.Chunk{
+					{
+						Expectations: []expectations.Expectation{
+							{
+								Line:   1,
+								Tags:   result.NewTags(),
+								Query:  "abc,def",
+								Status: []string{"FAIL", "SLOW"},
+							},
+						},
+					},
+				},
+			},
+		}, /////////////////////////////////////////////////////////////////////
+		{
+			name: "expectation, with single tag",
+			in:   `[ Win ] abc,def [ FAIL ]`,
+			expect: expectations.Content{
+				Chunks: []expectations.Chunk{
+					{
+						Expectations: []expectations.Expectation{
+							{
+								Line:   1,
+								Tags:   result.NewTags("Win"),
+								Query:  "abc,def",
+								Status: []string{"FAIL"},
+							},
+						},
+					},
+				},
+			},
+		}, /////////////////////////////////////////////////////////////////////
+		{
+			name: "expectation, with multiple tags",
+			in:   `[ Win Mac ] abc,def [ FAIL ]`,
+			expect: expectations.Content{
+				Chunks: []expectations.Chunk{
+					{
+						Expectations: []expectations.Expectation{
+							{
+								Line:   1,
+								Tags:   result.NewTags("Win", "Mac"),
+								Query:  "abc,def",
+								Status: []string{"FAIL"},
+							},
+						},
+					},
+				},
+			},
+		}, /////////////////////////////////////////////////////////////////////
+		{
+			name: "expectation, with bug",
+			in:   `crbug.com/123 abc,def [ FAIL ]`,
+			expect: expectations.Content{
+				Chunks: []expectations.Chunk{
+					{
+						Expectations: []expectations.Expectation{
+							{
+								Line:   1,
+								Bug:    "crbug.com/123",
+								Tags:   result.NewTags(),
+								Query:  "abc,def",
+								Status: []string{"FAIL"},
+							},
+						},
+					},
+				},
+			},
+		}, /////////////////////////////////////////////////////////////////////
+		{
+			name: "expectation, with bug and tag",
+			in:   `crbug.com/123 [ Win ] abc,def [ FAIL ]`,
+			expect: expectations.Content{
+				Chunks: []expectations.Chunk{
+					{
+						Expectations: []expectations.Expectation{
+							{
+								Line:   1,
+								Bug:    "crbug.com/123",
+								Tags:   result.NewTags("Win"),
+								Query:  "abc,def",
+								Status: []string{"FAIL"},
+							},
+						},
+					},
+				},
+			},
+		}, /////////////////////////////////////////////////////////////////////
+		{
+			name: "expectation, with comment",
+			in: `# a comment
+crbug.com/123 [ Win ] abc,def [ FAIL ]`,
+			expect: expectations.Content{
+				Chunks: []expectations.Chunk{
+					{
+						Comments: []string{`# a comment`},
+						Expectations: []expectations.Expectation{
+							{
+								Line:   2,
+								Bug:    "crbug.com/123",
+								Tags:   result.NewTags("Win"),
+								Query:  "abc,def",
+								Status: []string{"FAIL"},
+							},
+						},
+					},
+				},
+			},
+		}, /////////////////////////////////////////////////////////////////////
+		{
+			name: "expectation, with multiple comments",
+			in: `# comment 1
+# comment 2
+crbug.com/123 [ Win ] abc,def [ FAIL ]`,
+			expect: expectations.Content{
+				Chunks: []expectations.Chunk{
+					{
+						Comments: []string{`# comment 1`, `# comment 2`},
+						Expectations: []expectations.Expectation{
+							{
+								Line:   3,
+								Bug:    "crbug.com/123",
+								Tags:   result.NewTags("Win"),
+								Query:  "abc,def",
+								Status: []string{"FAIL"},
+							},
+						},
+					},
+				},
+			},
+		}, /////////////////////////////////////////////////////////////////////
+		{
+			name: "comment, test, newline, comment",
+			in: `# comment 1
+crbug.com/123 abc_def [ Skip ]
+
+### comment 2`,
+			expect: expectations.Content{
+				Chunks: []expectations.Chunk{
+					{
+						Comments: []string{`# comment 1`},
+						Expectations: []expectations.Expectation{
+							{
+								Line:   2,
+								Bug:    "crbug.com/123",
+								Tags:   result.NewTags(),
+								Query:  "abc_def",
+								Status: []string{"Skip"},
+							},
+						},
+					},
+					{},
+					{Comments: []string{`### comment 2`}},
+				},
+			},
+		}, /////////////////////////////////////////////////////////////////////
+		{
+			name: "complex",
+			in: `# comment 1
+
+# comment 2
+# comment 3
+
+crbug.com/123 [ Win ] abc,def [ FAIL ]
+
+# comment 4
+# comment 5
+crbug.com/456 [ Mac ] ghi_jkl [ PASS ]
+# comment 6
+
+# comment 7
+`,
+			expect: expectations.Content{
+				Chunks: []expectations.Chunk{
+					{Comments: []string{`# comment 1`}},
+					{},
+					{Comments: []string{`# comment 2`, `# comment 3`}},
+					{},
+					{
+						Expectations: []expectations.Expectation{
+							{
+								Line:   6,
+								Bug:    "crbug.com/123",
+								Tags:   result.NewTags("Win"),
+								Query:  "abc,def",
+								Status: []string{"FAIL"},
+							},
+						},
+					},
+					{},
+					{
+						Comments: []string{`# comment 4`, `# comment 5`},
+						Expectations: []expectations.Expectation{
+							{
+								Line:   10,
+								Bug:    "crbug.com/456",
+								Tags:   result.NewTags("Mac"),
+								Query:  "ghi_jkl",
+								Status: []string{"PASS"},
+							},
+						},
+					},
+					{Comments: []string{`# comment 6`}},
+					{},
+					{Comments: []string{`# comment 7`}},
+				},
+			},
+		}, /////////////////////////////////////////////////////////////////////
+		{
+			name: "tag header",
+			in: `
+# BEGIN TAG HEADER (autogenerated, see validate_tag_consistency.py)
+# Devices
+# tags: [ duck-fish-5 duck-fish-5x duck-horse-2 duck-horse-4
+#             duck-horse-6 duck-shield-duck-tv
+#         mouse-snake-frog mouse-snake-ant mouse-snake
+#         fly-snake-bat fly-snake-worm fly-snake-snail-rabbit ]
+# Platform
+# tags: [ hamster
+#         lion ]
+# Driver
+# tags: [ goat.1 ]
+# END TAG HEADER
+`,
+			expect: expectations.Content{
+				Chunks: []expectations.Chunk{
+					{},
+					{Comments: []string{
+						`# BEGIN TAG HEADER (autogenerated, see validate_tag_consistency.py)`,
+						`# Devices`,
+						`# tags: [ duck-fish-5 duck-fish-5x duck-horse-2 duck-horse-4`,
+						`#             duck-horse-6 duck-shield-duck-tv`,
+						`#         mouse-snake-frog mouse-snake-ant mouse-snake`,
+						`#         fly-snake-bat fly-snake-worm fly-snake-snail-rabbit ]`,
+						`# Platform`,
+						`# tags: [ hamster`,
+						`#         lion ]`,
+						`# Driver`,
+						`# tags: [ goat.1 ]`,
+						`# END TAG HEADER`,
+					}},
+				},
+				Tags: expectations.Tags{
+					ByName: map[string]expectations.TagSetAndPriority{
+						"duck-fish-5":            {Set: "Devices", Priority: 0},
+						"duck-fish-5x":           {Set: "Devices", Priority: 1},
+						"duck-horse-2":           {Set: "Devices", Priority: 2},
+						"duck-horse-4":           {Set: "Devices", Priority: 3},
+						"duck-horse-6":           {Set: "Devices", Priority: 4},
+						"duck-shield-duck-tv":    {Set: "Devices", Priority: 5},
+						"mouse-snake-frog":       {Set: "Devices", Priority: 6},
+						"mouse-snake-ant":        {Set: "Devices", Priority: 7},
+						"mouse-snake":            {Set: "Devices", Priority: 8},
+						"fly-snake-bat":          {Set: "Devices", Priority: 9},
+						"fly-snake-worm":         {Set: "Devices", Priority: 10},
+						"fly-snake-snail-rabbit": {Set: "Devices", Priority: 11},
+						"hamster":                {Set: "Platform", Priority: 0},
+						"lion":                   {Set: "Platform", Priority: 1},
+						"goat.1":                 {Set: "Driver", Priority: 0},
+					},
+					Sets: []expectations.TagSet{
+						{
+							Name: "Devices",
+							Tags: result.NewTags(
+								"duck-fish-5", "duck-fish-5x", "duck-horse-2",
+								"duck-horse-4", "duck-horse-6", "duck-shield-duck-tv",
+								"mouse-snake-frog", "mouse-snake-ant", "mouse-snake",
+								"fly-snake-bat", "fly-snake-worm", "fly-snake-snail-rabbit",
+							),
+						}, {
+							Name: "Platform",
+							Tags: result.NewTags("hamster", "lion"),
+						}, {
+							Name: "Driver",
+							Tags: result.NewTags("goat.1"),
+						},
+					},
+				},
+			},
+		}, /////////////////////////////////////////////////////////////////////
+		{
+			name:      "err missing tag ']'",
+			in:        `[`,
+			expectErr: "1:2: expected ']' for tags",
+		}, /////////////////////////////////////////////////////////////////////
+		{
+			name:      "err missing test query",
+			in:        `[ a ]`,
+			expectErr: "1:6: expected test query",
+		}, /////////////////////////////////////////////////////////////////////
+		{
+			name:      "err missing status EOL",
+			in:        `[ a ] b`,
+			expectErr: "1:8: expected status",
+		}, /////////////////////////////////////////////////////////////////////
+		{
+			name:      "err missing status comment",
+			in:        `[ a ] b # c`,
+			expectErr: "1:9: expected status",
+		}, /////////////////////////////////////////////////////////////////////
+		{
+			name:      "err missing status ']'",
+			in:        `[ a ] b [ c`,
+			expectErr: "1:12: expected ']' for status",
+		},
+	} {
+
+		got, err := expectations.Parse(test.in)
+		errMsg := ""
+		if err != nil {
+			errMsg = err.Error()
+		}
+		if diff := cmp.Diff(errMsg, test.expectErr); diff != "" {
+			t.Errorf("'%v': Parse() error %v", test.name, diff)
+			continue
+		}
+		if diff := cmp.Diff(got, test.expect); diff != "" {
+			t.Errorf("'%v': Parse() was not as expected:\n%v", test.name, diff)
+		}
+	}
+}