diff --git a/tools/src/cmd/intrinsic-gen/ast/ast.go b/tools/src/cmd/intrinsic-gen/ast/ast.go new file mode 100644 index 0000000000..9299f53d31 --- /dev/null +++ b/tools/src/cmd/intrinsic-gen/ast/ast.go @@ -0,0 +1,290 @@ +// Copyright 2021 The Tint Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package ast defines AST nodes that are produced by the Tint intrinsic +// definition parser +package ast + +import ( + "fmt" + "strings" + + "dawn.googlesource.com/tint/tools/src/cmd/intrinsic-gen/tok" +) + +// AST is the parsed syntax tree of the intrinsic definition file +type AST struct { + Enums []EnumDecl + Types []TypeDecl + Matchers []MatcherDecl + Functions []FunctionDecl +} + +func (a AST) String() string { + sb := strings.Builder{} + for _, e := range a.Enums { + fmt.Fprintf(&sb, "%v", e) + fmt.Fprintln(&sb) + } + for _, p := range a.Types { + fmt.Fprintf(&sb, "%v", p) + fmt.Fprintln(&sb) + } + for _, m := range a.Matchers { + fmt.Fprintf(&sb, "%v", m) + fmt.Fprintln(&sb) + } + for _, f := range a.Functions { + fmt.Fprintf(&sb, "%v", f) + fmt.Fprintln(&sb) + } + return sb.String() +} + +// EnumDecl describes an enumerator +type EnumDecl struct { + Source tok.Source + Name string + Entries []string +} + +// Format implements the fmt.Formatter interface +func (e EnumDecl) Format(w fmt.State, verb rune) { + fmt.Fprintf(w, "enum %v {\n", e.Name) + for _, e := range e.Entries { + fmt.Fprintf(w, " %s\n", e) + } + fmt.Fprintf(w, "}\n") +} + +// MatcherDecl describes a matcher declaration +type MatcherDecl struct { + Source tok.Source + Name string + Options MatcherOptions +} + +// Format implements the fmt.Formatter interface +func (m MatcherDecl) Format(w fmt.State, verb rune) { + fmt.Fprintf(w, "match %v", m.Name) + fmt.Fprintf(w, ": ") + m.Options.Format(w, verb) +} + +// FunctionDecl describes a function declaration +type FunctionDecl struct { + Source tok.Source + Name string + TemplateParams TemplateParams + Parameters Parameters + ReturnType *TemplatedName +} + +// Format implements the fmt.Formatter interface +func (f FunctionDecl) Format(w fmt.State, verb rune) { + fmt.Fprintf(w, "fn %v", f.Name) + f.TemplateParams.Format(w, verb) + f.Parameters.Format(w, verb) + if f.ReturnType != nil { + fmt.Fprintf(w, " -> ") + f.ReturnType.Format(w, verb) + } +} + +// Parameters is a list of parameter +type Parameters []Parameter + +// Format implements the fmt.Formatter interface +func (l Parameters) Format(w fmt.State, verb rune) { + fmt.Fprintf(w, "(") + for i, p := range l { + if i > 0 { + fmt.Fprintf(w, ", ") + } + p.Format(w, verb) + } + fmt.Fprintf(w, ")") +} + +// Parameter describes a single parameter of a function +type Parameter struct { + Source tok.Source + Name string // Optional + Type TemplatedName +} + +// Format implements the fmt.Formatter interface +func (p Parameter) Format(w fmt.State, verb rune) { + if p.Name != "" { + fmt.Fprintf(w, "%v ", p.Name) + } + p.Type.Format(w, verb) +} + +// MatcherOptions is a list of TemplatedName +type MatcherOptions TemplatedNames + +// Format implements the fmt.Formatter interface +func (o MatcherOptions) Format(w fmt.State, verb rune) { + for i, mo := range o { + if i > 0 { + fmt.Fprintf(w, " | ") + } + mo.Format(w, verb) + } +} + +// TemplatedNames is a list of TemplatedName +// Example: +// a, c +type TemplatedNames []TemplatedName + +// Format implements the fmt.Formatter interface +func (l TemplatedNames) Format(w fmt.State, verb rune) { + for i, n := range l { + if i > 0 { + fmt.Fprintf(w, ", ") + } + n.Format(w, verb) + } +} + +// TemplatedName is an identifier with optional templated arguments +// Example: +// vec +type TemplatedName struct { + Source tok.Source + Name string + TemplateArgs TemplatedNames +} + +// Format implements the fmt.Formatter interface +func (t TemplatedName) Format(w fmt.State, verb rune) { + fmt.Fprintf(w, "%v", t.Name) + if len(t.TemplateArgs) > 0 { + fmt.Fprintf(w, "<") + t.TemplateArgs.Format(w, verb) + fmt.Fprintf(w, ">") + } +} + +// TypeDecl describes a type declaration +type TypeDecl struct { + Source tok.Source + Decorations Decorations + Name string + TemplateParams TemplateParams +} + +// Format implements the fmt.Formatter interface +func (p TypeDecl) Format(w fmt.State, verb rune) { + if len(p.Decorations) > 0 { + p.Decorations.Format(w, verb) + fmt.Fprintf(w, " type %v", p.Name) + } + fmt.Fprintf(w, "type %v", p.Name) + p.TemplateParams.Format(w, verb) +} + +// TemplateParams is a list of TemplateParam +// Example: +// +type TemplateParams []TemplateParam + +// Format implements the fmt.Formatter interface +func (p TemplateParams) Format(w fmt.State, verb rune) { + if len(p) > 0 { + fmt.Fprintf(w, "<") + for i, tp := range p { + if i > 0 { + fmt.Fprintf(w, ", ") + } + tp.Format(w, verb) + } + fmt.Fprintf(w, ">") + } +} + +// TemplateParam describes a template parameter with optional type +// Example: +// +// +type TemplateParam struct { + Source tok.Source + Name string + Type TemplatedName // Optional +} + +// Format implements the fmt.Formatter interface +func (t TemplateParam) Format(w fmt.State, verb rune) { + fmt.Fprintf(w, "%v", t.Name) + if t.Type.Name != "" { + fmt.Fprintf(w, " : ") + t.Type.Format(w, verb) + } +} + +// Decorations is a list of Decoration +// Example: +// [[a(x), b(y)]] +type Decorations []Decoration + +// Format implements the fmt.Formatter interface +func (l Decorations) Format(w fmt.State, verb rune) { + fmt.Fprint(w, "[[") + for i, d := range l { + if i > 0 { + fmt.Fprintf(w, ", ") + } + d.Format(w, verb) + } + fmt.Fprint(w, "]]") +} + +// Take looks up the decoration with the given name. If the decoration is found +// it is removed from the Decorations list and returned, otherwise nil is +// returned and the Decorations are not altered. +func (l *Decorations) Take(name string) *Decoration { + for i, d := range *l { + if d.Name == name { + *l = append((*l)[:i], (*l)[i+1:]...) + return &d + } + } + return nil +} + +// Decoration describes a single decoration +// Example: +// a(x) +type Decoration struct { + Source tok.Source + Name string + Values []string +} + +// Format implements the fmt.Formatter interface +func (d Decoration) Format(w fmt.State, verb rune) { + fmt.Fprintf(w, "%v", d.Name) + if len(d.Values) > 0 { + fmt.Fprintf(w, "(") + for i, v := range d.Values { + if i > 0 { + fmt.Fprint(w, ", ") + } + fmt.Fprintf(w, "%v", v) + } + fmt.Fprintf(w, ")") + } +} diff --git a/tools/src/cmd/intrinsic-gen/parser/parser.go b/tools/src/cmd/intrinsic-gen/parser/parser.go new file mode 100644 index 0000000000..861e6f4aae --- /dev/null +++ b/tools/src/cmd/intrinsic-gen/parser/parser.go @@ -0,0 +1,292 @@ +// Copyright 2021 The Tint Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package parser provides a basic parser for the Tint intrinsic definition +// language +package parser + +import ( + "fmt" + + "dawn.googlesource.com/tint/tools/src/cmd/intrinsic-gen/ast" + "dawn.googlesource.com/tint/tools/src/cmd/intrinsic-gen/lexer" + "dawn.googlesource.com/tint/tools/src/cmd/intrinsic-gen/tok" +) + +// Parse produces a list of tokens for the given source code +func Parse(source, filepath string) (*ast.AST, error) { + runes := []rune(source) + tokens, err := lexer.Lex(runes, filepath) + if err != nil { + return nil, err + } + + p := parser{tokens: tokens} + return p.parse() +} + +type parser struct { + tokens []tok.Token + err error +} + +func (p *parser) parse() (*ast.AST, error) { + out := ast.AST{} + for p.err == nil { + t := p.peek(0) + if t == nil { + break + } + switch t.Kind { + case tok.Enum: + out.Enums = append(out.Enums, p.enumDecl()) + case tok.Match: + out.Matchers = append(out.Matchers, p.matcherDecl()) + case tok.Type, tok.Ldeco: + out.Types = append(out.Types, p.typeDecl()) + case tok.Function: + out.Functions = append(out.Functions, p.functionDecl()) + default: + p.err = fmt.Errorf("%v unexpected token '%v'", t.Source, t.Kind) + } + if p.err != nil { + return nil, p.err + } + } + return &out, nil +} + +func (p *parser) enumDecl() ast.EnumDecl { + p.expect(tok.Enum, "enum declaration") + name := p.expect(tok.Identifier, "enum name") + e := ast.EnumDecl{Source: name.Source, Name: string(name.Runes)} + p.expect(tok.Lbrace, "enum declaration") + for p.err == nil && p.match(tok.Rbrace) == nil { + e.Entries = append(e.Entries, p.ident("enumerator value")) + } + return e +} + +func (p *parser) matcherDecl() ast.MatcherDecl { + p.expect(tok.Match, "matcher declaration") + name := p.expect(tok.Identifier, "matcher name") + m := ast.MatcherDecl{Source: name.Source, Name: string(name.Runes)} + p.expect(tok.Colon, "matcher declaration") + for p.err == nil { + m.Options = append(m.Options, p.templatedName()) + if p.match(tok.Or) == nil { + break + } + } + return m +} + +func (p *parser) typeDecl() ast.TypeDecl { + decos := p.decorations() + p.expect(tok.Type, "type declaration") + name := p.expect(tok.Identifier, "type name") + m := ast.TypeDecl{ + Source: name.Source, + Decorations: decos, + Name: string(name.Runes), + } + if p.peekIs(0, tok.Lt) { + m.TemplateParams = p.templateParams() + } + return m +} + +func (p *parser) decorations() ast.Decorations { + if p.match(tok.Ldeco) == nil { + return nil + } + out := ast.Decorations{} + for p.err == nil { + name := p.expect(tok.Identifier, "decoration name") + values := []string{} + if p.match(tok.Lparen) != nil { + for p.err == nil { + values = append(values, p.string()) + if p.match(tok.Comma) == nil { + break + } + } + p.expect(tok.Rparen, "decoration values") + } + out = append(out, ast.Decoration{ + Source: name.Source, + Name: string(name.Runes), + Values: values, + }) + if !p.peekIs(0, tok.Comma) { + break + } + } + p.expect(tok.Rdeco, "decoration list") + return out +} + +func (p *parser) functionDecl() ast.FunctionDecl { + p.expect(tok.Function, "function declaration") + name := p.expect(tok.Identifier, "function name") + f := ast.FunctionDecl{Source: name.Source, Name: string(name.Runes)} + if p.peekIs(0, tok.Lt) { + f.TemplateParams = p.templateParams() + } + f.Parameters = p.parameters() + if p.match(tok.Arrow) != nil { + ret := p.templatedName() + f.ReturnType = &ret + } + return f +} + +func (p *parser) parameters() ast.Parameters { + l := ast.Parameters{} + p.expect(tok.Lparen, "function parameter list") + if p.match(tok.Rparen) == nil { + for p.err == nil { + l = append(l, p.parameter()) + if p.match(tok.Comma) == nil { + break + } + } + p.expect(tok.Rparen, "function parameter list") + } + return l +} + +func (p *parser) parameter() ast.Parameter { + if p.peekIs(1, tok.Colon) { + // name type + name := p.expect(tok.Identifier, "parameter name") + p.expect(tok.Colon, "parameter type") + return ast.Parameter{ + Source: name.Source, + Name: string(name.Runes), + Type: p.templatedName(), + } + } + // type + ty := p.templatedName() + return ast.Parameter{ + Source: ty.Source, + Type: ty, + } +} + +func (p *parser) string() string { + s := p.expect(tok.String, "string") + return string(s.Runes) +} + +func (p *parser) templatedName() ast.TemplatedName { + name := p.expect(tok.Identifier, "type name") + m := ast.TemplatedName{Source: name.Source, Name: string(name.Runes)} + if p.match(tok.Lt) != nil { + for p.err == nil { + m.TemplateArgs = append(m.TemplateArgs, p.templatedName()) + if p.match(tok.Comma) == nil { + break + } + } + p.expect(tok.Gt, "template argument type list") + } + return m +} + +func (p *parser) templateParams() ast.TemplateParams { + t := ast.TemplateParams{} + p.expect(tok.Lt, "template parameter list") + for p.err == nil && p.peekIs(0, tok.Identifier) { + t = append(t, p.templateParam()) + } + p.expect(tok.Gt, "template parameter list") + return t +} + +func (p *parser) templateParam() ast.TemplateParam { + name := p.match(tok.Identifier) + t := ast.TemplateParam{ + Source: name.Source, + Name: string(name.Runes), + } + if p.match(tok.Colon) != nil { + t.Type = p.templatedName() + } + p.match(tok.Comma) + return t +} + +func (p *parser) expect(kind tok.Kind, use string) tok.Token { + if p.err != nil { + return tok.Invalid + } + t := p.match(kind) + if t == nil { + if len(p.tokens) > 0 { + p.err = fmt.Errorf("%v expected '%v' for %v, got '%v'", + p.tokens[0].Source, kind, use, p.tokens[0].Kind) + } else { + p.err = fmt.Errorf("expected '%v' for %v, but reached end of file", kind, use) + } + return tok.Invalid + } + return *t +} + +func (p *parser) ident(use string) string { + return string(p.expect(tok.Identifier, use).Runes) +} + +// TODO(bclayton): Currently unused, but will be needed for integer bounds +// func (p *parser) integer(use string) int { +// t := p.expect(tok.Integer, use) +// if t.Kind != tok.Integer { +// return 0 +// } +// i, err := strconv.Atoi(string(t.Runes)) +// if err != nil { +// p.err = err +// return 0 +// } +// return i +// } + +func (p *parser) match(kind tok.Kind) *tok.Token { + if p.err != nil || len(p.tokens) == 0 { + return nil + } + t := p.tokens[0] + if t.Kind != kind { + return nil + } + p.tokens = p.tokens[1:] + return &t +} + +func (p *parser) peekIs(i int, kind tok.Kind) bool { + t := p.peek(i) + if t == nil { + return false + } + return t.Kind == kind +} + +func (p *parser) peek(i int) *tok.Token { + if len(p.tokens) <= i { + return nil + } + return &p.tokens[i] +} diff --git a/tools/src/cmd/intrinsic-gen/parser/parser_test.go b/tools/src/cmd/intrinsic-gen/parser/parser_test.go new file mode 100644 index 0000000000..009aa7c9f1 --- /dev/null +++ b/tools/src/cmd/intrinsic-gen/parser/parser_test.go @@ -0,0 +1,195 @@ +// Copyright 2021 The Tint Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package parser_test + +import ( + "testing" + + "dawn.googlesource.com/tint/tools/src/cmd/intrinsic-gen/ast" + "dawn.googlesource.com/tint/tools/src/cmd/intrinsic-gen/parser" +) + +func TestParser(t *testing.T) { + type test struct { + src string + expect ast.AST + } + + for _, test := range []test{ + {"enum E {}", ast.AST{ + Enums: []ast.EnumDecl{{Name: "E"}}, + }}, + {"enum E { A B C }", ast.AST{ + Enums: []ast.EnumDecl{{ + Name: "E", + Entries: []string{"A", "B", "C"}, + }}, + }}, + {"type T", ast.AST{ + Types: []ast.TypeDecl{{Name: "T"}}, + }}, + {"type T", ast.AST{ + Types: []ast.TypeDecl{{ + Name: "T", + TemplateParams: ast.TemplateParams{ + {Name: "A"}, + {Name: "B"}, + {Name: "C"}, + }, + }}, + }}, + {"[[deco]] type T", ast.AST{ + Types: []ast.TypeDecl{{ + Decorations: ast.Decorations{ + {Name: "deco"}, + }, + Name: "T", + }}, + }}, + {`[[deco("a", "b")]] type T`, ast.AST{ + Types: []ast.TypeDecl{{ + Decorations: ast.Decorations{ + {Name: "deco", Values: []string{"a", "b"}}, + }, + Name: "T", + }}, + }}, + {"match M : A", ast.AST{ + Matchers: []ast.MatcherDecl{{ + Name: "M", + Options: ast.MatcherOptions{ + ast.TemplatedName{Name: "A"}, + }, + }}, + }}, + {"match M : A | B", ast.AST{ + Matchers: []ast.MatcherDecl{{ + Name: "M", + Options: ast.MatcherOptions{ + ast.TemplatedName{Name: "A"}, + ast.TemplatedName{Name: "B"}, + }, + }}, + }}, + {"fn F()", ast.AST{ + Functions: []ast.FunctionDecl{{ + Name: "F", + }}, + }}, + {"fn F(a)", ast.AST{ + Functions: []ast.FunctionDecl{{ + Name: "F", + Parameters: ast.Parameters{ + {Type: ast.TemplatedName{Name: "a"}}, + }, + }}, + }}, + {"fn F(a T)", ast.AST{ + Functions: []ast.FunctionDecl{{ + Name: "F", + Parameters: ast.Parameters{ + {Name: "a", Type: ast.TemplatedName{Name: "T"}}, + }, + }}, + }}, + {"fn F(a, b)", ast.AST{ + Functions: []ast.FunctionDecl{{ + Name: "F", + Parameters: ast.Parameters{ + {Type: ast.TemplatedName{Name: "a"}}, + {Type: ast.TemplatedName{Name: "b"}}, + }, + }}, + }}, + {"fn F>()", ast.AST{ + Functions: []ast.FunctionDecl{{ + Name: "F", + TemplateParams: ast.TemplateParams{ + { + Name: "A", Type: ast.TemplatedName{ + Name: "B", + TemplateArgs: ast.TemplatedNames{ + {Name: "C"}, + }, + }, + }, + }, + }}, + }}, + {"fn F(a X, b Y)", ast.AST{ + Functions: []ast.FunctionDecl{{ + Name: "F", + TemplateParams: ast.TemplateParams{ + {Name: "T"}, + }, + Parameters: ast.Parameters{ + {Name: "a", Type: ast.TemplatedName{Name: "X"}}, + {Name: "b", Type: ast.TemplatedName{ + Name: "Y", + TemplateArgs: []ast.TemplatedName{{Name: "T"}}, + }}, + }, + }}, + }}, + {"fn F() -> X", ast.AST{ + Functions: []ast.FunctionDecl{{ + Name: "F", + ReturnType: &ast.TemplatedName{Name: "X"}, + }}, + }}, + {"fn F() -> X", ast.AST{ + Functions: []ast.FunctionDecl{{ + Name: "F", + ReturnType: &ast.TemplatedName{ + Name: "X", + TemplateArgs: []ast.TemplatedName{{Name: "T"}}, + }, + }}, + }}, + } { + got, err := parser.Parse(test.src, "file.txt") + if err != nil { + t.Errorf("While parsing:\n%s\nParse() returned error: %v", test.src, err) + continue + } + + gotStr, expectStr := got.String(), test.expect.String() + if gotStr != expectStr { + t.Errorf("While parsing:\n%s\nGot:\n%s\nExpected:\n%s", test.src, gotStr, expectStr) + } + } +} + +func TestErrors(t *testing.T) { + type test struct { + src string + expect string + } + + for _, test := range []test{ + {"+", "test.txt:1:1: unexpected '+'"}, + {"123", "test.txt:1:1 unexpected token 'integer'"}, + {"[[123]]", "test.txt:1:3 expected 'ident' for decoration name, got 'integer'"}, + {"[[abc", "expected ']]' for decoration list, but reached end of file"}, + } { + got, err := parser.Parse(test.src, "test.txt") + if gotErr := err.Error(); test.expect != gotErr { + t.Errorf(`Parse() returned error "%+v", expected error "%+v"`, gotErr, test.expect) + } + if got != nil { + t.Errorf("Lex() returned non-nil for error") + } + } +}