dawn_node: Add idlgen tool

A go WebIDL parser and template generator tool. Will be used to generate the NodeJS bindings for Dawn. We may wish to reimplement this in Python some day. Not today. Bug: dawn:1123 Change-Id: I31c868efcd8ba00084a6c25a1fc0e3ad774dfa53 Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/64746 Reviewed-by: Antonio Maiorano <amaiorano@google.com> Reviewed-by: Austin Eng <enga@chromium.org> Commit-Queue: Ben Clayton <bclayton@google.com>
2021-09-27 20:41:15 +00:00 · 2021-09-27 20:41:15 +00:00 · cea792f971
parent 029d67f2c8
commit cea792f971
3 changed files with 651 additions and 0 deletions
--- a/src/dawn_node/tools/cmd/idlgen/main.go
+++ b/src/dawn_node/tools/cmd/idlgen/main.go
@ -0,0 +1,620 @@
+// Copyright 2021 The Dawn Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// idlgen is a tool used to generate code from WebIDL files and a golang
+// template file
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"text/template"
+	"unicode"
+
+	"github.com/ben-clayton/webidlparser/ast"
+	"github.com/ben-clayton/webidlparser/parser"
+)
+
+func main() {
+	if err := run(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func showUsage() {
+	fmt.Println(`
+idlgen is a tool used to generate code from WebIDL files and a golang
+template file
+
+Usage:
+  idlgen --template=<template-path> --output=<output-path> <idl-file> [<idl-file>...]`)
+	os.Exit(1)
+}
+
+func run() error {
+	var templatePath string
+	var outputPath string
+	flag.StringVar(&templatePath, "template", "", "the template file run with the parsed WebIDL files")
+	flag.StringVar(&outputPath, "output", "", "the output file")
+	flag.Parse()
+
+	idlFiles := flag.Args()
+
+	// Check all required arguments are provided
+	if templatePath == "" || outputPath == "" || len(idlFiles) == 0 {
+		showUsage()
+	}
+
+	// Open up the output file
+	out := os.Stdout
+	if outputPath != "" {
+		file, err := os.Create(outputPath)
+		if err != nil {
+			return fmt.Errorf("failed to open output file '%v'", outputPath)
+		}
+		out = file
+		defer file.Close()
+	}
+
+	// Read the template file
+	tmpl, err := ioutil.ReadFile(templatePath)
+	if err != nil {
+		return fmt.Errorf("failed to open template file '%v'", templatePath)
+	}
+
+	// idl is the combination of the parsed idlFiles
+	idl := &ast.File{}
+
+	// Parse each of the WebIDL files and add the declarations to idl
+	for _, path := range idlFiles {
+		content, err := ioutil.ReadFile(path)
+		if err != nil {
+			return fmt.Errorf("failed to open file '%v'", path)
+		}
+		fileIDL := parser.Parse(string(content))
+		if numErrs := len(fileIDL.Errors); numErrs != 0 {
+			errs := make([]string, numErrs)
+			for i, e := range fileIDL.Errors {
+				errs[i] = e.Message
+			}
+			return fmt.Errorf("errors found while parsing %v:\n%v", path, strings.Join(errs, "\n"))
+		}
+		idl.Declarations = append(idl.Declarations, fileIDL.Declarations...)
+	}
+
+	// Initialize the generator
+	g := generator{t: template.New(templatePath)}
+	g.workingDir = filepath.Dir(templatePath)
+	g.funcs = map[string]interface{}{
+		// Functions exposed to the template
+		"AttributesOf":               attributesOf,
+		"ConstantsOf":                constantsOf,
+		"EnumEntryName":              enumEntryName,
+		"Eval":                       g.eval,
+		"Include":                    g.include,
+		"IsBasicLiteral":             is(ast.BasicLiteral{}),
+		"IsConstructor":              isConstructor,
+		"IsDefaultDictionaryLiteral": is(ast.DefaultDictionaryLiteral{}),
+		"IsDictionary":               is(ast.Dictionary{}),
+		"IsEnum":                     is(ast.Enum{}),
+		"IsInterface":                is(ast.Interface{}),
+		"IsInterfaceOrNamespace":     is(ast.Interface{}, ast.Namespace{}),
+		"IsMember":                   is(ast.Member{}),
+		"IsNamespace":                is(ast.Namespace{}),
+		"IsNullableType":             is(ast.NullableType{}),
+		"IsParametrizedType":         is(ast.ParametrizedType{}),
+		"IsRecordType":               is(ast.RecordType{}),
+		"IsSequenceType":             is(ast.SequenceType{}),
+		"IsTypedef":                  is(ast.Typedef{}),
+		"IsTypeName":                 is(ast.TypeName{}),
+		"IsUndefinedType":            isUndefinedType,
+		"IsUnionType":                is(ast.UnionType{}),
+		"Lookup":                     g.lookup,
+		"MethodsOf":                  methodsOf,
+		"Title":                      strings.Title,
+	}
+	t, err := g.t.
+		Option("missingkey=invalid").
+		Funcs(g.funcs).
+		Parse(string(tmpl))
+	if err != nil {
+		return fmt.Errorf("failed to parse template file '%v': %w", templatePath, err)
+	}
+
+	// simplify the definitions in the WebIDL before passing this to the template
+	idl, declarations := simplify(idl)
+	g.declarations = declarations
+
+	// Write the file header
+	fmt.Fprintf(out, header, strings.Join(os.Args[1:], "\n//   "))
+
+	// Execute the template
+	return t.Execute(out, idl)
+}
+
+// declarations is a map of WebIDL declaration name to its AST node.
+type declarations map[string]ast.Decl
+
+// nameOf returns the name of the AST node n.
+// Returns an empty string if the node is not named.
+func nameOf(n ast.Node) string {
+	switch n := n.(type) {
+	case *ast.Namespace:
+		return n.Name
+	case *ast.Interface:
+		return n.Name
+	case *ast.Dictionary:
+		return n.Name
+	case *ast.Enum:
+		return n.Name
+	case *ast.Typedef:
+		return n.Name
+	case *ast.Mixin:
+		return n.Name
+	case *ast.Includes:
+		return ""
+	default:
+		panic(fmt.Errorf("unhandled AST declaration %T", n))
+	}
+}
+
+// simplify processes the AST 'in', returning a new AST that:
+// * Has all partial interfaces merged into a single interface.
+// * Has all mixins flattened into their place of use.
+// * Has all the declarations ordered in dependency order (leaf first)
+// simplify also returns the map of declarations in the AST.
+func simplify(in *ast.File) (*ast.File, declarations) {
+	s := simplifier{
+		declarations: declarations{},
+		registered:   map[string]bool{},
+		out:          &ast.File{},
+	}
+
+	// Walk the IDL declarations to merge together partial interfaces and embed
+	// mixins into their uses.
+	{
+		interfaces := map[string]*ast.Interface{}
+		mixins := map[string]*ast.Mixin{}
+		for _, d := range in.Declarations {
+			switch d := d.(type) {
+			case *ast.Interface:
+				if i, ok := interfaces[d.Name]; ok {
+					// Merge partial body into one interface
+					i.Members = append(i.Members, d.Members...)
+				} else {
+					clone := *d
+					d := &clone
+					interfaces[d.Name] = d
+					s.declarations[d.Name] = d
+				}
+			case *ast.Mixin:
+				mixins[d.Name] = d
+				s.declarations[d.Name] = d
+			case *ast.Includes:
+				// Merge mixin into interface
+				i, ok := interfaces[d.Name]
+				if !ok {
+					panic(fmt.Errorf("%v includes %v, but %v is not an interface", d.Name, d.Source, d.Name))
+				}
+				m, ok := mixins[d.Source]
+				if !ok {
+					panic(fmt.Errorf("%v includes %v, but %v is not an mixin", d.Name, d.Source, d.Source))
+				}
+				// Merge mixin into the interface
+				for _, member := range m.Members {
+					if member, ok := member.(*ast.Member); ok {
+						i.Members = append(i.Members, member)
+					}
+				}
+			default:
+				if name := nameOf(d); name != "" {
+					s.declarations[nameOf(d)] = d
+				}
+			}
+		}
+	}
+
+	// Now traverse the declarations in to produce the dependency-ordered
+	// output `s.out`.
+	for _, d := range in.Declarations {
+		if name := nameOf(d); name != "" {
+			s.visit(s.declarations[nameOf(d)])
+		}
+	}
+
+	return s.out, s.declarations
+}
+
+// simplifier holds internal state for simplify()
+type simplifier struct {
+	// all AST declarations
+	declarations declarations
+	// set of visited declarations
+	registered map[string]bool
+	// the dependency-ordered output
+	out *ast.File
+}
+
+// visit traverses the AST declaration 'd' adding all dependent declarations to
+// s.out.
+func (s *simplifier) visit(d ast.Decl) {
+	register := func(name string) bool {
+		if s.registered[name] {
+			return true
+		}
+		s.registered[name] = true
+		return false
+	}
+	switch d := d.(type) {
+	case *ast.Namespace:
+		if register(d.Name) {
+			return
+		}
+		for _, m := range d.Members {
+			if m, ok := m.(*ast.Member); ok {
+				s.visitType(m.Type)
+				for _, p := range m.Parameters {
+					s.visitType(p.Type)
+				}
+			}
+		}
+	case *ast.Interface:
+		if register(d.Name) {
+			return
+		}
+		if d, ok := s.declarations[d.Inherits]; ok {
+			s.visit(d)
+		}
+		for _, m := range d.Members {
+			if m, ok := m.(*ast.Member); ok {
+				s.visitType(m.Type)
+				for _, p := range m.Parameters {
+					s.visitType(p.Type)
+				}
+			}
+		}
+	case *ast.Dictionary:
+		if register(d.Name) {
+			return
+		}
+		if d, ok := s.declarations[d.Inherits]; ok {
+			s.visit(d)
+		}
+		for _, m := range d.Members {
+			s.visitType(m.Type)
+			for _, p := range m.Parameters {
+				s.visitType(p.Type)
+			}
+		}
+	case *ast.Typedef:
+		if register(d.Name) {
+			return
+		}
+		s.visitType(d.Type)
+	case *ast.Mixin:
+		if register(d.Name) {
+			return
+		}
+		for _, m := range d.Members {
+			if m, ok := m.(*ast.Member); ok {
+				s.visitType(m.Type)
+				for _, p := range m.Parameters {
+					s.visitType(p.Type)
+				}
+			}
+		}
+	case *ast.Enum:
+		if register(d.Name) {
+			return
+		}
+	case *ast.Includes:
+		if register(d.Name) {
+			return
+		}
+	default:
+		panic(fmt.Errorf("unhandled AST declaration %T", d))
+	}
+
+	s.out.Declarations = append(s.out.Declarations, d)
+}
+
+// visitType traverses the AST type 't' adding all dependent declarations to
+// s.out.
+func (s *simplifier) visitType(t ast.Type) {
+	switch t := t.(type) {
+	case *ast.TypeName:
+		if d, ok := s.declarations[t.Name]; ok {
+			s.visit(d)
+		}
+	case *ast.UnionType:
+		for _, t := range t.Types {
+			s.visitType(t)
+		}
+	case *ast.ParametrizedType:
+		for _, t := range t.Elems {
+			s.visitType(t)
+		}
+	case *ast.NullableType:
+		s.visitType(t.Type)
+	case *ast.SequenceType:
+		s.visitType(t.Elem)
+	case *ast.RecordType:
+		s.visitType(t.Elem)
+	default:
+		panic(fmt.Errorf("unhandled AST type %T", t))
+	}
+}
+
+// generator holds the template generator state
+type generator struct {
+	// the root template
+	t *template.Template
+	// the working directory
+	workingDir string
+	// map of function name to function exposed to the template executor
+	funcs map[string]interface{}
+	// dependency-sorted declarations
+	declarations declarations
+}
+
+// eval executes the sub-template with the given name and arguments, returning
+// the generated output
+// args can be a single argument:
+//   arg[0]
+// or a list of name-value pairs:
+//   (args[0]: name, args[1]: value), (args[2]: name, args[3]: value)...
+func (g *generator) eval(template string, args ...interface{}) (string, error) {
+	target := g.t.Lookup(template)
+	if target == nil {
+		return "", fmt.Errorf("template '%v' not found", template)
+	}
+	sb := strings.Builder{}
+	var err error
+	if len(args) == 1 {
+		err = target.Execute(&sb, args[0])
+	} else {
+		m := newMap()
+		if len(args)%2 != 0 {
+			return "", fmt.Errorf("Eval expects a single argument or list name-value pairs")
+		}
+		for i := 0; i < len(args); i += 2 {
+			name, ok := args[i].(string)
+			if !ok {
+				return "", fmt.Errorf("Eval argument %v is not a string", i)
+			}
+			m.Put(name, args[i+1])
+		}
+		err = target.Execute(&sb, m)
+	}
+	if err != nil {
+		return "", fmt.Errorf("while evaluating '%v': %v", template, err)
+	}
+	return sb.String(), nil
+}
+
+// lookup returns the declaration with the given name, or nil if not found.
+func (g *generator) lookup(name string) ast.Decl {
+	return g.declarations[name]
+}
+
+// include loads the template with the given path, importing the declarations
+// into the scope of the current template.
+func (g *generator) include(path string) (string, error) {
+	t, err := g.t.
+		Option("missingkey=invalid").
+		Funcs(g.funcs).
+		ParseFiles(filepath.Join(g.workingDir, path))
+	if err != nil {
+		return "", err
+	}
+	g.t.AddParseTree(path, t.Tree)
+	return "", nil
+}
+
+// Map is a simple generic key-value map, which can be used in the template
+type Map map[interface{}]interface{}
+
+func newMap() Map { return Map{} }
+
+// Put adds the key-value pair into the map.
+// Put always returns an empty string so nothing is printed in the template.
+func (m Map) Put(key, value interface{}) string {
+	m[key] = value
+	return ""
+}
+
+// Get looks up and returns the value with the given key. If the map does not
+// contain the given key, then nil is returned.
+func (m Map) Get(key interface{}) interface{} {
+	return m[key]
+}
+
+// is returns a function that returns true if the value passed to the function
+// matches any of the types of the objects in 'prototypes'.
+func is(prototypes ...interface{}) func(interface{}) bool {
+	types := make([]reflect.Type, len(prototypes))
+	for i, p := range prototypes {
+		types[i] = reflect.TypeOf(p)
+	}
+	return func(v interface{}) bool {
+		ty := reflect.TypeOf(v)
+		for _, rty := range types {
+			if ty == rty || ty == reflect.PtrTo(rty) {
+				return true
+			}
+		}
+		return false
+	}
+}
+
+// isConstructor returns true if the object is a constructor ast.Member.
+func isConstructor(v interface{}) bool {
+	if member, ok := v.(*ast.Member); ok {
+		if ty, ok := member.Type.(*ast.TypeName); ok {
+			return ty.Name == "constructor"
+		}
+	}
+	return false
+}
+
+// isUndefinedType returns true if the type is 'undefined'
+func isUndefinedType(ty ast.Type) bool {
+	if ty, ok := ty.(*ast.TypeName); ok {
+		return ty.Name == "undefined"
+	}
+	return false
+}
+
+// enumEntryName formats the enum entry name 's' for use in a C++ enum.
+func enumEntryName(s string) string {
+	return "k" + strings.ReplaceAll(pascalCase(strings.Trim(s, `"`)), "-", "")
+}
+
+// Method describes a WebIDL interface method
+type Method struct {
+	// Name of the method
+	Name string
+	// The list of overloads of the method
+	Overloads []*ast.Member
+}
+
+// methodsOf returns all the methods of the given WebIDL interface.
+func methodsOf(obj interface{}) []*Method {
+	iface, ok := obj.(*ast.Interface)
+	if !ok {
+		return nil
+	}
+	byName := map[string]*Method{}
+	out := []*Method{}
+	for _, member := range iface.Members {
+		member := member.(*ast.Member)
+		if !member.Const && !member.Attribute && !isConstructor(member) {
+			if method, ok := byName[member.Name]; ok {
+				method.Overloads = append(method.Overloads, member)
+			} else {
+				method = &Method{
+					Name:      member.Name,
+					Overloads: []*ast.Member{member},
+				}
+				byName[member.Name] = method
+				out = append(out, method)
+			}
+		}
+	}
+	return out
+}
+
+// attributesOf returns all the attributes of the given WebIDL interface or
+// namespace.
+func attributesOf(obj interface{}) []*ast.Member {
+	out := []*ast.Member{}
+	add := func(m interface{}) {
+		if m := m.(*ast.Member); m.Attribute {
+			out = append(out, m)
+		}
+	}
+	switch obj := obj.(type) {
+	case *ast.Interface:
+		for _, m := range obj.Members {
+			add(m)
+		}
+	case *ast.Namespace:
+		for _, m := range obj.Members {
+			add(m)
+		}
+	default:
+		return nil
+	}
+	return out
+}
+
+// constantsOf returns all the constant values of the given WebIDL interface or
+// namespace.
+func constantsOf(obj interface{}) []*ast.Member {
+	out := []*ast.Member{}
+	add := func(m interface{}) {
+		if m := m.(*ast.Member); m.Const {
+			out = append(out, m)
+		}
+	}
+	switch obj := obj.(type) {
+	case *ast.Interface:
+		for _, m := range obj.Members {
+			add(m)
+		}
+	case *ast.Namespace:
+		for _, m := range obj.Members {
+			add(m)
+		}
+	default:
+		return nil
+	}
+	return out
+}
+
+// pascalCase returns the snake-case string s transformed into 'PascalCase',
+// Rules:
+// * The first letter of the string is capitalized
+// * Characters following an underscore, hyphen or number are capitalized
+// * Underscores are removed from the returned string
+// See: https://en.wikipedia.org/wiki/Camel_case
+func pascalCase(s string) string {
+	b := strings.Builder{}
+	upper := true
+	for _, r := range s {
+		if r == '_' || r == '-' {
+			upper = true
+			continue
+		}
+		if upper {
+			b.WriteRune(unicode.ToUpper(r))
+			upper = false
+		} else {
+			b.WriteRune(r)
+		}
+		if unicode.IsNumber(r) {
+			upper = true
+		}
+	}
+	return b.String()
+}
+
+const header = `// Copyright 2021 The Dawn Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+////////////////////////////////////////////////////////////////////////////////
+// File generated by tools/cmd/idlgen.go, with the arguments:
+//   %v
+//
+// Do not modify this file directly
+////////////////////////////////////////////////////////////////////////////////
+
+`
--- a/src/dawn_node/tools/go.mod
+++ b/src/dawn_node/tools/go.mod
@ -0,0 +1,5 @@
+module dawn.googlesource.com/dawn/src/dawn_node/tools
+
+go 1.16
+
+require github.com/ben-clayton/webidlparser v0.0.0-20210923100217-8ba896ded094
--- a/src/dawn_node/tools/go.sum
+++ b/src/dawn_node/tools/go.sum
@ -0,0 +1,26 @@
+github.com/ben-clayton/webidlparser v0.0.0-20210920164050-74f1f9d55323 h1:z6T07jitSi9kfzvR2cHgRQgp32vJsIPMGJDJiWeShok=
+github.com/ben-clayton/webidlparser v0.0.0-20210920164050-74f1f9d55323/go.mod h1:bV550SPlMos7UhMprxlm14XTBTpKHSUZ8Q4Id5qQuyw=
+github.com/ben-clayton/webidlparser v0.0.0-20210923100217-8ba896ded094 h1:CTVJdI6oUCRNucMEmoh3c2U88DesoPtefsxKhoZ1WuQ=
+github.com/ben-clayton/webidlparser v0.0.0-20210923100217-8ba896ded094/go.mod h1:bV550SPlMos7UhMprxlm14XTBTpKHSUZ8Q4Id5qQuyw=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
+github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
+github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=