intrinsic-gen: Additional functionality

* Add new template utilty functions, including the ability to split out multiple files.
* Add basic printing of the semantic overloads.
* Add a pointer from the overload to the function
* Change TemplateArguments from a list of FQN to a list of interface{} (any).  This is required as once the overload is permutated, some arguments will need to hold integers.

This will be used by the test generator.

Bug: tint:832
Change-Id: Idbfbe85e52489b31850cbb0ee7430bb4b021530e
Reviewed-on: https://dawn-review.googlesource.com/c/tint/+/53046
Kokoro: Kokoro <noreply+kokoro@google.com>
Commit-Queue: Ben Clayton <bclayton@google.com>
Reviewed-by: Antonio Maiorano <amaiorano@google.com>

src/intrinsic_table.inl

@@ -14,6 +14,11 @@
 
 ////////////////////////////////////////////////////////////////////////////////
 // File generated by tools/intrinsic-gen
+// using the template:
+//   src/intrinsic_table.inl.tmpl
+// and the intrinsic defintion file:
+//   src/intrinsics.def
+//
 // Do not modify this file directly
 ////////////////////////////////////////////////////////////////////////////////
 

src/sem/intrinsic_type.cc

@@ -14,6 +14,11 @@
 
 ////////////////////////////////////////////////////////////////////////////////
 // File generated by tools/intrinsic-gen
+// using the template:
+//   src/sem/intrinsic_type.cc.tmpl
+// and the intrinsic defintion file:
+//   src/intrinsics.def
+//
 // Do not modify this file directly
 ////////////////////////////////////////////////////////////////////////////////
 

src/sem/intrinsic_type.h

@@ -14,6 +14,11 @@
 
 ////////////////////////////////////////////////////////////////////////////////
 // File generated by tools/intrinsic-gen
+// using the template:
+//   src/sem/intrinsic_type.h.tmpl
+// and the intrinsic defintion file:
+//   src/intrinsics.def
+//
 // Do not modify this file directly
 ////////////////////////////////////////////////////////////////////////////////
 

src/sem/parameter_usage.cc

@@ -14,6 +14,11 @@
 
 ////////////////////////////////////////////////////////////////////////////////
 // File generated by tools/intrinsic-gen
+// using the template:
+//   src/sem/parameter_usage.cc.tmpl
+// and the intrinsic defintion file:
+//   src/intrinsics.def
+//
 // Do not modify this file directly
 ////////////////////////////////////////////////////////////////////////////////
 

src/sem/parameter_usage.h

@@ -14,6 +14,11 @@
 
 ////////////////////////////////////////////////////////////////////////////////
 // File generated by tools/intrinsic-gen
+// using the template:
+//   src/sem/parameter_usage.h.tmpl
+// and the intrinsic defintion file:
+//   src/intrinsics.def
+//
 // Do not modify this file directly
 ////////////////////////////////////////////////////////////////////////////////
 

tools/src/cmd/intrinsic-gen/gen/generate.go

@@ -27,46 +27,74 @@ import (
 
 type generator struct {
 	s      *sem.Sem
+	t      *template.Template
 	cached struct {
 		intrinsicTable *IntrinsicTable // lazily built by intrinsicTable()
 	}
 }
 
+// WriteFile is a function that Generate() may call to emit a new file from a
+// template.
+// relpath is the relative path from the currently executing template.
+// content is the file content to write.
+type WriteFile func(relpath, content string) error
+
 // Generate executes the template tmpl using the provided semantic
 // information, writing the output to w.
 // See https://golang.org/pkg/text/template/ for documentation on the template
 // syntax.
-func Generate(s *sem.Sem, tmpl string, w io.Writer) error {
+func Generate(s *sem.Sem, tmpl string, w io.Writer, writeFile WriteFile) error {
 	g := generator{s: s}
-	return g.generate(tmpl, w)
+	return g.generate(tmpl, w, writeFile)
 }
 
-func (g *generator) generate(tmpl string, w io.Writer) error {
+func (g *generator) generate(tmpl string, w io.Writer, writeFile WriteFile) error {
-	t, err := template.
+	t, err := template.New("<template>").Funcs(map[string]interface{}{
-		New("<template>").
-		Funcs(map[string]interface{}{
 		"Map":                   newMap,
 		"Iterate":               iterate,
 		"Title":                 strings.Title,
 		"PascalCase":            pascalCase,
 		"SplitDisplayName":      splitDisplayName,
-			"IsTemplateTypeParam":   is(&sem.TemplateTypeParam{}),
+		"HasPrefix":             strings.HasPrefix,
-			"IsTemplateNumberParam": is(&sem.TemplateNumberParam{}),
+		"HasSuffix":             strings.HasSuffix,
-			"IsTemplateEnumParam":   is(&sem.TemplateEnumParam{}),
+		"IsEnumEntry":           is(sem.EnumEntry{}),
+		"IsEnumMatcher":         is(sem.EnumMatcher{}),
+		"IsFQN":                 is(sem.FullyQualifiedName{}),
+		"IsInt":                 is(1),
+		"IsTemplateEnumParam":   is(sem.TemplateEnumParam{}),
+		"IsTemplateNumberParam": is(sem.TemplateNumberParam{}),
+		"IsTemplateTypeParam":   is(sem.TemplateTypeParam{}),
+		"IsType":                is(sem.Type{}),
 		"IsFirstIn":             isFirstIn,
 		"IsLastIn":              isLastIn,
 		"IntrinsicTable":        g.intrinsicTable,
-		}).
+		"Eval":                  g.eval,
-		Option("missingkey=error").
+		"WriteFile":             func(relpath, content string) (string, error) { return "", writeFile(relpath, content) },
+	}).Option("missingkey=error").
 		Parse(tmpl)
 	if err != nil {
 		return err
 	}
+	g.t = t
 	return t.Execute(w, map[string]interface{}{
 		"Sem": g.s,
 	})
 }
 
+// eval executes the sub-template with the given name and argument, returning
+// the generated output
+func (g *generator) eval(template string, arg interface{}) (string, error) {
+	target := g.t.Lookup(template)
+	if target == nil {
+		return "", fmt.Errorf("template '%v' not found", template)
+	}
+	sb := strings.Builder{}
+	if err := target.Execute(&sb, arg); err != nil {
+		return "", fmt.Errorf("while evaluating '%v': %v", template, err)
+	}
+	return sb.String(), nil
+}
+
 // intrinsicTable lazily calls and returns the result of buildIntrinsicTable(),
 // caching the result for repeated calls.
 func (g *generator) intrinsicTable() (*IntrinsicTable, error) {
@@ -103,7 +131,8 @@ func (m Map) Get(key interface{}) interface{} {
 func is(ty interface{}) func(interface{}) bool {
 	rty := reflect.TypeOf(ty)
 	return func(v interface{}) bool {
-		return reflect.TypeOf(v) == rty
+		ty := reflect.TypeOf(v)
+		return ty == rty || ty == reflect.PtrTo(rty)
 	}
 }
 

tools/src/cmd/intrinsic-gen/gen/intrinsic_table.go

@@ -327,7 +327,7 @@ func (b *overloadBuilder) collectMatcherIndices(fqn sem.FullyQualifiedName) ([]i
 	}
 	out := []int{idx}
 	for _, arg := range fqn.TemplateArguments {
-		indices, err := b.collectMatcherIndices(arg)
+		indices, err := b.collectMatcherIndices(arg.(sem.FullyQualifiedName))
 		if err != nil {
 			return nil, err
 		}

tools/src/cmd/intrinsic-gen/main.go

@@ -80,18 +80,19 @@ func run() error {
 	}
 
 	// Recursively find all the template files in the <tint>/src directory
-	srcDir := filepath.Join(projectRoot, "src")
+	files, err := glob.Scan(projectRoot, glob.MustParseConfig(`{
-	files, err := glob.Scan(srcDir, glob.MustParseConfig(`{
+		"paths": [{"include": [
-		"paths": [{"include": [ "**.tmpl" ]}]
+			"src/**.tmpl"
+		]}]
 	}`))
 	if err != nil {
 		return err
 	}
 
 	// For each template file...
-	for _, tmplPath := range files {
+	for _, relTmplPath := range files {
 		// Make tmplPath absolute
-		tmplPath := filepath.Join(srcDir, tmplPath)
+		tmplPath := filepath.Join(projectRoot, relTmplPath)
 
 		// Read the template file
 		tmpl, err := ioutil.ReadFile(tmplPath)
@@ -99,31 +100,49 @@ func run() error {
 			return fmt.Errorf("failed to open '%v': %w", tmplPath, err)
 		}
 
+		// Create or update the file at relpath if the file content has changed
+		// relpath is a path relative to the template
+		writeFile := func(relpath, body string) error {
 			// Write the common file header
 			sb := strings.Builder{}
-		sb.WriteString(header)
+			sb.WriteString(fmt.Sprintf(header, relTmplPath, defProjectRelPath))
+			sb.WriteString(body)
+			content := sb.String()
+			abspath := filepath.Join(filepath.Dir(tmplPath), relpath)
+			return writeFileIfChanged(abspath, content)
+		}
 
 		// Write the content generated using the template and semantic info
-		if err := gen.Generate(sem, string(tmpl), &sb); err != nil {
+		sb := strings.Builder{}
+		if err := gen.Generate(sem, string(tmpl), &sb, writeFile); err != nil {
 			return fmt.Errorf("while processing '%v': %w", tmplPath, err)
 		}
 
-		// Create or update the output file if the content has not changed
+		if body := sb.String(); body != "" {
-		filePath := strings.TrimSuffix(tmplPath, ".tmpl")
+			_, tmplFileName := filepath.Split(tmplPath)
-		if err := writeFileIfChanged(filePath, sb.String()); err != nil {
+			outFileName := strings.TrimSuffix(tmplFileName, ".tmpl")
-			return fmt.Errorf("failed to write '%v': %w", filePath, err)
+			if err := writeFile(outFileName, body); err != nil {
+				return err
+			}
 		}
 	}
 
 	return nil
 }
 
+// writes content to path if the file has changed
 func writeFileIfChanged(path, content string) error {
 	existing, err := ioutil.ReadFile(path)
 	if err == nil && string(existing) == content {
 		return nil // Not changed
 	}
-	return ioutil.WriteFile(path, []byte(content), 0666)
+	if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
+		return fmt.Errorf("failed to create directory for '%v': %w", path, err)
+	}
+	if err := ioutil.WriteFile(path, []byte(content), 0666); err != nil {
+		return fmt.Errorf("failed to write file '%v': %w", path, err)
+	}
+	return nil
 }
 
 const header = `// Copyright 2021 The Tint Authors.
@@ -142,6 +161,11 @@ const header = `// Copyright 2021 The Tint Authors.
 
 ////////////////////////////////////////////////////////////////////////////////
 // File generated by tools/intrinsic-gen
+// using the template:
+//   %v
+// and the intrinsic defintion file:
+//   %v
+//
 // Do not modify this file directly
 ////////////////////////////////////////////////////////////////////////////////
 

tools/src/cmd/intrinsic-gen/resolver/resolve.go

@@ -246,6 +246,7 @@ func (r *resolver) function(a ast.FunctionDecl) error {
 	// Construct the semantic overload and append it to the function
 	overload := &sem.Overload{
 		Decl:           a,
+		Function:       f,
 		Parameters:     make([]sem.Parameter, len(a.Parameters)),
 		TemplateParams: templateParams,
 	}
@@ -335,7 +336,7 @@ func (r *resolver) fullyQualifiedName(s *scope, arg ast.TemplatedName) (sem.Full
 
 	fqn := sem.FullyQualifiedName{
 		Target:            target,
-		TemplateArguments: make([]sem.FullyQualifiedName, len(arg.TemplateArgs)),
+		TemplateArguments: make([]interface{}, len(arg.TemplateArgs)),
 	}
 	for i, a := range arg.TemplateArgs {
 		arg, err := r.fullyQualifiedName(s, a)

tools/src/cmd/intrinsic-gen/sem/sem.go

@@ -15,6 +15,8 @@
 package sem
 
 import (
+	"fmt"
+
 	"dawn.googlesource.com/tint/tools/src/cmd/intrinsic-gen/ast"
 )
 
@@ -68,6 +70,14 @@ type EnumEntry struct {
 	IsInternal bool // True if this entry is not part of the WGSL grammar
 }
 
+// Format implements the fmt.Formatter interface
+func (e EnumEntry) Format(w fmt.State, verb rune) {
+	if e.IsInternal {
+		fmt.Fprint(w, "[[internal]] ")
+	}
+	fmt.Fprint(w, e.Name)
+}
+
 // Type declares a type
 type Type struct {
 	TemplateParams []TemplateParam
@@ -120,6 +130,7 @@ type Function struct {
 // Overload describes a single overload of a function
 type Overload struct {
 	Decl           ast.FunctionDecl
+	Function       *Function
 	TemplateParams []TemplateParam
 	OpenTypes      []*TemplateTypeParam
 	OpenNumbers    []TemplateParam
@@ -127,16 +138,65 @@ type Overload struct {
 	Parameters     []Parameter
 }
 
+// Format implements the fmt.Formatter interface
+func (o Overload) Format(w fmt.State, verb rune) {
+	fmt.Fprintf(w, "fn %v", o.Function.Name)
+	if len(o.TemplateParams) > 0 {
+		fmt.Fprintf(w, "<")
+		for i, t := range o.TemplateParams {
+			if i > 0 {
+				fmt.Fprint(w, ", ")
+			}
+			fmt.Fprintf(w, "%v", t)
+		}
+		fmt.Fprintf(w, ">")
+	}
+	fmt.Fprint(w, "(")
+	for i, p := range o.Parameters {
+		if i > 0 {
+			fmt.Fprint(w, ", ")
+		}
+		fmt.Fprintf(w, "%v", p)
+	}
+	fmt.Fprint(w, ")")
+	if o.ReturnType != nil {
+		fmt.Fprintf(w, " -> %v", o.ReturnType)
+	}
+}
+
 // Parameter describes a single parameter of a function overload
 type Parameter struct {
 	Name string
 	Type FullyQualifiedName
 }
 
+// Format implements the fmt.Formatter interface
+func (p Parameter) Format(w fmt.State, verb rune) {
+	if p.Name != "" {
+		fmt.Fprintf(w, "%v: ", p.Name)
+	}
+	fmt.Fprintf(w, "%v", p.Type)
+}
+
 // FullyQualifiedName is the usage of a Type, TypeMatcher or TemplateTypeParam
 type FullyQualifiedName struct {
 	Target            Named
-	TemplateArguments []FullyQualifiedName
+	TemplateArguments []interface{}
+}
+
+// Format implements the fmt.Formatter interface
+func (f FullyQualifiedName) Format(w fmt.State, verb rune) {
+	fmt.Fprint(w, f.Target.GetName())
+	if len(f.TemplateArguments) > 0 {
+		fmt.Fprintf(w, "<")
+		for i, t := range f.TemplateArguments {
+			if i > 0 {
+				fmt.Fprint(w, ", ")
+			}
+			fmt.Fprintf(w, "%v", t)
+		}
+		fmt.Fprintf(w, ">")
+	}
 }
 
 // TemplateParam is a TemplateEnumParam, TemplateTypeParam or TemplateNumberParam