tools: Shuffle 'intrinsic-gen' tooling

Rename to 'gen', so that more templating can be added without having a confusing name.

Can now be run with './tools/run gen'

Move the bulk of the intrinsic-gen logic to `tools/src/tint/intrinsic`

Change-Id: I750989a5aa86272c10c2ad37adffe7def11c61f2
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/97141
Commit-Queue: Ben Clayton <bclayton@google.com>
Kokoro: Kokoro <noreply+kokoro@google.com>
Reviewed-by: Antonio Maiorano <amaiorano@google.com>
Reviewed-by: Dan Sinclair <dsinclair@chromium.org>

src/tint/resolver/ctor_conv_intrinsic.cc

@@ -13,11 +13,9 @@
 // limitations under the License.
 
 ////////////////////////////////////////////////////////////////////////////////
-// File generated by tools/intrinsic-gen
+// File generated by tools/src/cmd/gen
 // using the template:
 //   src/tint/resolver/ctor_conv_intrinsic.cc.tmpl
-// and the intrinsic defintion file:
-//   src/tint/intrinsics.def
 //
 // Do not modify this file directly
 ////////////////////////////////////////////////////////////////////////////////

src/tint/resolver/ctor_conv_intrinsic.cc.tmpl

@@ -1,9 +1,9 @@
 {{- /*
 --------------------------------------------------------------------------------
-Template file for use with tools/builtin-gen to generate ctor_conv_intrinsic.cc
+Template file for use with tools/src/cmd/gen to generate ctor_conv_intrinsic.cc
 
 See:
-* tools/cmd/intrinsic-gen/gen for structures used by this template
+* tools/src/cmd/gen for structures used by this template
 * https://golang.org/pkg/text/template/ for documentation on the template syntax
 --------------------------------------------------------------------------------
 */ -}}
@@ -16,7 +16,7 @@ const char* str(CtorConvIntrinsic i) {
     switch (i) {
         case CtorConvIntrinsic::kNone:
             return "<none>";
-{{- range .Sem.ConstructorsAndConverters  }}
+{{- range Sem.ConstructorsAndConverters  }}
         case CtorConvIntrinsic::k{{Title .Name}}:
             return "{{.Name}}";
 {{- end  }}

src/tint/resolver/ctor_conv_intrinsic.h

@@ -13,11 +13,9 @@
 // limitations under the License.
 
 ////////////////////////////////////////////////////////////////////////////////
-// File generated by tools/intrinsic-gen
+// File generated by tools/src/cmd/gen
 // using the template:
 //   src/tint/resolver/ctor_conv_intrinsic.h.tmpl
-// and the intrinsic defintion file:
-//   src/tint/intrinsics.def
 //
 // Do not modify this file directly
 ////////////////////////////////////////////////////////////////////////////////

src/tint/resolver/ctor_conv_intrinsic.h.tmpl

@@ -1,9 +1,9 @@
 {{- /*
 --------------------------------------------------------------------------------
-Template file for use with tools/builtin-gen to generate ctor_conv_intrinsic.h
+Template file for use with tools/src/cmd/gen to generate ctor_conv_intrinsic.h
 
 See:
-* tools/cmd/intrinsic-gen/gen for structures used by this template
+* tools/src/cmd/gen for structures used by this template
 * https://golang.org/pkg/text/template/ for documentation on the template syntax
 --------------------------------------------------------------------------------
 */ -}}
@@ -19,7 +19,7 @@ namespace tint::resolver {
 /// declared in the intrinsic table.
 enum class CtorConvIntrinsic {
     kNone = -1,
-{{- range .Sem.ConstructorsAndConverters }}
+{{- range Sem.ConstructorsAndConverters }}
     k{{Title .Name}},
 {{- end }}
 };

src/tint/resolver/intrinsic_table.inl

@@ -13,11 +13,9 @@
 // limitations under the License.
 
 ////////////////////////////////////////////////////////////////////////////////
-// File generated by tools/intrinsic-gen
+// File generated by tools/src/cmd/gen
 // using the template:
 //   src/tint/resolver/intrinsic_table.inl.tmpl
-// and the intrinsic defintion file:
-//   src/tint/intrinsics.def
 //
 // Do not modify this file directly
 ////////////////////////////////////////////////////////////////////////////////

src/tint/resolver/intrinsic_table.inl.tmpl

@@ -1,17 +1,17 @@
 {{- /*
 --------------------------------------------------------------------------------
-Template file for use with tools/builtin-gen to generate builtin_table.inl
+Template file for use with tools/src/cmd/gen to generate builtin_table.inl
 Used by BuiltinTable.cc for builtin overload resolution.
 
 See:
-* tools/cmd/intrinsic-gen/gen for structures used by this template
+* tools/src/cmd/gen for structures used by this template
 * https://golang.org/pkg/text/template/ for documentation on the template syntax
 --------------------------------------------------------------------------------
 */ -}}
 
 // clang-format off
 
-{{  with .Sem -}}
+{{  with Sem -}}
 {{    range .Types -}}
 {{      template "Type" . }}
 {{    end -}}
@@ -339,27 +339,27 @@ class Matchers {
  private:
 {{- $t_names := Map -}}
 {{- $n_names := Map -}}
-{{- range Iterate .Sem.MaxTemplateTypes -}}
+{{- range Iterate Sem.MaxTemplateTypes -}}
 {{-   $name := printf "template_type_%v" . -}}
 {{-   $t_names.Put . $name }}
   TemplateTypeMatcher {{$name}}_{ {{- . -}} };
 {{- end }}
-{{- range Iterate .Sem.MaxTemplateNumbers -}}
+{{- range Iterate Sem.MaxTemplateNumbers -}}
 {{-   $name := printf "template_number_%v" . -}}
 {{-   $n_names.Put . $name }}
   TemplateNumberMatcher {{$name}}_{ {{- . -}} };
 {{- end }}
-{{- range .Sem.Types -}}
+{{- range Sem.Types -}}
 {{-   $name := PascalCase .Name -}}
 {{-   $t_names.Put . $name }}
   {{$name}} {{$name}}_;
 {{- end }}
-{{- range .Sem.TypeMatchers -}}
+{{- range Sem.TypeMatchers -}}
 {{-   $name := PascalCase .Name -}}
 {{-   $t_names.Put . $name }}
   {{$name}} {{$name}}_;
 {{- end }}
-{{- range .Sem.EnumMatchers -}}
+{{- range Sem.EnumMatchers -}}
 {{-   $name := PascalCase .Name -}}
 {{-   $n_names.Put . $name }}
   {{$name}} {{$name}}_;

src/tint/sem/builtin_type.cc

@@ -13,11 +13,9 @@
 // limitations under the License.
 
 ////////////////////////////////////////////////////////////////////////////////
-// File generated by tools/intrinsic-gen
+// File generated by tools/src/cmd/gen
 // using the template:
 //   src/tint/sem/builtin_type.cc.tmpl
-// and the intrinsic defintion file:
-//   src/tint/intrinsics.def
 //
 // Do not modify this file directly
 ////////////////////////////////////////////////////////////////////////////////

src/tint/sem/builtin_type.cc.tmpl

@@ -1,9 +1,9 @@
 {{- /*
 --------------------------------------------------------------------------------
-Template file for use with tools/builtin-gen to generate builtin_type.cc
+Template file for use with tools/src/cmd/gen to generate builtin_type.cc
 
 See:
-* tools/cmd/intrinsic-gen/gen for structures used by this template
+* tools/src/cmd/gen for structures used by this template
 * https://golang.org/pkg/text/template/ for documentation on the template syntax
 --------------------------------------------------------------------------------
 */ -}}
@@ -15,7 +15,7 @@ See:
 namespace tint::sem {
 
 BuiltinType ParseBuiltinType(const std::string& name) {
-{{- range .Sem.Builtins  }}
+{{- range Sem.Builtins  }}
     if (name == "{{.Name}}") {
         return BuiltinType::k{{Title .Name}};
     }
@@ -27,7 +27,7 @@ const char* str(BuiltinType i) {
     switch (i) {
         case BuiltinType::kNone:
             return "<none>";
-{{- range .Sem.Builtins  }}
+{{- range Sem.Builtins  }}
         case BuiltinType::k{{Title .Name}}:
             return "{{.Name}}";
 {{- end  }}

src/tint/sem/builtin_type.h

@@ -13,11 +13,9 @@
 // limitations under the License.
 
 ////////////////////////////////////////////////////////////////////////////////
-// File generated by tools/intrinsic-gen
+// File generated by tools/src/cmd/gen
 // using the template:
 //   src/tint/sem/builtin_type.h.tmpl
-// and the intrinsic defintion file:
-//   src/tint/intrinsics.def
 //
 // Do not modify this file directly
 ////////////////////////////////////////////////////////////////////////////////

src/tint/sem/builtin_type.h.tmpl

@@ -1,9 +1,9 @@
 {{- /*
 --------------------------------------------------------------------------------
-Template file for use with tools/builtin-gen to generate builtin_type.h
+Template file for use with tools/src/cmd/gen to generate builtin_type.h
 
 See:
-* tools/cmd/intrinsic-gen/gen for structures used by this template
+* tools/src/cmd/gen for structures used by this template
 * https://golang.org/pkg/text/template/ for documentation on the template syntax
 --------------------------------------------------------------------------------
 */ -}}
@@ -19,7 +19,7 @@ namespace tint::sem {
 /// Enumerator of all builtin functions
 enum class BuiltinType {
     kNone = -1,
-{{- range .Sem.Builtins }}
+{{- range Sem.Builtins }}
     k{{Title .Name}},
 {{- end }}
 };

src/tint/sem/parameter_usage.cc

@@ -13,11 +13,9 @@
 // limitations under the License.
 
 ////////////////////////////////////////////////////////////////////////////////
-// File generated by tools/intrinsic-gen
+// File generated by tools/src/cmd/gen
 // using the template:
 //   src/tint/sem/parameter_usage.cc.tmpl
-// and the intrinsic defintion file:
-//   src/tint/intrinsics.def
 //
 // Do not modify this file directly
 ////////////////////////////////////////////////////////////////////////////////

src/tint/sem/parameter_usage.cc.tmpl

@@ -1,9 +1,9 @@
 {{- /*
 --------------------------------------------------------------------------------
-Template file for use with tools/builtin-gen to generate parameter_usage.cc
+Template file for use with tools/src/cmd/gen to generate parameter_usage.cc
 
 See:
-* tools/cmd/intrinsic-gen/gen for structures used by this template
+* tools/src/cmd/gen for structures used by this template
 * https://golang.org/pkg/text/template/ for documentation on the template syntax
 --------------------------------------------------------------------------------
 */ -}}
@@ -16,7 +16,7 @@ const char* str(ParameterUsage usage) {
     switch (usage) {
         case ParameterUsage::kNone:
             return "none";
-{{- range .Sem.UniqueParameterNames  }}
+{{- range Sem.UniqueParameterNames  }}
         case ParameterUsage::k{{PascalCase .}}:
             return "{{.}}";
 {{- end  }}

src/tint/sem/parameter_usage.h

@@ -13,11 +13,9 @@
 // limitations under the License.
 
 ////////////////////////////////////////////////////////////////////////////////
-// File generated by tools/intrinsic-gen
+// File generated by tools/src/cmd/gen
 // using the template:
 //   src/tint/sem/parameter_usage.h.tmpl
-// and the intrinsic defintion file:
-//   src/tint/intrinsics.def
 //
 // Do not modify this file directly
 ////////////////////////////////////////////////////////////////////////////////

src/tint/sem/parameter_usage.h.tmpl

@@ -1,9 +1,9 @@
 {{- /*
 --------------------------------------------------------------------------------
-Template file for use with tools/builtin-gen to generate parameter_usage.h
+Template file for use with tools/src/cmd/gen to generate parameter_usage.h
 
 See:
-* tools/cmd/intrinsic-gen/gen for structures used by this template
+* tools/src/cmd/gen for structures used by this template
 * https://golang.org/pkg/text/template/ for documentation on the template syntax
 --------------------------------------------------------------------------------
 */ -}}
@@ -17,7 +17,7 @@ namespace tint::sem {
 /// overload position
 enum class ParameterUsage {
     kNone = -1,
-{{- range .Sem.UniqueParameterNames  }}
+{{- range Sem.UniqueParameterNames  }}
     k{{PascalCase .}},
 {{- end  }}
 };

test/tint/builtins/gen/gen.wgsl.tmpl

@@ -1,16 +1,16 @@
 {{- /*
 --------------------------------------------------------------------------------
-Template file for use with tools/builtin-gen to generate the wgsl files in the
+Template file for use with tools/src/cmd/gen to generate the wgsl files in the
 ./literal/...  and ./var/... subdirectories
 
 See:
-* tools/cmd/intrinsic-gen/gen for structures used by this template
+* tools/src/cmd/gen for structures used by this template
 * https://golang.org/pkg/text/template/ for documentation on the template syntax
 --------------------------------------------------------------------------------
 */ -}}
 
 {{- /* For each permutation of each overload of each function... */ -}}
-{{- range .Sem.Builtins -}}
+{{- range Sem.Builtins -}}
 {{- /* TODO(crbug.com/tint/1536): Remove the bodge below after we support [[extension("extension_name")]] in intrinsics.def */}}
 {{- if not (or (eq .Name "dot4I8Packed") (eq .Name "dot4U8Packed"))}}
 {{-   range .Overloads  -}}

tools/src/cmd/gen/main.go

@@ -12,25 +12,158 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package gen
+// gen parses the <tint>/src/tint/intrinsics.def file, then scans the
+// project directory for '<file>.tmpl' files, to produce source code files.
+package main
 
 import (
+	"flag"
 	"fmt"
 	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
 	"reflect"
 	"strings"
 	"text/template"
 	"unicode"
 
-	"dawn.googlesource.com/dawn/tools/src/cmd/intrinsic-gen/sem"
+	"dawn.googlesource.com/dawn/tools/src/fileutils"
+	"dawn.googlesource.com/dawn/tools/src/glob"
+	"dawn.googlesource.com/dawn/tools/src/tint/intrinsic/gen"
+	"dawn.googlesource.com/dawn/tools/src/tint/intrinsic/parser"
+	"dawn.googlesource.com/dawn/tools/src/tint/intrinsic/resolver"
+	"dawn.googlesource.com/dawn/tools/src/tint/intrinsic/sem"
 )
 
+const defProjectRelPath = "src/tint/intrinsics.def"
+
+func main() {
+	if err := run(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func showUsage() {
+	fmt.Println(`
+gen generates the templated code for the Tint compiler
+
+gen scans the project directory for '<file>.tmpl' files, to produce source code
+files.
+
+If the templates use the 'IntrinsicTable' function then gen will parse and
+resolve the <tint>/src/tint/intrinsics.def file.
+
+usage:
+  gen
+
+optional flags:`)
+	flag.PrintDefaults()
+	fmt.Println(``)
+	os.Exit(1)
+}
+
+func run() error {
+	projectRoot := fileutils.ProjectRoot()
+
+	// Recursively find all the template files in the <tint>/src directory
+	files, err := glob.Scan(projectRoot, glob.MustParseConfig(`{
+		"paths": [{"include": [
+			"src/tint/**.tmpl",
+			"test/tint/**.tmpl"
+		]}]
+	}`))
+	if err != nil {
+		return err
+	}
+
+	// For each template file...
+	for _, relTmplPath := range files {
+		// Make tmplPath absolute
+		tmplPath := filepath.Join(projectRoot, relTmplPath)
+
+		// Read the template file
+		tmpl, err := ioutil.ReadFile(tmplPath)
+		if err != nil {
+			return fmt.Errorf("failed to open '%v': %w", tmplPath, err)
+		}
+
+		// Create or update the file at relpath if the file content has changed
+		// relpath is a path relative to the template
+		writeFile := func(relpath, body string) error {
+			// Write the common file header
+			sb := strings.Builder{}
+			sb.WriteString(fmt.Sprintf(header, filepath.ToSlash(relTmplPath)))
+			sb.WriteString(body)
+			content := sb.String()
+			abspath := filepath.Join(filepath.Dir(tmplPath), relpath)
+			return writeFileIfChanged(abspath, content)
+		}
+
+		// Write the content generated using the template and semantic info
+		sb := strings.Builder{}
+		if err := generate(string(tmpl), &sb, writeFile); err != nil {
+			return fmt.Errorf("while processing '%v': %w", tmplPath, err)
+		}
+
+		if body := sb.String(); body != "" {
+			_, tmplFileName := filepath.Split(tmplPath)
+			outFileName := strings.TrimSuffix(tmplFileName, ".tmpl")
+			if err := writeFile(outFileName, body); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// writes content to path if the file has changed
+func writeFileIfChanged(path, content string) error {
+	existing, err := ioutil.ReadFile(path)
+	if err == nil && string(existing) == content {
+		return nil // Not changed
+	}
+	if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
+		return fmt.Errorf("failed to create directory for '%v': %w", path, err)
+	}
+	if err := ioutil.WriteFile(path, []byte(content), 0666); err != nil {
+		return fmt.Errorf("failed to write file '%v': %w", path, err)
+	}
+	return nil
+}
+
+const header = `// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+////////////////////////////////////////////////////////////////////////////////
+// File generated by tools/src/cmd/gen
+// using the template:
+//   %v
+//
+// Do not modify this file directly
+////////////////////////////////////////////////////////////////////////////////
+
+`
+
 type generator struct {
-	s      *sem.Sem
 	t      *template.Template
 	cached struct {
-		intrinsicTable *IntrinsicTable // lazily built by intrinsicTable()
+		sem            *sem.Sem            // lazily built by sem()
-		permuter       *Permuter       // lazily built by permute()
+		intrinsicTable *gen.IntrinsicTable // lazily built by intrinsicTable()
+		permuter       *gen.Permuter       // lazily built by permute()
 	}
 }
 
@@ -40,12 +173,11 @@ type generator struct {
 // content is the file content to write.
 type WriteFile func(relpath, content string) error
 
-// Generate executes the template tmpl using the provided semantic
+// generate executes the template tmpl, writing the output to w.
-// information, writing the output to w.
 // See https://golang.org/pkg/text/template/ for documentation on the template
 // syntax.
-func Generate(s *sem.Sem, tmpl string, w io.Writer, writeFile WriteFile) error {
+func generate(tmpl string, w io.Writer, writeFile WriteFile) error {
-	g := generator{s: s}
+	g := generator{}
 	return g.generate(tmpl, w, writeFile)
 }
 
@@ -55,7 +187,7 @@ func (g *generator) generate(tmpl string, w io.Writer, writeFile WriteFile) erro
 		"Iterate":               iterate,
 		"Title":                 strings.Title,
 		"PascalCase":            pascalCase,
-		"SplitDisplayName":      splitDisplayName,
+		"SplitDisplayName":      gen.SplitDisplayName,
 		"HasPrefix":             strings.HasPrefix,
 		"HasSuffix":             strings.HasSuffix,
 		"TrimPrefix":            strings.TrimPrefix,
@@ -70,10 +202,11 @@ func (g *generator) generate(tmpl string, w io.Writer, writeFile WriteFile) erro
 		"IsTemplateNumberParam": is(sem.TemplateNumberParam{}),
 		"IsTemplateTypeParam":   is(sem.TemplateTypeParam{}),
 		"IsType":                is(sem.Type{}),
-		"IsAbstract":            isAbstract,
+		"IsAbstract":            gen.IsAbstract,
-		"IsDeclarable":          isDeclarable,
+		"IsDeclarable":          gen.IsDeclarable,
 		"IsFirstIn":             isFirstIn,
 		"IsLastIn":              isLastIn,
+		"Sem":                   g.sem,
 		"IntrinsicTable":        g.intrinsicTable,
 		"Permute":               g.permute,
 		"Eval":                  g.eval,
@@ -84,9 +217,7 @@ func (g *generator) generate(tmpl string, w io.Writer, writeFile WriteFile) erro
 		return err
 	}
 	g.t = t
-	return t.Execute(w, map[string]interface{}{
+	return t.Execute(w, map[string]interface{}{})
-		"Sem": g.s,
-	})
 }
 
 // eval executes the sub-template with the given name and argument, returning
@@ -122,12 +253,43 @@ func (g *generator) eval(template string, args ...interface{}) (string, error) {
 	return sb.String(), nil
 }
 
+// sem lazily parses and resolves the intrinsic.def file, returning the semantic info.
+func (g *generator) sem() (*sem.Sem, error) {
+	if g.cached.sem == nil {
+		// Load the builtins definition file
+		defPath := filepath.Join(fileutils.ProjectRoot(), defProjectRelPath)
+
+		defSource, err := ioutil.ReadFile(defPath)
+		if err != nil {
+			return nil, err
+		}
+
+		// Parse the definition file to produce an AST
+		ast, err := parser.Parse(string(defSource), defProjectRelPath)
+		if err != nil {
+			return nil, err
+		}
+
+		// Resolve the AST to produce the semantic info
+		sem, err := resolver.Resolve(ast)
+		if err != nil {
+			return nil, err
+		}
+
+		g.cached.sem = sem
+	}
+	return g.cached.sem, nil
+}
+
 // intrinsicTable lazily calls and returns the result of buildIntrinsicTable(),
 // caching the result for repeated calls.
-func (g *generator) intrinsicTable() (*IntrinsicTable, error) {
+func (g *generator) intrinsicTable() (*gen.IntrinsicTable, error) {
 	if g.cached.intrinsicTable == nil {
-		var err error
+		sem, err := g.sem()
-		g.cached.intrinsicTable, err = buildIntrinsicTable(g.s)
+		if err != nil {
+			return nil, err
+		}
+		g.cached.intrinsicTable, err = gen.BuildIntrinsicTable(sem)
 		if err != nil {
 			return nil, err
 		}
@@ -137,10 +299,13 @@ func (g *generator) intrinsicTable() (*IntrinsicTable, error) {
 
 // permute lazily calls buildPermuter(), caching the result for repeated
 // calls, then passes the argument to Permutator.Permute()
-func (g *generator) permute(overload *sem.Overload) ([]Permutation, error) {
+func (g *generator) permute(overload *sem.Overload) ([]gen.Permutation, error) {
 	if g.cached.permuter == nil {
-		var err error
+		sem, err := g.sem()
-		g.cached.permuter, err = buildPermuter(g.s)
+		if err != nil {
+			return nil, err
+		}
+		g.cached.permuter, err = gen.NewPermuter(sem)
 		if err != nil {
 			return nil, err
 		}
@@ -206,26 +371,6 @@ func iterate(n int) []int {
 	return out
 }
 
-// isAbstract returns true if the FullyQualifiedName refers to an abstract
-// numeric type
-func isAbstract(fqn sem.FullyQualifiedName) bool {
-	switch fqn.Target.GetName() {
-	case "ia", "fa":
-		return true
-	case "vec":
-		return isAbstract(fqn.TemplateArguments[1].(sem.FullyQualifiedName))
-	case "mat":
-		return isAbstract(fqn.TemplateArguments[2].(sem.FullyQualifiedName))
-	}
-	return false
-}
-
-// isDeclarable returns false if the FullyQualifiedName refers to an abstract
-// numeric type, or if it starts with a leading underscore.
-func isDeclarable(fqn sem.FullyQualifiedName) bool {
-	return !isAbstract(fqn) && !strings.HasPrefix(fqn.Target.GetName(), "_")
-}
-
 // pascalCase returns the snake-case string s transformed into 'PascalCase',
 // Rules:
 // * The first letter of the string is capitalized
@@ -252,34 +397,3 @@ func pascalCase(s string) string {
 	}
 	return b.String()
 }
-
-// splitDisplayName splits displayName into parts, where text wrapped in {}
-// braces are not quoted and the rest is quoted. This is used to help process
-// the string value of the [[display()]] decoration. For example:
-//   splitDisplayName("vec{N}<{T}>")
-// would return the strings:
-//   [`"vec"`, `N`, `"<"`, `T`, `">"`]
-func splitDisplayName(displayName string) []string {
-	parts := []string{}
-	pending := strings.Builder{}
-	for _, r := range displayName {
-		switch r {
-		case '{':
-			if pending.Len() > 0 {
-				parts = append(parts, fmt.Sprintf(`"%v"`, pending.String()))
-				pending.Reset()
-			}
-		case '}':
-			if pending.Len() > 0 {
-				parts = append(parts, pending.String())
-				pending.Reset()
-			}
-		default:
-			pending.WriteRune(r)
-		}
-	}
-	if pending.Len() > 0 {
-		parts = append(parts, fmt.Sprintf(`"%v"`, pending.String()))
-	}
-	return parts
-}

tools/src/cmd/intrinsic-gen/main.go

@@ -1,173 +0,0 @@
-// Copyright 2021 The Tint Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// intrinsic-gen parses the <tint>/src/tint/intrinsics.def file, then scans the
-// project directory for '<file>.tmpl' files, to produce '<file>' source code
-// files.
-package main
-
-import (
-	"flag"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"dawn.googlesource.com/dawn/tools/src/cmd/intrinsic-gen/gen"
-	"dawn.googlesource.com/dawn/tools/src/cmd/intrinsic-gen/parser"
-	"dawn.googlesource.com/dawn/tools/src/cmd/intrinsic-gen/resolver"
-	"dawn.googlesource.com/dawn/tools/src/fileutils"
-	"dawn.googlesource.com/dawn/tools/src/glob"
-)
-
-const defProjectRelPath = "src/tint/intrinsics.def"
-
-func main() {
-	if err := run(); err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-}
-
-func showUsage() {
-	fmt.Println(`
-intrinsic-gen generates the intrinsic table for the Tint compiler
-
-intrinsic-gen parses the <tint>/src/tint/intrinsics.def file, then scans the project
-directory for '<file>.tmpl' files, to produce '<file>' source code files.
-
-usage:
-  intrinsic-gen
-
-optional flags:`)
-	flag.PrintDefaults()
-	fmt.Println(``)
-	os.Exit(1)
-}
-
-func run() error {
-	// Load the builtins definition file
-	projectRoot := fileutils.ProjectRoot()
-	defPath := filepath.Join(projectRoot, defProjectRelPath)
-
-	defSource, err := ioutil.ReadFile(defPath)
-	if err != nil {
-		return err
-	}
-
-	// Parse the definition file to produce an AST
-	ast, err := parser.Parse(string(defSource), defProjectRelPath)
-	if err != nil {
-		return err
-	}
-
-	// Resolve the AST to produce the semantic info
-	sem, err := resolver.Resolve(ast)
-	if err != nil {
-		return err
-	}
-
-	// Recursively find all the template files in the <tint>/src directory
-	files, err := glob.Scan(projectRoot, glob.MustParseConfig(`{
-		"paths": [{"include": [
-			"src/tint/**.tmpl",
-			"test/tint/**.tmpl"
-		]}]
-	}`))
-	if err != nil {
-		return err
-	}
-
-	// For each template file...
-	for _, relTmplPath := range files {
-		// Make tmplPath absolute
-		tmplPath := filepath.Join(projectRoot, relTmplPath)
-
-		// Read the template file
-		tmpl, err := ioutil.ReadFile(tmplPath)
-		if err != nil {
-			return fmt.Errorf("failed to open '%v': %w", tmplPath, err)
-		}
-
-		// Create or update the file at relpath if the file content has changed
-		// relpath is a path relative to the template
-		writeFile := func(relpath, body string) error {
-			// Write the common file header
-			sb := strings.Builder{}
-			sb.WriteString(fmt.Sprintf(header, filepath.ToSlash(relTmplPath), filepath.ToSlash(defProjectRelPath)))
-			sb.WriteString(body)
-			content := sb.String()
-			abspath := filepath.Join(filepath.Dir(tmplPath), relpath)
-			return writeFileIfChanged(abspath, content)
-		}
-
-		// Write the content generated using the template and semantic info
-		sb := strings.Builder{}
-		if err := gen.Generate(sem, string(tmpl), &sb, writeFile); err != nil {
-			return fmt.Errorf("while processing '%v': %w", tmplPath, err)
-		}
-
-		if body := sb.String(); body != "" {
-			_, tmplFileName := filepath.Split(tmplPath)
-			outFileName := strings.TrimSuffix(tmplFileName, ".tmpl")
-			if err := writeFile(outFileName, body); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-// writes content to path if the file has changed
-func writeFileIfChanged(path, content string) error {
-	existing, err := ioutil.ReadFile(path)
-	if err == nil && string(existing) == content {
-		return nil // Not changed
-	}
-	if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
-		return fmt.Errorf("failed to create directory for '%v': %w", path, err)
-	}
-	if err := ioutil.WriteFile(path, []byte(content), 0666); err != nil {
-		return fmt.Errorf("failed to write file '%v': %w", path, err)
-	}
-	return nil
-}
-
-const header = `// Copyright 2021 The Tint Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-////////////////////////////////////////////////////////////////////////////////
-// File generated by tools/intrinsic-gen
-// using the template:
-//   %v
-// and the intrinsic defintion file:
-//   %v
-//
-// Do not modify this file directly
-////////////////////////////////////////////////////////////////////////////////
-
-`

tools/src/tint/intrinsic/ast/ast.go

@@ -20,7 +20,7 @@ import (
 	"fmt"
 	"strings"
 
-	"dawn.googlesource.com/dawn/tools/src/cmd/intrinsic-gen/tok"
+	"dawn.googlesource.com/dawn/tools/src/tint/intrinsic/tok"
 )
 
 // AST is the parsed syntax tree of the intrinsic definition file

tools/src/tint/intrinsic/gen/gen.go

@@ -12,14 +12,19 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+/// Package gen holds types and helpers for generating templated code from the
+/// intrinsic.def file.
+///
+/// Used by tools/src/cmd/gen/main.go
 package gen
 
 import (
 	"fmt"
+	"strings"
 
-	"dawn.googlesource.com/dawn/tools/src/cmd/intrinsic-gen/sem"
 	"dawn.googlesource.com/dawn/tools/src/list"
 	"dawn.googlesource.com/dawn/tools/src/lut"
+	"dawn.googlesource.com/dawn/tools/src/tint/intrinsic/sem"
 )
 
 // IntrinsicTable holds data specific to the intrinsic_table.inl.tmpl template
@@ -352,8 +357,8 @@ func (b *overloadBuilder) collectMatcherIndices(fqn sem.FullyQualifiedName) ([]i
 	return out, nil
 }
 
-// buildIntrinsicTable builds the IntrinsicTable from the semantic info
+// BuildIntrinsicTable builds the IntrinsicTable from the semantic info
-func buildIntrinsicTable(s *sem.Sem) (*IntrinsicTable, error) {
+func BuildIntrinsicTable(s *sem.Sem) (*IntrinsicTable, error) {
 	b := IntrinsicTableBuilder{
 		IntrinsicTable: IntrinsicTable{
 			Sem:           s,
@@ -407,3 +412,54 @@ func buildIntrinsicTable(s *sem.Sem) (*IntrinsicTable, error) {
 
 	return &b.IntrinsicTable, nil
 }
+
+// SplitDisplayName splits displayName into parts, where text wrapped in {}
+// braces are not quoted and the rest is quoted. This is used to help process
+// the string value of the [[display()]] decoration. For example:
+//   SplitDisplayName("vec{N}<{T}>")
+// would return the strings:
+//   [`"vec"`, `N`, `"<"`, `T`, `">"`]
+func SplitDisplayName(displayName string) []string {
+	parts := []string{}
+	pending := strings.Builder{}
+	for _, r := range displayName {
+		switch r {
+		case '{':
+			if pending.Len() > 0 {
+				parts = append(parts, fmt.Sprintf(`"%v"`, pending.String()))
+				pending.Reset()
+			}
+		case '}':
+			if pending.Len() > 0 {
+				parts = append(parts, pending.String())
+				pending.Reset()
+			}
+		default:
+			pending.WriteRune(r)
+		}
+	}
+	if pending.Len() > 0 {
+		parts = append(parts, fmt.Sprintf(`"%v"`, pending.String()))
+	}
+	return parts
+}
+
+// IsAbstract returns true if the FullyQualifiedName refers to an abstract
+// numeric type
+func IsAbstract(fqn sem.FullyQualifiedName) bool {
+	switch fqn.Target.GetName() {
+	case "ia", "fa":
+		return true
+	case "vec":
+		return IsAbstract(fqn.TemplateArguments[1].(sem.FullyQualifiedName))
+	case "mat":
+		return IsAbstract(fqn.TemplateArguments[2].(sem.FullyQualifiedName))
+	}
+	return false
+}
+
+// IsDeclarable returns false if the FullyQualifiedName refers to an abstract
+// numeric type, or if it starts with a leading underscore.
+func IsDeclarable(fqn sem.FullyQualifiedName) bool {
+	return !IsAbstract(fqn) && !strings.HasPrefix(fqn.Target.GetName(), "_")
+}

tools/src/tint/intrinsic/gen/permutate.go

@@ -20,8 +20,8 @@ import (
 	"fmt"
 	"strings"
 
-	"dawn.googlesource.com/dawn/tools/src/cmd/intrinsic-gen/sem"
 	"dawn.googlesource.com/dawn/tools/src/fileutils"
+	"dawn.googlesource.com/dawn/tools/src/tint/intrinsic/sem"
 )
 
 // Permuter generates permutations of intrinsic overloads
@@ -30,8 +30,8 @@ type Permuter struct {
 	allTypes []sem.FullyQualifiedName
 }
 
-// buildPermuter returns a new initialized Permuter
+// NewPermuter returns a new initialized Permuter
-func buildPermuter(s *sem.Sem) (*Permuter, error) {
+func NewPermuter(s *sem.Sem) (*Permuter, error) {
 	// allTypes are the list of FQNs that are used for unconstrained types
 	allTypes := []sem.FullyQualifiedName{}
 	for _, ty := range s.Types {
@@ -323,6 +323,10 @@ func (s *permutationState) permutateFQN(in sem.FullyQualifiedName) ([]sem.FullyQ
 }
 
 func validate(fqn sem.FullyQualifiedName, uses *sem.StageUses) bool {
+	if strings.HasPrefix(fqn.Target.GetName(), "_") {
+		return false // Builtin, untypeable return type
+	}
+
 	switch fqn.Target.GetName() {
 	case "array":
 		elTy := fqn.TemplateArguments[0].(sem.FullyQualifiedName)
@@ -332,7 +336,7 @@ func validate(fqn sem.FullyQualifiedName, uses *sem.StageUses) bool {
 			strings.Contains(elTyName, "sampler"),
 			strings.Contains(elTyName, "texture"):
 			return false // Not storable
-		case isAbstract(elTy):
+		case IsAbstract(elTy):
 			return false // Abstract types are not typeable nor supported by arrays
 		}
 	case "ptr":
@@ -367,10 +371,6 @@ func validate(fqn sem.FullyQualifiedName, uses *sem.StageUses) bool {
 		}
 	}
 
-	if strings.HasPrefix(fqn.Target.GetName(), "_") {
-		return false // Core, undeclarable WGSL type
-	}
-
 	for _, arg := range fqn.TemplateArguments {
 		if argFQN, ok := arg.(sem.FullyQualifiedName); ok {
 			if !validate(argFQN, uses) {

tools/src/tint/intrinsic/lexer/lexer.go

@@ -20,7 +20,7 @@ import (
 	"fmt"
 	"unicode"
 
-	"dawn.googlesource.com/dawn/tools/src/cmd/intrinsic-gen/tok"
+	"dawn.googlesource.com/dawn/tools/src/tint/intrinsic/tok"
 )
 
 // Lex produces a list of tokens for the given source code

tools/src/tint/intrinsic/lexer/lexer_test.go

@@ -18,8 +18,8 @@ import (
 	"fmt"
 	"testing"
 
-	"dawn.googlesource.com/dawn/tools/src/cmd/intrinsic-gen/lexer"
+	"dawn.googlesource.com/dawn/tools/src/tint/intrinsic/lexer"
-	"dawn.googlesource.com/dawn/tools/src/cmd/intrinsic-gen/tok"
+	"dawn.googlesource.com/dawn/tools/src/tint/intrinsic/tok"
 )
 
 func TestLexTokens(t *testing.T) {

tools/src/tint/intrinsic/parser/parser.go

@@ -19,9 +19,9 @@ package parser
 import (
 	"fmt"
 
-	"dawn.googlesource.com/dawn/tools/src/cmd/intrinsic-gen/ast"
+	"dawn.googlesource.com/dawn/tools/src/tint/intrinsic/ast"
-	"dawn.googlesource.com/dawn/tools/src/cmd/intrinsic-gen/lexer"
+	"dawn.googlesource.com/dawn/tools/src/tint/intrinsic/lexer"
-	"dawn.googlesource.com/dawn/tools/src/cmd/intrinsic-gen/tok"
+	"dawn.googlesource.com/dawn/tools/src/tint/intrinsic/tok"
 )
 
 // Parse produces a list of tokens for the given source code

tools/src/tint/intrinsic/parser/parser_test.go

@@ -17,8 +17,8 @@ package parser_test
 import (
 	"testing"
 
-	"dawn.googlesource.com/dawn/tools/src/cmd/intrinsic-gen/ast"
+	"dawn.googlesource.com/dawn/tools/src/tint/intrinsic/ast"
-	"dawn.googlesource.com/dawn/tools/src/cmd/intrinsic-gen/parser"
+	"dawn.googlesource.com/dawn/tools/src/tint/intrinsic/parser"
 	"dawn.googlesource.com/dawn/tools/src/utils"
 	"github.com/google/go-cmp/cmp"
 )

tools/src/tint/intrinsic/resolver/resolve.go

@@ -19,9 +19,9 @@ import (
 	"sort"
 	"strconv"
 
-	"dawn.googlesource.com/dawn/tools/src/cmd/intrinsic-gen/ast"
+	"dawn.googlesource.com/dawn/tools/src/tint/intrinsic/ast"
-	"dawn.googlesource.com/dawn/tools/src/cmd/intrinsic-gen/sem"
+	"dawn.googlesource.com/dawn/tools/src/tint/intrinsic/sem"
-	"dawn.googlesource.com/dawn/tools/src/cmd/intrinsic-gen/tok"
+	"dawn.googlesource.com/dawn/tools/src/tint/intrinsic/tok"
 )
 
 type resolver struct {

tools/src/tint/intrinsic/resolver/resolver_test.go

@@ -19,8 +19,8 @@ import (
 	"strings"
 	"testing"
 
-	"dawn.googlesource.com/dawn/tools/src/cmd/intrinsic-gen/parser"
+	"dawn.googlesource.com/dawn/tools/src/tint/intrinsic/parser"
-	"dawn.googlesource.com/dawn/tools/src/cmd/intrinsic-gen/resolver"
+	"dawn.googlesource.com/dawn/tools/src/tint/intrinsic/resolver"
 )
 
 func TestResolver(t *testing.T) {

tools/src/tint/intrinsic/sem/sem.go

@@ -18,7 +18,7 @@ import (
 	"fmt"
 	"sort"
 
-	"dawn.googlesource.com/dawn/tools/src/cmd/intrinsic-gen/ast"
+	"dawn.googlesource.com/dawn/tools/src/tint/intrinsic/ast"
 )
 
 // Sem is the root of the semantic tree