tools/cmd/gen: Add more utilities to the templates

* Ensure that copyright years are preserved when regenerating.
* Add 'Split' function which maps to strings.Split.
* Add 'Scramble' function with messes with a string.
* Add 'Import' function which allows templates to be imported, allowing
  for reusable macros.

Change-Id: Ib77f59a989cf55addcced3e337c9031062a83470
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/97149
Commit-Queue: Dan Sinclair <dsinclair@chromium.org>
Reviewed-by: Dan Sinclair <dsinclair@chromium.org>
Kokoro: Kokoro <noreply+kokoro@google.com>
This commit is contained in:
Ben Clayton 2022-07-27 01:10:15 +00:00 committed by Dawn LUCI CQ
parent 2aea0e957f
commit 695a6d82ae
2 changed files with 202 additions and 85 deletions

View File

@ -21,13 +21,18 @@ import (
"fmt" "fmt"
"io" "io"
"io/ioutil" "io/ioutil"
"math/rand"
"os" "os"
"path/filepath" "path/filepath"
"reflect" "reflect"
"regexp"
"strconv"
"strings" "strings"
"text/template" "text/template"
"time"
"unicode" "unicode"
"dawn.googlesource.com/dawn/tools/src/container"
"dawn.googlesource.com/dawn/tools/src/fileutils" "dawn.googlesource.com/dawn/tools/src/fileutils"
"dawn.googlesource.com/dawn/tools/src/glob" "dawn.googlesource.com/dawn/tools/src/glob"
"dawn.googlesource.com/dawn/tools/src/tint/intrinsic/gen" "dawn.googlesource.com/dawn/tools/src/tint/intrinsic/gen"
@ -78,6 +83,8 @@ func run() error {
return err return err
} }
cache := &genCache{}
// For each template file... // For each template file...
for _, relTmplPath := range files { for _, relTmplPath := range files {
// Make tmplPath absolute // Make tmplPath absolute
@ -89,21 +96,36 @@ func run() error {
return fmt.Errorf("failed to open '%v': %w", tmplPath, err) return fmt.Errorf("failed to open '%v': %w", tmplPath, err)
} }
// Create or update the file at relpath if the file content has changed // Create or update the file at relpath if the file content has changed,
// preserving the copyright year in the header.
// relpath is a path relative to the template // relpath is a path relative to the template
writeFile := func(relpath, body string) error { writeFile := func(relpath, body string) error {
abspath := filepath.Join(filepath.Dir(tmplPath), relpath)
copyrightYear := time.Now().Year()
// Load the old file
existing, err := ioutil.ReadFile(abspath)
if err == nil {
// Look for the existing copyright year
if match := copyrightRegex.FindStringSubmatch(string(existing)); len(match) == 2 {
if year, err := strconv.Atoi(match[1]); err == nil {
copyrightYear = year
}
}
}
// Write the common file header // Write the common file header
sb := strings.Builder{} sb := strings.Builder{}
sb.WriteString(fmt.Sprintf(header, filepath.ToSlash(relTmplPath))) sb.WriteString(fmt.Sprintf(header, copyrightYear, filepath.ToSlash(relTmplPath)))
sb.WriteString(body) sb.WriteString(body)
content := sb.String() content := sb.String()
abspath := filepath.Join(filepath.Dir(tmplPath), relpath) return writeFileIfChanged(abspath, content, string(existing))
return writeFileIfChanged(abspath, content)
} }
// Write the content generated using the template and semantic info // Write the content generated using the template and semantic info
sb := strings.Builder{} sb := strings.Builder{}
if err := generate(string(tmpl), &sb, writeFile); err != nil { if err := generate(string(tmpl), cache, &sb, writeFile); err != nil {
return fmt.Errorf("while processing '%v': %w", tmplPath, err) return fmt.Errorf("while processing '%v': %w", tmplPath, err)
} }
@ -119,22 +141,92 @@ func run() error {
return nil return nil
} }
// Cache for objects that are expensive to build, and can be reused between templates.
type genCache struct {
cached struct {
sem *sem.Sem // lazily built by sem()
intrinsicTable *gen.IntrinsicTable // lazily built by intrinsicTable()
permuter *gen.Permuter // lazily built by permute()
}
}
// sem lazily parses and resolves the intrinsic.def file, returning the semantic info.
func (g *genCache) sem() (*sem.Sem, error) {
if g.cached.sem == nil {
// Load the builtins definition file
defPath := filepath.Join(fileutils.ProjectRoot(), defProjectRelPath)
defSource, err := ioutil.ReadFile(defPath)
if err != nil {
return nil, err
}
// Parse the definition file to produce an AST
ast, err := parser.Parse(string(defSource), defProjectRelPath)
if err != nil {
return nil, err
}
// Resolve the AST to produce the semantic info
sem, err := resolver.Resolve(ast)
if err != nil {
return nil, err
}
g.cached.sem = sem
}
return g.cached.sem, nil
}
// intrinsicTable lazily calls and returns the result of buildIntrinsicTable(),
// caching the result for repeated calls.
func (g *genCache) intrinsicTable() (*gen.IntrinsicTable, error) {
if g.cached.intrinsicTable == nil {
sem, err := g.sem()
if err != nil {
return nil, err
}
g.cached.intrinsicTable, err = gen.BuildIntrinsicTable(sem)
if err != nil {
return nil, err
}
}
return g.cached.intrinsicTable, nil
}
// permute lazily calls buildPermuter(), caching the result for repeated
// calls, then passes the argument to Permutator.Permute()
func (g *genCache) permute(overload *sem.Overload) ([]gen.Permutation, error) {
if g.cached.permuter == nil {
sem, err := g.sem()
if err != nil {
return nil, err
}
g.cached.permuter, err = gen.NewPermuter(sem)
if err != nil {
return nil, err
}
}
return g.cached.permuter.Permute(overload)
}
// writes content to path if the file has changed // writes content to path if the file has changed
func writeFileIfChanged(path, content string) error { func writeFileIfChanged(path, newContent, oldContent string) error {
existing, err := ioutil.ReadFile(path) if oldContent == newContent {
if err == nil && string(existing) == content {
return nil // Not changed return nil // Not changed
} }
if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil { if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
return fmt.Errorf("failed to create directory for '%v': %w", path, err) return fmt.Errorf("failed to create directory for '%v': %w", path, err)
} }
if err := ioutil.WriteFile(path, []byte(content), 0666); err != nil { if err := ioutil.WriteFile(path, []byte(newContent), 0666); err != nil {
return fmt.Errorf("failed to write file '%v': %w", path, err) return fmt.Errorf("failed to write file '%v': %w", path, err)
} }
return nil return nil
} }
const header = `// Copyright 2021 The Tint Authors. var copyrightRegex = regexp.MustCompile(`// Copyright (\d+) The`)
const header = `// Copyright %v The Tint Authors.
// //
// Licensed under the Apache License, Version 2.0 (the "License"); // Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. // you may not use this file except in compliance with the License.
@ -159,12 +251,10 @@ const header = `// Copyright 2021 The Tint Authors.
` `
type generator struct { type generator struct {
t *template.Template template *template.Template
cached struct { cache *genCache
sem *sem.Sem // lazily built by sem() writeFile WriteFile
intrinsicTable *gen.IntrinsicTable // lazily built by intrinsicTable() rnd *rand.Rand
permuter *gen.Permuter // lazily built by permute()
}
} }
// WriteFile is a function that Generate() may call to emit a new file from a // WriteFile is a function that Generate() may call to emit a new file from a
@ -176,13 +266,21 @@ type WriteFile func(relpath, content string) error
// generate executes the template tmpl, writing the output to w. // generate executes the template tmpl, writing the output to w.
// See https://golang.org/pkg/text/template/ for documentation on the template // See https://golang.org/pkg/text/template/ for documentation on the template
// syntax. // syntax.
func generate(tmpl string, w io.Writer, writeFile WriteFile) error { func generate(tmpl string, cache *genCache, w io.Writer, writeFile WriteFile) error {
g := generator{} g := generator{
return g.generate(tmpl, w, writeFile) template: template.New("<template>"),
cache: cache,
writeFile: writeFile,
rnd: rand.New(rand.NewSource(4561123)),
}
if err := g.bindAndParse(g.template, tmpl); err != nil {
return err
}
return g.template.Execute(w, nil)
} }
func (g *generator) generate(tmpl string, w io.Writer, writeFile WriteFile) error { func (g *generator) bindAndParse(t *template.Template, text string) error {
t, err := template.New("<template>").Funcs(map[string]interface{}{ _, err := t.Funcs(map[string]interface{}{
"Map": newMap, "Map": newMap,
"Iterate": iterate, "Iterate": iterate,
"Title": strings.Title, "Title": strings.Title,
@ -194,6 +292,8 @@ func (g *generator) generate(tmpl string, w io.Writer, writeFile WriteFile) erro
"TrimSuffix": strings.TrimSuffix, "TrimSuffix": strings.TrimSuffix,
"TrimLeft": strings.TrimLeft, "TrimLeft": strings.TrimLeft,
"TrimRight": strings.TrimRight, "TrimRight": strings.TrimRight,
"Split": strings.Split,
"Scramble": g.scramble,
"IsEnumEntry": is(sem.EnumEntry{}), "IsEnumEntry": is(sem.EnumEntry{}),
"IsEnumMatcher": is(sem.EnumMatcher{}), "IsEnumMatcher": is(sem.EnumMatcher{}),
"IsFQN": is(sem.FullyQualifiedName{}), "IsFQN": is(sem.FullyQualifiedName{}),
@ -206,24 +306,20 @@ func (g *generator) generate(tmpl string, w io.Writer, writeFile WriteFile) erro
"IsDeclarable": gen.IsDeclarable, "IsDeclarable": gen.IsDeclarable,
"IsFirstIn": isFirstIn, "IsFirstIn": isFirstIn,
"IsLastIn": isLastIn, "IsLastIn": isLastIn,
"Sem": g.sem, "Sem": g.cache.sem,
"IntrinsicTable": g.intrinsicTable, "IntrinsicTable": g.cache.intrinsicTable,
"Permute": g.permute, "Permute": g.cache.permute,
"Eval": g.eval, "Eval": g.eval,
"WriteFile": func(relpath, content string) (string, error) { return "", writeFile(relpath, content) }, "Import": g.importTmpl,
}).Option("missingkey=error"). "WriteFile": func(relpath, content string) (string, error) { return "", g.writeFile(relpath, content) },
Parse(tmpl) }).Option("missingkey=error").Parse(text)
if err != nil {
return err return err
}
g.t = t
return t.Execute(w, map[string]interface{}{})
} }
// eval executes the sub-template with the given name and argument, returning // eval executes the sub-template with the given name and argument, returning
// the generated output // the generated output
func (g *generator) eval(template string, args ...interface{}) (string, error) { func (g *generator) eval(template string, args ...interface{}) (string, error) {
target := g.t.Lookup(template) target := g.template.Lookup(template)
if target == nil { if target == nil {
return "", fmt.Errorf("template '%v' not found", template) return "", fmt.Errorf("template '%v' not found", template)
} }
@ -253,64 +349,54 @@ func (g *generator) eval(template string, args ...interface{}) (string, error) {
return sb.String(), nil return sb.String(), nil
} }
// sem lazily parses and resolves the intrinsic.def file, returning the semantic info. // importTmpl parses the template at the given project-relative path, merging
func (g *generator) sem() (*sem.Sem, error) { // the template definitions into the global namespace.
if g.cached.sem == nil { // Note: The body of the template is not executed.
// Load the builtins definition file func (g *generator) importTmpl(path string) (string, error) {
defPath := filepath.Join(fileutils.ProjectRoot(), defProjectRelPath) if strings.Contains(path, "..") {
return "", fmt.Errorf("import path must not contain '..'")
defSource, err := ioutil.ReadFile(defPath) }
path = filepath.Join(fileutils.ProjectRoot(), path)
data, err := ioutil.ReadFile(path)
if err != nil { if err != nil {
return nil, err return "", fmt.Errorf("failed to open '%v': %w", path, err)
} }
t := g.template.New("")
// Parse the definition file to produce an AST if err := g.bindAndParse(t, string(data)); err != nil {
ast, err := parser.Parse(string(defSource), defProjectRelPath) return "", fmt.Errorf("failed to parse '%v': %w", path, err)
if err != nil {
return nil, err
} }
return "", nil
// Resolve the AST to produce the semantic info
sem, err := resolver.Resolve(ast)
if err != nil {
return nil, err
}
g.cached.sem = sem
}
return g.cached.sem, nil
} }
// intrinsicTable lazily calls and returns the result of buildIntrinsicTable(), // scramble randomly modifies the input string so that it is no longer equal to
// caching the result for repeated calls. // any of the strings in 'avoid'.
func (g *generator) intrinsicTable() (*gen.IntrinsicTable, error) { func (g *generator) scramble(str string, avoid container.Set[string]) (string, error) {
if g.cached.intrinsicTable == nil { bytes := []byte(str)
sem, err := g.sem() passes := g.rnd.Intn(5) + 1
if err != nil {
return nil, err
}
g.cached.intrinsicTable, err = gen.BuildIntrinsicTable(sem)
if err != nil {
return nil, err
}
}
return g.cached.intrinsicTable, nil
}
// permute lazily calls buildPermuter(), caching the result for repeated const chars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz"
// calls, then passes the argument to Permutator.Permute()
func (g *generator) permute(overload *sem.Overload) ([]gen.Permutation, error) { char := func() byte { return chars[g.rnd.Intn(len(chars))] }
if g.cached.permuter == nil { replace := func(at int) { bytes[at] = char() }
sem, err := g.sem() delete := func(at int) { bytes = append(bytes[:at], bytes[at+1:]...) }
if err != nil { insert := func(at int) { bytes = append(append(bytes[:at], char()), bytes[at:]...) }
return nil, err
for i := 0; i < passes || avoid.Contains(string(bytes)); i++ {
if len(bytes) > 0 {
at := g.rnd.Intn(len(bytes))
switch g.rnd.Intn(3) {
case 0:
replace(at)
case 1:
delete(at)
case 2:
insert(at)
} }
g.cached.permuter, err = gen.NewPermuter(sem) } else {
if err != nil { insert(0)
return nil, err
} }
} }
return g.cached.permuter.Permute(overload) return string(bytes), nil
} }
// Map is a simple generic key-value map, which can be used in the template // Map is a simple generic key-value map, which can be used in the template

View File

@ -18,6 +18,7 @@ import (
"fmt" "fmt"
"sort" "sort"
"dawn.googlesource.com/dawn/tools/src/container"
"dawn.googlesource.com/dawn/tools/src/tint/intrinsic/ast" "dawn.googlesource.com/dawn/tools/src/tint/intrinsic/ast"
) )
@ -39,6 +40,16 @@ type Sem struct {
UniqueParameterNames []string UniqueParameterNames []string
} }
// Enum returns the enum with the given name
func (s *Sem) Enum(name string) *Enum {
for _, e := range s.Enums {
if e.Name == name {
return e
}
}
return nil
}
// New returns a new Sem // New returns a new Sem
func New() *Sem { func New() *Sem {
return &Sem{ return &Sem{
@ -69,6 +80,26 @@ func (e *Enum) FindEntry(name string) *EnumEntry {
return nil return nil
} }
// PublicEntries returns the enum entries that are not annotated with @internal
func (e *Enum) PublicEntries() []*EnumEntry {
out := make([]*EnumEntry, 0, len(e.Entries))
for _, entry := range e.Entries {
if !entry.IsInternal {
out = append(out, entry)
}
}
return out
}
// NameSet returns a set of all the enum entry names
func (e *Enum) NameSet() container.Set[string] {
out := container.NewSet[string]()
for _, entry := range e.Entries {
out.Add(entry.Name)
}
return out
}
// EnumEntry is an entry in an enumerator // EnumEntry is an entry in an enumerator
type EnumEntry struct { type EnumEntry struct {
Enum *Enum Enum *Enum