encounter/dawn-cmake
1
0
Fork 0
mirror of https://github.com/encounter/dawn-cmake.git synced 2025-12-09 05:27:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'tint/main' into HEAD

Browse Source 
Integrates Tint repo into Dawn

KIs:
- Building docs for Tint is turned off, because it fails due to lack
  of annotations in Dawn source files.
- Dawn CQ needs to be updated to run Tint specific tests
- Significant post-merge cleanup needed

R=bclayton,cwallez
BUG=dawn:1339

Change-Id: I6c9714a0030934edd6c51f3cac4684dcd59d1ea3
This commit is contained in:
Ryan Harrison
2022-04-06 15:37:27 -04:00
parent 04c18267bc 28674d7c50
commit e87ac76f7d
12772 changed files with 839109 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
tools/benchdiff Executable file
View File

@@ -0,0 +1,33 @@
#!/usr/bin/env bash
# Copyright 2022 The Tint Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e # Fail on any error.
if [ ! -x "$(which go)" ] ; then
echo "error: go needs to be on \$PATH to use $0"
exit 1
fi
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )"
ROOT_DIR="$( cd "${SCRIPT_DIR}/.." >/dev/null 2>&1 && pwd )"
BINARY="${SCRIPT_DIR}/bin/benchdiff"
# Rebuild the binary.
# Note, go caches build artifacts, so this is quick for repeat calls
pushd "${SCRIPT_DIR}/src/cmd/benchdiff" > /dev/null
go build -o "${BINARY}" main.go
popd > /dev/null
"${BINARY}" "$@"

33
tools/builtin-gen Executable file
View File

@@ -0,0 +1,33 @@
#!/usr/bin/env bash
# Copyright 2021 The Tint Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e # Fail on any error.
if [ ! -x "$(which go)" ] ; then
echo "error: go needs to be on \$PATH to use $0"
exit 1
fi
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )"
ROOT_DIR="$( cd "${SCRIPT_DIR}/.." >/dev/null 2>&1 && pwd )"
BINARY="${SCRIPT_DIR}/bin/builtin-gen"
# Rebuild the binary.
# Note, go caches build artifacts, so this is quick for repeat calls
pushd "${SCRIPT_DIR}/src/cmd/builtin-gen" > /dev/null
go build -o "${BINARY}" main.go
popd > /dev/null
"${BINARY}" "$@"

33
tools/check-spec-examples Executable file
View File

@@ -0,0 +1,33 @@
#!/usr/bin/env bash
# Copyright 2021 The Tint Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e # Fail on any error.
if [ ! -x "$(which go)" ] ; then
echo "error: go needs to be on \$PATH to use $0"
exit 1
fi
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )"
ROOT_DIR="$( cd "${SCRIPT_DIR}/.." >/dev/null 2>&1 && pwd )"
BINARY="${SCRIPT_DIR}/bin/check-spec-examples"
# Rebuild the binary.
# Note, go caches build artifacts, so this is quick for repeat calls
pushd "${SCRIPT_DIR}/src/cmd/check-spec-examples" > /dev/null
go build -o "${BINARY}" main.go
popd > /dev/null
"${BINARY}" "$@"

35
tools/fix-tests Executable file
View File

@@ -0,0 +1,35 @@
#!/usr/bin/env bash
# Copyright 2021 The Tint Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# See https://clang.llvm.org/docs/SourceBasedCodeCoverage.html
set -e # Fail on any error.
if [ ! -x "$(which go)" ] ; then
echo "error: go needs to be on \$PATH to use $0"
exit 1
fi
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )"
ROOT_DIR="$( cd "${SCRIPT_DIR}/.." >/dev/null 2>&1 && pwd )"
BINARY="${SCRIPT_DIR}/bin/fix-tests"
# Rebuild the binary.
# Note, go caches build artifacts, so this is quick for repeat calls
pushd "${SCRIPT_DIR}/src/cmd/fix-tests" > /dev/null
go build -o "${BINARY}" main.go
popd > /dev/null
"${BINARY}" "$@"

19
tools/format Executable file
View File

@@ -0,0 +1,19 @@
#!/bin/bash
# Copyright 2020 The Tint Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
find src -name "*.h" -exec clang-format -i {} \;
find src -name "*.cc" -exec clang-format -i {} \;
find src/tint/cmd -name "*.h" -exec clang-format -i {} \;
find src/tint/cmd -name "*.cc" -exec clang-format -i {} \;

33
tools/gerrit-stats Executable file
View File

@@ -0,0 +1,33 @@
#!/usr/bin/env bash
# Copyright 2021 The Tint Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e # Fail on any error.
if [ ! -x "$(which go)" ] ; then
echo "error: go needs to be on \$PATH to use $0"
exit 1
fi
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )"
ROOT_DIR="$( cd "${SCRIPT_DIR}/.." >/dev/null 2>&1 && pwd )"
BINARY="${SCRIPT_DIR}/bin/gerrit-stats"
# Rebuild the binary.
# Note, go caches build artifacts, so this is quick for repeat calls
pushd "${SCRIPT_DIR}/src/cmd/gerrit-stats" > /dev/null
go build -o "${BINARY}" main.go
popd > /dev/null
"${BINARY}" "$@"

33
tools/get-test-plan Executable file
View File

@@ -0,0 +1,33 @@
#!/usr/bin/env bash
# Copyright 2021 The Tint Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e # Fail on any error.
if [ ! -x "$(which go)" ] ; then
echo "error: go needs to be on \$PATH to use $0"
exit 1
fi
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )"
ROOT_DIR="$( cd "${SCRIPT_DIR}/.." >/dev/null 2>&1 && pwd )"
BINARY="${SCRIPT_DIR}/bin/get-test-plan"
# Rebuild the binary.
# Note, go caches build artifacts, so this is quick for repeat calls
pushd "${SCRIPT_DIR}/src/cmd/get-test-plan" > /dev/null
go build -o "${BINARY}" main.go
popd > /dev/null
"${BINARY}" "$@"

12
tools/known_tint_failures Normal file
View File

@@ -0,0 +1,12 @@
break-outside-for-or-switch.fail.wgsl
continue-outside-for.fail.wgsl
duplicate-stuct-name-v2.fail.wgsl
duplicate-stuct-name.fail.wgsl
duplicate-var-name-within-func.fail.wgsl
global-vars-must-be-unique-v2.fail.wgsl
global-vars-must-be-unique-v3.fail.wgsl
runtime-array-not-last.fail.wgsl
self-recursion-v2.fail.wgsl
self-recursion.fail.wgsl
struct-member-def-before-use-v3.fail.wgsl
struct-use-before-def.fail.wgsl

41
tools/lint Executable file
View File

@@ -0,0 +1,41 @@
#!/bin/bash
# Copyright 2020 The Tint Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )"
ROOT_DIR="$( cd "${SCRIPT_DIR}/.." >/dev/null 2>&1 && pwd )"
set -e # fail on error
if ! command -v cpplint.py &> /dev/null; then
echo "cpplint.py not found on PATH"
exit 1
fi
FILTER="-runtime/references"
FILES="`find src -type f` `find src/tint/cmd -type f`"
FILES+="`find tools/src -type f` `find src/tint/cmd -type f`"
if command -v go &> /dev/null; then
# Go is installed. Run cpplint in parallel for speed wins
go run $SCRIPT_DIR/src/cmd/run-parallel/main.go \
--only-print-failures \
cpplint.py \
--root=$ROOT_DIR \
--filter="$FILTER" \
$ -- $FILES
else
cpplint.py --root=$ROOT_DIR --filter="$FILTER" $FILES
fi

54
tools/roll-all Executable file
View File

@@ -0,0 +1,54 @@
#!/usr/bin/env bash
# Copyright 2020 The Tint Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Defined to use origin/master instead of origin/main
build=1
buildtools=1
clang=1
cpplint=1
googletest=1
spirv_headers=1
spirv_tools=1
testing=1
# This script assumes it's parent directory is the repo root.
repo_path=$(dirname "$0")/..
cd "$repo_path"
if [[ $(git diff --stat) != '' ]]; then
echo "Working tree is dirty, commit changes before attempting to roll DEPS"
exit 1
fi
old_head=$(git rev-parse HEAD)
for i in $(find third_party -maxdepth 1 -mindepth 1 -type d); do
name=`echo "${i%%/}" | cut -f2 -d'/' | tr '-' '_'`
if [ "x${name}" == "xcpplint" ]; then
continue
fi
trunk="origin/main"
if [ ! -z ${!name} ]; then
trunk="origin/master"
fi
roll-dep --ignore-dirty-tree --roll-to="${trunk}" "${i%%/}";
done
git rebase --interactive "${old_head}"

260
tools/run_tests.py Executable file
View File

@@ -0,0 +1,260 @@
#!/usr/bin/env python
# Copyright 2020 The Tint Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Test runner for executing a test of tests with Tint. The runner will
# find all .wgsl files in the given folder and attempt to convert them
# to each of the backend formats. If the file contains a '.fail.' in the
# name then the runner will expect the file to fail conversion.
import base64
import copy
import difflib
import optparse
import os
import platform
import re
import subprocess
import sys
import tempfile
"""
A single test case to be executed. Stores the path to the test file
and the result of executing the test.
"""
class TestCase:
def __init__(self, input_path, parse_only):
self.input_path = input_path
self.parse_only = parse_only
self.results = {}
def IsExpectedFail(self):
fail_re = re.compile('^.+[\.]fail[\.]wgsl')
return fail_re.match(self.GetInputPath())
def IsParseOnly(self):
return self.parse_only
def GetInputPath(self):
return self.input_path
def GetResult(self, fmt):
return self.results[fmt]
def GetReason(self):
with open(self.GetInputPath()) as test:
first_line = test.readline()
if (first_line.startswith("# v-")):
reason = first_line[2:8]
else:
reason = ''
return reason
"""
The test runner, will execute a series of test cases and record the
results.
"""
class TestRunner:
def RunTest(self, tc):
"""Runs a single test."""
print("Testing {}".format(tc.GetInputPath()))
cmd = [self.options.test_prog_path]
if tc.IsParseOnly():
cmd += ['--parse-only']
languages = ["wgsl", "spvasm", "msl", "hlsl"]
try:
for lang in languages:
lang_cmd = copy.copy(cmd)
lang_cmd += ['--format', lang]
lang_cmd += [tc.GetInputPath()]
err = subprocess.check_output(lang_cmd,
stderr=subprocess.STDOUT)
except Exception as e:
failure_reason = "{}".format("".join(map(chr,
bytearray(e.output))))
if tc.IsExpectedFail():
right_reason = tc.GetReason()
if (right_reason in failure_reason):
return False, ""
else:
return False, right_reason
if not tc.IsExpectedFail():
print(failure_reason)
print(e)
return False, ""
return True, ""
def RunTests(self):
"""Runs a set of test cases"""
for tc in self.test_cases:
result, reason = self.RunTest(tc)
"""evaluate final result based on result, tc.IsExpectedFail() and reason"""
if not result:
# result == false, expected true, reason:don't care
if not tc.IsExpectedFail():
print("Expected: " + tc.GetInputPath() +
" to pass but failed.")
self.failures.append(tc.GetInputPath())
# result == false, expected false, reason: wrong
else:
if reason.startswith("v-"):
print("Failed for a wrong reason: " +
tc.GetInputPath() +
" expected with error code: " + reason)
self.failures_wrong_reason.append(tc.GetInputPath())
# result == true, expected false, reason:don't care
elif tc.IsExpectedFail():
print("Expected: " + tc.GetInputPath() +
" to fail but passed.")
self.failures.append(tc.GetInputPath())
def GetUnexpectedFailures(self):
for failure in self.failures + self.failures_wrong_reason:
if failure not in self.known_failures:
self.unexpected_failures.append(failure)
return
def SummarizeResults(self):
"""Prints a summarization of the test results to STDOUT"""
if len(self.unexpected_failures):
self.unexpected_failures.sort()
print('\nSummary of unexpected failures:')
for unexpected_fail in self.unexpected_failures:
print(unexpected_fail)
for f in self.known_failures:
if f not in self.failures_wrong_reason + self.failures:
self.unexpected_successes.append(f)
if len(self.unexpected_successes):
print('\nSummary of unexpected successes:')
for s in self.unexpected_successes:
print(s)
print('')
print('Test cases executed: {}'.format(len(self.test_cases)))
print(' Successes: {}'.format(
(len(self.test_cases) - len(self.failures) -
len(self.failures_wrong_reason))))
print(' Failures: {}'.format(
len(self.failures) + len(self.failures_wrong_reason)))
print(' Unexpected Failures: {}'.format(len(
self.unexpected_failures)))
print(' Unexpected Successes: {}'.format(
len(self.unexpected_successes)))
print('')
def Run(self):
"""Executes the test runner."""
base_path = os.path.abspath(
os.path.join(os.path.dirname(__file__), '..'))
usage = 'usage: %prog [options] (file)'
parser = optparse.OptionParser(usage=usage)
parser.add_option('--build-dir',
default=os.path.join(base_path, 'out', 'Debug'),
help='path to build directory')
parser.add_option('--test-dir',
default=os.path.join(os.path.dirname(__file__), '..',
'third_party', 'gpuweb-cts',
'src', 'webgpu', 'shader',
'validation', 'wgsl'),
help='path to directory containing test files')
parser.add_option(
'--known-failures-file',
default=os.path.join(base_path, 'tools', 'known_tint_failures'),
help='path to directory containing the known failures file')
parser.add_option(
'--test-prog-path',
default=None,
help='path to program to test (default build-dir/tint)')
parser.add_option('--parse-only',
action="store_true",
default=False,
help='only parse test cases; do not compile')
self.options, self.args = parser.parse_args()
if self.options.test_prog_path == None:
test_prog = os.path.abspath(
os.path.join(self.options.build_dir, 'tint'))
if not os.path.isfile(test_prog):
print("Cannot find test program {}".format(test_prog))
return 1
self.options.test_prog_path = test_prog
if not os.path.isfile(self.options.test_prog_path):
print("Cannot find test program '{}'".format(
self.options.test_prog_path))
return 1
input_file_re = re.compile('^.+[\.]wgsl')
self.test_cases = []
if self.args:
for filename in self.args:
input_path = os.path.join(self.options.test_dir, filename)
if not os.path.isfile(input_path):
print("Cannot find test file '{}'".format(filename))
return 1
self.test_cases.append(
TestCase(input_path, self.options.parse_only))
else:
for file_dir, _, filename_list in os.walk(self.options.test_dir):
for input_filename in filename_list:
if input_file_re.match(input_filename):
input_path = os.path.join(file_dir, input_filename)
if os.path.isfile(input_path):
self.test_cases.append(
TestCase(input_path, self.options.parse_only))
known_failure_file = self.options.known_failures_file
self.known_failures = []
with open(known_failure_file, 'r') as f:
for failure_filename in f.read().splitlines():
self.known_failures.append(
os.path.join(self.options.test_dir, failure_filename))
self.failures = []
self.failures_wrong_reason = []
self.unexpected_failures = []
self.unexpected_successes = []
self.RunTests()
self.GetUnexpectedFailures()
self.SummarizeResults()
return not len(self.unexpected_failures + self.unexpected_successes)
def main():
runner = TestRunner()
return runner.Run()
if __name__ == '__main__':
sys.exit(main())

33
tools/snippets Executable file
View File

@@ -0,0 +1,33 @@
#!/usr/bin/env bash
# Copyright 2021 The Tint Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e # Fail on any error.
if [ ! -x "$(which go)" ] ; then
echo "error: go needs to be on \$PATH to use $0"
exit 1
fi
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )"
ROOT_DIR="$( cd "${SCRIPT_DIR}/.." >/dev/null 2>&1 && pwd )"
BINARY="${SCRIPT_DIR}/bin/snippets"
# Rebuild the binary.
# Note, go caches build artifacts, so this is quick for repeat calls
pushd "${SCRIPT_DIR}/src/cmd/snippets" > /dev/null
go build -o "${BINARY}" main.go
popd > /dev/null
"${BINARY}" "$@"

417
tools/src/bench/bench.go Normal file
View File

@@ -0,0 +1,417 @@
// Copyright 2022 The Tint Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package bench provides types and methods for parsing Google benchmark results.
package bench
import (
"encoding/json"
"errors"
"fmt"
"regexp"
"sort"
"strconv"
"strings"
"time"
"unicode/utf8"
)
// Run holds all the benchmark results for a run, along with the context
// information for the run.
type Run struct {
Benchmarks []Benchmark
Context *Context
}
// Context provides information about the environment used to perform the
// benchmark.
type Context struct {
Date time.Time
HostName string
Executable string
NumCPUs int
MhzPerCPU int
CPUScalingEnabled bool
Caches []ContextCache
LoadAvg []float32
LibraryBuildType string
}
// ContextCache holds information about one of the system caches.
type ContextCache struct {
Type string
Level int
Size int
NumSharing int
}
// Benchmark holds the results of a single benchmark test.
type Benchmark struct {
Name string
Duration time.Duration
AggregateType AggregateType
}
// AggregateType is an enumerator of benchmark aggregate types.
type AggregateType string
// Enumerator values of AggregateType
const (
NonAggregate AggregateType = "NonAggregate"
Mean AggregateType = "mean"
Median AggregateType = "median"
Stddev AggregateType = "stddev"
)
// Parse parses the benchmark results from the string s.
// Parse will handle the json and 'console' formats.
func Parse(s string) (Run, error) {
type Parser = func(s string) (Run, error)
for _, parser := range []Parser{parseConsole, parseJSON} {
r, err := parser(s)
switch err {
case nil:
return r, nil
case errWrongFormat:
default:
return Run{}, err
}
}
return Run{}, errors.New("Unrecognised file format")
}
var errWrongFormat = errors.New("Wrong format")
var consoleLineRE = regexp.MustCompile(`([\w/:]+)\s+([0-9]+(?:.[0-9]+)?) ns\s+[0-9]+(?:.[0-9]+) ns\s+([0-9]+)`)
func parseConsole(s string) (Run, error) {
blocks := strings.Split(s, "------------------------------------------------------------------------------------------")
if len(blocks) != 3 {
return Run{}, errWrongFormat
}
lines := strings.Split(blocks[2], "\n")
b := make([]Benchmark, 0, len(lines))
for _, line := range lines {
if len(line) == 0 {
continue
}
matches := consoleLineRE.FindStringSubmatch(line)
if len(matches) != 4 {
return Run{}, fmt.Errorf("Unable to parse the line:\n" + line)
}
ns, err := strconv.ParseFloat(matches[2], 64)
if err != nil {
return Run{}, fmt.Errorf("Unable to parse the duration: " + matches[2])
}
b = append(b, Benchmark{
Name: trimAggregateSuffix(matches[1]),
Duration: time.Nanosecond * time.Duration(ns),
})
}
return Run{Benchmarks: b}, nil
}
func parseJSON(s string) (Run, error) {
type Data struct {
Context struct {
Date time.Time `json:"date"`
HostName string `json:"host_name"`
Executable string `json:"executable"`
NumCPUs int `json:"num_cpus"`
MhzPerCPU int `json:"mhz_per_cpu"`
CPUScalingEnabled bool `json:"cpu_scaling_enabled"`
LoadAvg []float32 `json:"load_avg"`
LibraryBuildType string `json:"library_build_type"`
Caches []struct {
Type string `json:"type"`
Level int `json:"level"`
Size int `json:"size"`
NumSharing int `json:"num_sharing"`
} `json:"caches"`
} `json:"context"`
Benchmarks []struct {
Name string `json:"name"`
Time float64 `json:"real_time"`
AggregateType AggregateType `json:"aggregate_name"`
} `json:"benchmarks"`
}
data := Data{}
d := json.NewDecoder(strings.NewReader(s))
if err := d.Decode(&data); err != nil {
return Run{}, err
}
out := Run{
Benchmarks: make([]Benchmark, len(data.Benchmarks)),
Context: &Context{
Date: data.Context.Date,
HostName: data.Context.HostName,
Executable: data.Context.Executable,
NumCPUs: data.Context.NumCPUs,
MhzPerCPU: data.Context.MhzPerCPU,
CPUScalingEnabled: data.Context.CPUScalingEnabled,
LoadAvg: data.Context.LoadAvg,
LibraryBuildType: data.Context.LibraryBuildType,
Caches: make([]ContextCache, len(data.Context.Caches)),
},
}
for i, c := range data.Context.Caches {
out.Context.Caches[i] = ContextCache{
Type: c.Type,
Level: c.Level,
Size: c.Size,
NumSharing: c.NumSharing,
}
}
for i, b := range data.Benchmarks {
out.Benchmarks[i] = Benchmark{
Name: trimAggregateSuffix(b.Name),
Duration: time.Nanosecond * time.Duration(int64(b.Time)),
AggregateType: b.AggregateType,
}
}
return out, nil
}
// Diff describes the difference between two benchmarks
type Diff struct {
TestName string
Delta time.Duration // Δ (A → B)
PercentChangeAB float64 // % (A → B)
PercentChangeBA float64 // % (A → B)
MultiplierChangeAB float64 // × (A → B)
MultiplierChangeBA float64 // × (A → B)
TimeA time.Duration // A
TimeB time.Duration // B
}
// Diffs is a list of Diff
type Diffs []Diff
// DiffFormat describes how a list of diffs should be formatted
type DiffFormat struct {
TestName bool
Delta bool
PercentChangeAB bool
PercentChangeBA bool
MultiplierChangeAB bool
MultiplierChangeBA bool
TimeA bool
TimeB bool
}
func (diffs Diffs) Format(f DiffFormat) string {
if len(diffs) == 0 {
return "<no changes>"
}
type row []string
header := row{}
if f.TestName {
header = append(header, "Test name")
}
if f.Delta {
header = append(header, "Δ (A → B)")
}
if f.PercentChangeAB {
header = append(header, "% (A → B)")
}
if f.PercentChangeBA {
header = append(header, "% (B → A)")
}
if f.MultiplierChangeAB {
header = append(header, "× (A → B)")
}
if f.MultiplierChangeBA {
header = append(header, "× (B → A)")
}
if f.TimeA {
header = append(header, "A")
}
if f.TimeB {
header = append(header, "B")
}
if len(header) == 0 {
return ""
}
columns := []row{}
for _, d := range diffs {
r := make(row, 0, len(header))
if f.TestName {
r = append(r, d.TestName)
}
if f.Delta {
r = append(r, fmt.Sprintf("%v", d.Delta))
}
if f.PercentChangeAB {
r = append(r, fmt.Sprintf("%+2.1f%%", d.PercentChangeAB))
}
if f.PercentChangeBA {
r = append(r, fmt.Sprintf("%+2.1f%%", d.PercentChangeBA))
}
if f.MultiplierChangeAB {
r = append(r, fmt.Sprintf("%+.4f", d.MultiplierChangeAB))
}
if f.MultiplierChangeBA {
r = append(r, fmt.Sprintf("%+.4f", d.MultiplierChangeBA))
}
if f.TimeA {
r = append(r, fmt.Sprintf("%v", d.TimeA))
}
if f.TimeB {
r = append(r, fmt.Sprintf("%v", d.TimeB))
}
columns = append(columns, r)
}
// measure
widths := make([]int, len(header))
for i, h := range header {
widths[i] = utf8.RuneCountInString(h)
}
for _, row := range columns {
for i, cell := range row {
l := utf8.RuneCountInString(cell)
if widths[i] < l {
widths[i] = l
}
}
}
pad := func(s string, i int) string {
if n := i - utf8.RuneCountInString(s); n > 0 {
return s + strings.Repeat(" ", n)
}
return s
}
// Draw table
b := &strings.Builder{}
horizontal_bar := func() {
for i := range header {
fmt.Fprintf(b, "+%v", strings.Repeat("-", 2+widths[i]))
}
fmt.Fprintln(b, "+")
}
horizontal_bar()
for i, h := range header {
fmt.Fprintf(b, "| %v ", pad(h, widths[i]))
}
fmt.Fprintln(b, "|")
horizontal_bar()
for _, row := range columns {
for i, cell := range row {
fmt.Fprintf(b, "| %v ", pad(cell, widths[i]))
}
fmt.Fprintln(b, "|")
}
horizontal_bar()
return b.String()
}
// Compare returns a string describing differences in the two benchmarks
// Absolute benchmark differences less than minDiff are omitted
// Absolute relative differences between [1, 1+x] are omitted
func Compare(a, b []Benchmark, minDiff time.Duration, minRelDiff float64) Diffs {
type times struct {
a time.Duration
b time.Duration
}
byName := map[string]times{}
for _, test := range a {
byName[test.Name] = times{a: test.Duration}
}
for _, test := range b {
t := byName[test.Name]
t.b = test.Duration
byName[test.Name] = t
}
type delta struct {
name string
times times
relDiff float64
absRelDiff float64
}
deltas := []delta{}
for name, times := range byName {
if times.a == 0 || times.b == 0 {
continue // Assuming test was missing from a or b
}
diff := times.b - times.a
absDiff := diff
if absDiff < 0 {
absDiff = -absDiff
}
if absDiff < minDiff {
continue
}
relDiff := float64(times.b) / float64(times.a)
absRelDiff := relDiff
if absRelDiff < 1 {
absRelDiff = 1.0 / absRelDiff
}
if absRelDiff < (1.0 + minRelDiff) {
continue
}
d := delta{
name: name,
times: times,
relDiff: relDiff,
absRelDiff: absRelDiff,
}
deltas = append(deltas, d)
}
sort.Slice(deltas, func(i, j int) bool { return deltas[j].relDiff < deltas[i].relDiff })
out := make(Diffs, len(deltas))
for i, delta := range deltas {
a2b := delta.times.b - delta.times.a
out[i] = Diff{
TestName: delta.name,
Delta: a2b,
PercentChangeAB: 100 * float64(a2b) / float64(delta.times.a),
PercentChangeBA: 100 * float64(-a2b) / float64(delta.times.b),
MultiplierChangeAB: float64(delta.times.b) / float64(delta.times.a),
MultiplierChangeBA: float64(delta.times.a) / float64(delta.times.b),
TimeA: delta.times.a,
TimeB: delta.times.b,
}
}
return out
}
func trimAggregateSuffix(name string) string {
name = strings.TrimSuffix(name, "_stddev")
name = strings.TrimSuffix(name, "_mean")
name = strings.TrimSuffix(name, "_median")
return name
}

252
tools/src/bench/bench_test.go Normal file
View File

@@ -0,0 +1,252 @@
// Copyright 2022 The Tint Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bench_test
import (
"reflect"
"testing"
"time"
"dawn.googlesource.com/tint/tools/src/bench"
)
func TestParseJson(t *testing.T) {
json := `
{
"context": {
"date": "2022-01-24T10:28:13+00:00",
"host_name": "hostname",
"executable": "./myexe",
"num_cpus": 16,
"mhz_per_cpu": 2400,
"cpu_scaling_enabled": false,
"caches": [
{
"type": "Data",
"level": 1,
"size": 32768,
"num_sharing": 2
},
{
"type": "Instruction",
"level": 1,
"size": 32768,
"num_sharing": 2
},
{
"type": "Unified",
"level": 2,
"size": 262144,
"num_sharing": 2
},
{
"type": "Unified",
"level": 3,
"size": 16777216,
"num_sharing": 16
}
],
"load_avg": [2.60938,2.59863,2.55566],
"library_build_type": "release"
},
"benchmarks": [
{
"name": "MyBenchmark",
"family_index": 0,
"per_family_instance_index": 0,
"run_name": "MyBenchmark",
"run_type": "iteration",
"repetitions": 2,
"repetition_index": 0,
"threads": 1,
"iterations": 402,
"real_time": 1.6392272438353568e+06,
"cpu_time": 1.6387412935323382e+06,
"time_unit": "ns"
},
{
"name": "MyBenchmark",
"family_index": 0,
"per_family_instance_index": 0,
"run_name": "MyBenchmark",
"run_type": "iteration",
"repetitions": 2,
"repetition_index": 1,
"threads": 1,
"iterations": 402,
"real_time": 1.7143936117703272e+06,
"cpu_time": 1.7124004975124374e+06,
"time_unit": "ns"
},
{
"name": "MyBenchmark_mean",
"family_index": 0,
"per_family_instance_index": 0,
"run_name": "MyBenchmark",
"run_type": "aggregate",
"repetitions": 2,
"threads": 1,
"aggregate_name": "mean",
"iterations": 2,
"real_time": 1.6768104278028419e+06,
"cpu_time": 1.6755708955223879e+06,
"time_unit": "ns"
},
{
"name": "MyBenchmark_median",
"family_index": 0,
"per_family_instance_index": 0,
"run_name": "MyBenchmark",
"run_type": "aggregate",
"repetitions": 2,
"threads": 1,
"aggregate_name": "median",
"iterations": 2,
"real_time": 1.6768104278028419e+06,
"cpu_time": 1.6755708955223879e+06,
"time_unit": "ns"
},
{
"name": "MyBenchmark_stddev",
"family_index": 0,
"per_family_instance_index": 0,
"run_name": "MyBenchmark",
"run_type": "aggregate",
"repetitions": 2,
"threads": 1,
"aggregate_name": "stddev",
"iterations": 2,
"real_time": 5.3150648483981553e+04,
"cpu_time": 5.2084922631119407e+04,
"time_unit": "ns"
}
]
}
`
got, err := bench.Parse(json)
if err != nil {
t.Errorf("bench.Parse() returned %v", err)
return
}
expectedDate, err := time.Parse(time.RFC1123, "Mon, 24 Jan 2022 10:28:13 GMT")
if err != nil {
t.Errorf("time.Parse() returned %v", err)
return
}
expect := bench.Run{
Benchmarks: []bench.Benchmark{
{Name: "MyBenchmark", Duration: time.Nanosecond * 1639227, AggregateType: ""},
{Name: "MyBenchmark", Duration: time.Nanosecond * 1714393, AggregateType: ""},
{Name: "MyBenchmark", Duration: time.Nanosecond * 1676810, AggregateType: "mean"},
{Name: "MyBenchmark", Duration: time.Nanosecond * 1676810, AggregateType: "median"},
{Name: "MyBenchmark", Duration: time.Nanosecond * 53150, AggregateType: "stddev"},
},
Context: &bench.Context{
Date: expectedDate,
HostName: "hostname",
Executable: "./myexe",
NumCPUs: 16,
MhzPerCPU: 2400, CPUScalingEnabled: false,
Caches: []bench.ContextCache{
{Type: "Data", Level: 1, Size: 32768, NumSharing: 2},
{Type: "Instruction", Level: 1, Size: 32768, NumSharing: 2},
{Type: "Unified", Level: 2, Size: 262144, NumSharing: 2},
{Type: "Unified", Level: 3, Size: 16777216, NumSharing: 16},
},
LoadAvg: []float32{2.60938, 2.59863, 2.55566}, LibraryBuildType: "release"},
}
expectEqual(t, "bench.Parse().Benchmarks", got.Benchmarks, expect.Benchmarks)
expectEqual(t, "bench.Parse().Context", got.Benchmarks, expect.Benchmarks)
}
func TestCompare(t *testing.T) {
a := []bench.Benchmark{
{Name: "MyBenchmark1", Duration: time.Nanosecond * 1714393},
{Name: "MyBenchmark0", Duration: time.Nanosecond * 1639227},
{Name: "MyBenchmark3", Duration: time.Nanosecond * 1676810},
{Name: "MyBenchmark4", Duration: time.Nanosecond * 53150},
{Name: "MyBenchmark2", Duration: time.Nanosecond * 1676810},
}
b := []bench.Benchmark{
{Name: "MyBenchmark1", Duration: time.Nanosecond * 56747654},
{Name: "MyBenchmark0", Duration: time.Nanosecond * 236246},
{Name: "MyBenchmark2", Duration: time.Nanosecond * 675865},
{Name: "MyBenchmark4", Duration: time.Nanosecond * 2352336},
{Name: "MyBenchmark3", Duration: time.Nanosecond * 87657868},
}
minDiff := time.Millisecond * 2
minRelDiff := 35.0
cmp := bench.Compare(a, b, minDiff, minRelDiff)
expectEqual(t, "bench.Compare().Format", cmp.Format(bench.DiffFormat{}), "")
expectEqual(t, "bench.Compare().Format", "\n"+cmp.Format(bench.DiffFormat{TimeA: true}), `
+-----------+
| A |
+-----------+
| 1.67681ms |
| 53.15µs |
+-----------+
`)
expectEqual(t, "bench.Compare().Format", "\n"+cmp.Format(bench.DiffFormat{TimeA: true, TimeB: true}), `
+-----------+-------------+
| A | B |
+-----------+-------------+
| 1.67681ms | 87.657868ms |
| 53.15µs | 2.352336ms |
+-----------+-------------+
`)
expectEqual(t, "bench.Compare().Format", "\n"+cmp.Format(bench.DiffFormat{
TestName: true,
Delta: true,
PercentChangeAB: true,
TimeA: true,
TimeB: true,
}), `
+--------------+-------------+-----------+-----------+-------------+
| Test name | Δ (A → B) | % (A → B) | A | B |
+--------------+-------------+-----------+-----------+-------------+
| MyBenchmark3 | 85.981058ms | +5127.7% | 1.67681ms | 87.657868ms |
| MyBenchmark4 | 2.299186ms | +4325.8% | 53.15µs | 2.352336ms |
+--------------+-------------+-----------+-----------+-------------+
`)
expectEqual(t, "bench.Compare().Format", "\n"+cmp.Format(bench.DiffFormat{
TestName: true,
Delta: true,
PercentChangeAB: true,
PercentChangeBA: true,
MultiplierChangeAB: true,
MultiplierChangeBA: true,
TimeA: true,
TimeB: true,
}), `
+--------------+-------------+-----------+-----------+-----------+-----------+-----------+-------------+
| Test name | Δ (A → B) | % (A → B) | % (B → A) | × (A → B) | × (B → A) | A | B |
+--------------+-------------+-----------+-----------+-----------+-----------+-----------+-------------+
| MyBenchmark3 | 85.981058ms | +5127.7% | -98.1% | +52.2766 | +0.0191 | 1.67681ms | 87.657868ms |
| MyBenchmark4 | 2.299186ms | +4325.8% | -97.7% | +44.2584 | +0.0226 | 53.15µs | 2.352336ms |
+--------------+-------------+-----------+-----------+-----------+-----------+-----------+-------------+
`)
}
func expectEqual(t *testing.T, desc string, got, expect interface{}) {
if !reflect.DeepEqual(got, expect) {
t.Errorf("%v was not expected:\nGot:\n%+v\nExpected:\n%+v", desc, got, expect)
}
}

96
tools/src/cmd/benchdiff/main.go Normal file
View File

@@ -0,0 +1,96 @@
// Copyright 2022 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// benchdiff is a tool that compares two Google benchmark results and displays
// sorted performance differences.
package main
import (
"errors"
"flag"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"time"
"dawn.googlesource.com/tint/tools/src/bench"
)
var (
minDiff = flag.Duration("min-diff", time.Microsecond*10, "Filter away time diffs less than this duration")
minRelDiff = flag.Float64("min-rel-diff", 0.01, "Filter away absolute relative diffs between [1, 1+x]")
)
func main() {
flag.ErrHelp = errors.New("benchdiff is a tool to compare two benchmark results")
flag.Parse()
flag.Usage = func() {
fmt.Fprintln(os.Stderr, "benchdiff <benchmark-a> <benchmark-b>")
flag.PrintDefaults()
}
args := flag.Args()
if len(args) < 2 {
flag.Usage()
os.Exit(1)
}
pathA, pathB := args[0], args[1]
if err := run(pathA, pathB); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(-1)
}
}
func run(pathA, pathB string) error {
fileA, err := ioutil.ReadFile(pathA)
if err != nil {
return err
}
benchA, err := bench.Parse(string(fileA))
if err != nil {
return err
}
fileB, err := ioutil.ReadFile(pathB)
if err != nil {
return err
}
benchB, err := bench.Parse(string(fileB))
if err != nil {
return err
}
cmp := bench.Compare(benchA.Benchmarks, benchB.Benchmarks, *minDiff, *minRelDiff)
diff := cmp.Format(bench.DiffFormat{
TestName: true,
Delta: true,
PercentChangeAB: true,
TimeA: true,
TimeB: true,
})
fmt.Println("A:", pathA, " B:", pathB)
fmt.Println()
fmt.Println(diff)
return nil
}
func fileName(path string) string {
_, name := filepath.Split(path)
return name
}

307
tools/src/cmd/builtin-gen/ast/ast.go Normal file
View File

@@ -0,0 +1,307 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package ast defines AST nodes that are produced by the Tint intrinsic
// definition parser
package ast
import (
"fmt"
"strings"
"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/tok"
)
// AST is the parsed syntax tree of the intrinsic definition file
type AST struct {
Enums []EnumDecl
Types []TypeDecl
Matchers []MatcherDecl
Functions []FunctionDecl
}
func (a AST) String() string {
sb := strings.Builder{}
for _, e := range a.Enums {
fmt.Fprintf(&sb, "%v", e)
fmt.Fprintln(&sb)
}
for _, p := range a.Types {
fmt.Fprintf(&sb, "%v", p)
fmt.Fprintln(&sb)
}
for _, m := range a.Matchers {
fmt.Fprintf(&sb, "%v", m)
fmt.Fprintln(&sb)
}
for _, f := range a.Functions {
fmt.Fprintf(&sb, "%v", f)
fmt.Fprintln(&sb)
}
return sb.String()
}
// EnumDecl describes an enumerator
type EnumDecl struct {
Source tok.Source
Name string
Entries []EnumEntry
}
// Format implements the fmt.Formatter interface
func (e EnumDecl) Format(w fmt.State, verb rune) {
fmt.Fprintf(w, "enum %v {\n", e.Name)
for _, e := range e.Entries {
fmt.Fprintf(w, " %v\n", e)
}
fmt.Fprintf(w, "}\n")
}
// EnumEntry describes an entry in a enumerator
type EnumEntry struct {
Source tok.Source
Name string
Decorations Decorations
}
// Format implements the fmt.Formatter interface
func (e EnumEntry) Format(w fmt.State, verb rune) {
if len(e.Decorations) > 0 {
fmt.Fprintf(w, "%v %v", e.Decorations, e.Name)
} else {
fmt.Fprint(w, e.Name)
}
}
// MatcherDecl describes a matcher declaration
type MatcherDecl struct {
Source tok.Source
Name string
Options MatcherOptions
}
// Format implements the fmt.Formatter interface
func (m MatcherDecl) Format(w fmt.State, verb rune) {
fmt.Fprintf(w, "match %v", m.Name)
fmt.Fprintf(w, ": ")
m.Options.Format(w, verb)
}
// FunctionDecl describes a function declaration
type FunctionDecl struct {
Source tok.Source
Name string
Decorations Decorations
TemplateParams TemplateParams
Parameters Parameters
ReturnType *TemplatedName
}
// Format implements the fmt.Formatter interface
func (f FunctionDecl) Format(w fmt.State, verb rune) {
fmt.Fprintf(w, "fn %v", f.Name)
f.TemplateParams.Format(w, verb)
f.Parameters.Format(w, verb)
if f.ReturnType != nil {
fmt.Fprintf(w, " -> ")
f.ReturnType.Format(w, verb)
}
}
// Parameters is a list of parameter
type Parameters []Parameter
// Format implements the fmt.Formatter interface
func (l Parameters) Format(w fmt.State, verb rune) {
fmt.Fprintf(w, "(")
for i, p := range l {
if i > 0 {
fmt.Fprintf(w, ", ")
}
p.Format(w, verb)
}
fmt.Fprintf(w, ")")
}
// Parameter describes a single parameter of a function
type Parameter struct {
Source tok.Source
Name string // Optional
Type TemplatedName
}
// Format implements the fmt.Formatter interface
func (p Parameter) Format(w fmt.State, verb rune) {
if p.Name != "" {
fmt.Fprintf(w, "%v: ", p.Name)
}
p.Type.Format(w, verb)
}
// MatcherOptions is a list of TemplatedName
type MatcherOptions TemplatedNames
// Format implements the fmt.Formatter interface
func (o MatcherOptions) Format(w fmt.State, verb rune) {
for i, mo := range o {
if i > 0 {
fmt.Fprintf(w, " | ")
}
mo.Format(w, verb)
}
}
// TemplatedNames is a list of TemplatedName
// Example:
// a<b>, c<d, e>
type TemplatedNames []TemplatedName
// Format implements the fmt.Formatter interface
func (l TemplatedNames) Format(w fmt.State, verb rune) {
for i, n := range l {
if i > 0 {
fmt.Fprintf(w, ", ")
}
n.Format(w, verb)
}
}
// TemplatedName is an identifier with optional templated arguments
// Example:
// vec<N, T>
type TemplatedName struct {
Source tok.Source
Name string
TemplateArgs TemplatedNames
}
// Format implements the fmt.Formatter interface
func (t TemplatedName) Format(w fmt.State, verb rune) {
fmt.Fprintf(w, "%v", t.Name)
if len(t.TemplateArgs) > 0 {
fmt.Fprintf(w, "<")
t.TemplateArgs.Format(w, verb)
fmt.Fprintf(w, ">")
}
}
// TypeDecl describes a type declaration
type TypeDecl struct {
Source tok.Source
Decorations Decorations
Name string
TemplateParams TemplateParams
}
// Format implements the fmt.Formatter interface
func (p TypeDecl) Format(w fmt.State, verb rune) {
if len(p.Decorations) > 0 {
p.Decorations.Format(w, verb)
fmt.Fprintf(w, " type %v", p.Name)
}
fmt.Fprintf(w, "type %v", p.Name)
p.TemplateParams.Format(w, verb)
}
// TemplateParams is a list of TemplateParam
// Example:
// <A, B : TyB>
type TemplateParams []TemplateParam
// Format implements the fmt.Formatter interface
func (p TemplateParams) Format(w fmt.State, verb rune) {
if len(p) > 0 {
fmt.Fprintf(w, "<")
for i, tp := range p {
if i > 0 {
fmt.Fprintf(w, ", ")
}
tp.Format(w, verb)
}
fmt.Fprintf(w, ">")
}
}
// TemplateParam describes a template parameter with optional type
// Example:
// <Name>
// <Name: Type>
type TemplateParam struct {
Source tok.Source
Name string
Type TemplatedName // Optional
}
// Format implements the fmt.Formatter interface
func (t TemplateParam) Format(w fmt.State, verb rune) {
fmt.Fprintf(w, "%v", t.Name)
if t.Type.Name != "" {
fmt.Fprintf(w, " : ")
t.Type.Format(w, verb)
}
}
// Decorations is a list of Decoration
// Example:
// [[a(x), b(y)]]
type Decorations []Decoration
// Format implements the fmt.Formatter interface
func (l Decorations) Format(w fmt.State, verb rune) {
fmt.Fprint(w, "[[")
for i, d := range l {
if i > 0 {
fmt.Fprintf(w, ", ")
}
d.Format(w, verb)
}
fmt.Fprint(w, "]]")
}
// Take looks up the decoration with the given name. If the decoration is found
// it is removed from the Decorations list and returned, otherwise nil is
// returned and the Decorations are not altered.
func (l *Decorations) Take(name string) *Decoration {
for i, d := range *l {
if d.Name == name {
*l = append((*l)[:i], (*l)[i+1:]...)
return &d
}
}
return nil
}
// Decoration describes a single decoration
// Example:
// a(x)
type Decoration struct {
Source tok.Source
Name string
Values []string
}
// Format implements the fmt.Formatter interface
func (d Decoration) Format(w fmt.State, verb rune) {
fmt.Fprintf(w, "%v", d.Name)
if len(d.Values) > 0 {
fmt.Fprintf(w, "(")
for i, v := range d.Values {
if i > 0 {
fmt.Fprint(w, ", ")
}
fmt.Fprintf(w, "%v", v)
}
fmt.Fprintf(w, ")")
}
}

387
tools/src/cmd/builtin-gen/gen/builtin_table.go Normal file
View File

@@ -0,0 +1,387 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gen
import (
"fmt"
"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/sem"
"dawn.googlesource.com/tint/tools/src/list"
"dawn.googlesource.com/tint/tools/src/lut"
)
// BuiltinTable holds data specific to the intrinsic_table.inl.tmpl template
type BuiltinTable struct {
// The semantic info
Sem *sem.Sem
// TMatchers are all the sem.OpenType, sem.Type and sem.TypeMatchers.
// These are all implemented by classes deriving from tint::TypeMatcher
TMatchers []sem.Named
TMatcherIndex map[sem.Named]int // [object -> index] in TMatcher
// NMatchers are all the sem.OpenNumber and sem.EnumMatchers.
// These are all implemented by classes deriving from tint::NumberMatcher
NMatchers []sem.Named
NMatcherIndex map[sem.Named]int // [object -> index] in NMatchers
MatcherIndices []int // kMatcherIndices table content
OpenTypes []OpenType // kOpenTypes table content
OpenNumbers []OpenNumber // kOpenNumbers table content
Parameters []Parameter // kParameters table content
Overloads []Overload // kOverloads table content
Functions []Function // kBuiltins table content
}
// OpenType is used to create the C++ OpenTypeInfo structure
type OpenType struct {
// Name of the open type (e.g. 'T')
Name string
// Optional type matcher constraint.
// Either an index in Matchers::type, or -1
MatcherIndex int
}
// OpenNumber is used to create the C++ OpenNumberInfo structure
type OpenNumber struct {
// Name of the open number (e.g. 'N')
Name string
// Optional type matcher constraint.
// Either an index in Matchers::type, or -1
MatcherIndex int
}
// Parameter is used to create the C++ ParameterInfo structure
type Parameter struct {
// The parameter usage (parameter name)
Usage string
// Index into BuiltinTable.MatcherIndices, beginning the list of matchers
// required to match the parameter type. The matcher indices index
// into BuiltinTable::TMatchers and / or BuiltinTable::NMatchers.
// These indices are consumed by the matchers themselves.
// The first index is always a TypeMatcher.
MatcherIndicesOffset *int
}
// Overload is used to create the C++ OverloadInfo structure
type Overload struct {
// Total number of parameters for the overload
NumParameters int
// Total number of open types for the overload
NumOpenTypes int
// Total number of open numbers for the overload
NumOpenNumbers int
// Index to the first open type in BuiltinTable.OpenTypes
OpenTypesOffset *int
// Index to the first open number in BuiltinTable.OpenNumbers
OpenNumbersOffset *int
// Index to the first parameter in BuiltinTable.Parameters
ParametersOffset *int
// Index into BuiltinTable.MatcherIndices, beginning the list of matchers
// required to match the return type. The matcher indices index
// into BuiltinTable::TMatchers and / or BuiltinTable::NMatchers.
// These indices are consumed by the matchers themselves.
// The first index is always a TypeMatcher.
ReturnMatcherIndicesOffset *int
// StageUses describes the stages an overload can be used in
CanBeUsedInStage sem.StageUses
// True if the overload is marked as deprecated
IsDeprecated bool
}
// Function is used to create the C++ IntrinsicInfo structure
type Function struct {
OverloadDescriptions []string
NumOverloads int
OverloadsOffset *int
}
// Helper for building the BuiltinTable
type BuiltinTableBuilder struct {
// The output of the builder
BuiltinTable
// Lookup tables.
// These are packed (compressed) once all the entries have been added.
lut struct {
matcherIndices lut.LUT
openTypes lut.LUT
openNumbers lut.LUT
parameters lut.LUT
overloads lut.LUT
}
}
// Helper for building a single overload
type overloadBuilder struct {
*BuiltinTableBuilder
// Maps TemplateParam to index in openTypes
openTypeIndex map[sem.TemplateParam]int
// Maps TemplateParam to index in openNumbers
openNumberIndex map[sem.TemplateParam]int
// Open types used by the overload
openTypes []OpenType
// Open numbers used by the overload
openNumbers []OpenNumber
// All parameters declared by the overload
parameters []Parameter
// Index into BuiltinTable.MatcherIndices, beginning the list of matchers
// required to match the return type. The matcher indices index
// into BuiltinTable::TMatchers and / or BuiltinTable::NMatchers.
// These indices are consumed by the matchers themselves.
// The first index is always a TypeMatcher.
returnTypeMatcherIndicesOffset *int
}
// layoutMatchers assigns each of the TMatchers and NMatchers a unique index
// in the C++ Matchers::type and Matchers::number arrays, respectively.
func (b *BuiltinTableBuilder) layoutMatchers(s *sem.Sem) {
// First MaxOpenTypes of TMatchers are open types
b.TMatchers = make([]sem.Named, s.MaxOpenTypes)
for _, m := range s.Types {
b.TMatcherIndex[m] = len(b.TMatchers)
b.TMatchers = append(b.TMatchers, m)
}
for _, m := range s.TypeMatchers {
b.TMatcherIndex[m] = len(b.TMatchers)
b.TMatchers = append(b.TMatchers, m)
}
// First MaxOpenNumbers of NMatchers are open numbers
b.NMatchers = make([]sem.Named, s.MaxOpenNumbers)
for _, m := range s.EnumMatchers {
b.NMatcherIndex[m] = len(b.NMatchers)
b.NMatchers = append(b.NMatchers, m)
}
}
// buildOverload constructs an Overload for a sem.Overload
func (b *BuiltinTableBuilder) buildOverload(o *sem.Overload) (Overload, error) {
ob := overloadBuilder{
BuiltinTableBuilder: b,
openTypeIndex: map[sem.TemplateParam]int{},
openNumberIndex: map[sem.TemplateParam]int{},
}
if err := ob.buildOpenTypes(o); err != nil {
return Overload{}, err
}
if err := ob.buildOpenNumbers(o); err != nil {
return Overload{}, err
}
if err := ob.buildParameters(o); err != nil {
return Overload{}, err
}
if err := ob.buildReturnType(o); err != nil {
return Overload{}, err
}
return Overload{
NumParameters: len(ob.parameters),
NumOpenTypes: len(ob.openTypes),
NumOpenNumbers: len(ob.openNumbers),
OpenTypesOffset: b.lut.openTypes.Add(ob.openTypes),
OpenNumbersOffset: b.lut.openNumbers.Add(ob.openNumbers),
ParametersOffset: b.lut.parameters.Add(ob.parameters),
ReturnMatcherIndicesOffset: ob.returnTypeMatcherIndicesOffset,
CanBeUsedInStage: o.CanBeUsedInStage,
IsDeprecated: o.IsDeprecated,
}, nil
}
// buildOpenTypes constructs the OpenTypes used by the overload, populating
// b.openTypes
func (b *overloadBuilder) buildOpenTypes(o *sem.Overload) error {
b.openTypes = make([]OpenType, len(o.OpenTypes))
for i, t := range o.OpenTypes {
b.openTypeIndex[t] = i
matcherIndex := -1
if t.Type != nil {
var err error
matcherIndex, err = b.matcherIndex(t.Type)
if err != nil {
return err
}
}
b.openTypes[i] = OpenType{
Name: t.Name,
MatcherIndex: matcherIndex,
}
}
return nil
}
// buildOpenNumbers constructs the OpenNumbers used by the overload, populating
// b.openNumbers
func (b *overloadBuilder) buildOpenNumbers(o *sem.Overload) error {
b.openNumbers = make([]OpenNumber, len(o.OpenNumbers))
for i, t := range o.OpenNumbers {
b.openNumberIndex[t] = i
matcherIndex := -1
if e, ok := t.(*sem.TemplateEnumParam); ok && e.Matcher != nil {
var err error
matcherIndex, err = b.matcherIndex(e.Matcher)
if err != nil {
return err
}
}
b.openNumbers[i] = OpenNumber{
Name: t.GetName(),
MatcherIndex: matcherIndex,
}
}
return nil
}
// buildParameters constructs the Parameters used by the overload, populating
// b.parameters
func (b *overloadBuilder) buildParameters(o *sem.Overload) error {
b.parameters = make([]Parameter, len(o.Parameters))
for i, p := range o.Parameters {
indices, err := b.collectMatcherIndices(p.Type)
if err != nil {
return err
}
b.parameters[i] = Parameter{
Usage: p.Name,
MatcherIndicesOffset: b.lut.matcherIndices.Add(indices),
}
}
return nil
}
// buildParameters calculates the matcher indices required to match the
// overload's return type (if the overload has a return value), possibly
// populating b.returnTypeMatcherIndicesOffset
func (b *overloadBuilder) buildReturnType(o *sem.Overload) error {
if o.ReturnType != nil {
indices, err := b.collectMatcherIndices(*o.ReturnType)
if err != nil {
return err
}
b.returnTypeMatcherIndicesOffset = b.lut.matcherIndices.Add(indices)
}
return nil
}
// matcherIndex returns the index of TMatcher or NMatcher in
// BuiltinTable.TMatcher or BuiltinTable.NMatcher, respectively.
func (b *overloadBuilder) matcherIndex(n sem.Named) (int, error) {
switch n := n.(type) {
case *sem.Type, *sem.TypeMatcher:
if i, ok := b.TMatcherIndex[n]; ok {
return i, nil
}
return 0, fmt.Errorf("matcherIndex missing entry for %v %T", n.GetName(), n)
case *sem.TemplateTypeParam:
if i, ok := b.openTypeIndex[n]; ok {
return i, nil
}
return 0, fmt.Errorf("openTypeIndex missing entry for %v %T", n.Name, n)
case *sem.EnumMatcher:
if i, ok := b.NMatcherIndex[n]; ok {
return i, nil
}
return 0, fmt.Errorf("matcherIndex missing entry for %v %T", n.GetName(), n)
case *sem.TemplateEnumParam:
if i, ok := b.openNumberIndex[n]; ok {
return i, nil
}
return 0, fmt.Errorf("openNumberIndex missing entry for %v %T", n, n)
case *sem.TemplateNumberParam:
if i, ok := b.openNumberIndex[n]; ok {
return i, nil
}
return 0, fmt.Errorf("openNumberIndex missing entry for %v %T", n, n)
default:
return 0, fmt.Errorf("overload.matcherIndex() does not handle %v %T", n, n)
}
}
// collectMatcherIndices returns the full list of matcher indices required to
// match the fully-qualified-name. For names that have do not have templated
// arguments, collectMatcherIndices() will return a single TMatcher index.
// For names that do have templated arguments, collectMatcherIndices() returns
// a list of type matcher indices, starting with the target of the fully
// qualified name, then followed by each of the template arguments from left to
// right. Note that template arguments may themselves have template arguments,
// and so collectMatcherIndices() may call itself.
// The order of returned matcher indices is always the order of the fully
// qualified name as read from left to right.
// For example, calling collectMatcherIndices() for the fully qualified name:
// A<B<C, D>, E<F, G<H>, I>
// Would return the matcher indices:
// A, B, C, D, E, F, G, H, I
func (b *overloadBuilder) collectMatcherIndices(fqn sem.FullyQualifiedName) ([]int, error) {
idx, err := b.matcherIndex(fqn.Target)
if err != nil {
return nil, err
}
out := []int{idx}
for _, arg := range fqn.TemplateArguments {
indices, err := b.collectMatcherIndices(arg.(sem.FullyQualifiedName))
if err != nil {
return nil, err
}
out = append(out, indices...)
}
return out, nil
}
// buildBuiltinTable builds the BuiltinTable from the semantic info
func buildBuiltinTable(s *sem.Sem) (*BuiltinTable, error) {
b := BuiltinTableBuilder{
BuiltinTable: BuiltinTable{
Sem: s,
TMatcherIndex: map[sem.Named]int{},
NMatcherIndex: map[sem.Named]int{},
},
}
b.lut.matcherIndices = lut.New(list.Wrap(&b.MatcherIndices))
b.lut.openTypes = lut.New(list.Wrap(&b.OpenTypes))
b.lut.openNumbers = lut.New(list.Wrap(&b.OpenNumbers))
b.lut.parameters = lut.New(list.Wrap(&b.Parameters))
b.lut.overloads = lut.New(list.Wrap(&b.Overloads))
b.layoutMatchers(s)
for _, f := range s.Functions {
overloads := make([]Overload, len(f.Overloads))
overloadDescriptions := make([]string, len(f.Overloads))
for i, o := range f.Overloads {
overloadDescriptions[i] = fmt.Sprint(o.Decl)
var err error
if overloads[i], err = b.buildOverload(o); err != nil {
return nil, err
}
}
b.Functions = append(b.Functions, Function{
OverloadDescriptions: overloadDescriptions,
NumOverloads: len(overloads),
OverloadsOffset: b.lut.overloads.Add(overloads),
})
}
b.lut.matcherIndices.Compact()
b.lut.openTypes.Compact()
b.lut.openNumbers.Compact()
b.lut.parameters.Compact()
b.lut.overloads.Compact()
return &b.BuiltinTable, nil
}

271
tools/src/cmd/builtin-gen/gen/generate.go Normal file
View File

@@ -0,0 +1,271 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gen
import (
"fmt"
"io"
"reflect"
"strings"
"text/template"
"unicode"
"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/sem"
)
type generator struct {
s *sem.Sem
t *template.Template
cached struct {
builtinTable *BuiltinTable // lazily built by builtinTable()
permuter *Permuter // lazily built by permute()
}
}
// WriteFile is a function that Generate() may call to emit a new file from a
// template.
// relpath is the relative path from the currently executing template.
// content is the file content to write.
type WriteFile func(relpath, content string) error
// Generate executes the template tmpl using the provided semantic
// information, writing the output to w.
// See https://golang.org/pkg/text/template/ for documentation on the template
// syntax.
func Generate(s *sem.Sem, tmpl string, w io.Writer, writeFile WriteFile) error {
g := generator{s: s}
return g.generate(tmpl, w, writeFile)
}
func (g *generator) generate(tmpl string, w io.Writer, writeFile WriteFile) error {
t, err := template.New("<template>").Funcs(map[string]interface{}{
"Map": newMap,
"Iterate": iterate,
"Title": strings.Title,
"PascalCase": pascalCase,
"SplitDisplayName": splitDisplayName,
"HasPrefix": strings.HasPrefix,
"HasSuffix": strings.HasSuffix,
"TrimPrefix": strings.TrimPrefix,
"TrimSuffix": strings.TrimSuffix,
"TrimLeft": strings.TrimLeft,
"TrimRight": strings.TrimRight,
"IsEnumEntry": is(sem.EnumEntry{}),
"IsEnumMatcher": is(sem.EnumMatcher{}),
"IsFQN": is(sem.FullyQualifiedName{}),
"IsInt": is(1),
"IsTemplateEnumParam": is(sem.TemplateEnumParam{}),
"IsTemplateNumberParam": is(sem.TemplateNumberParam{}),
"IsTemplateTypeParam": is(sem.TemplateTypeParam{}),
"IsType": is(sem.Type{}),
"IsDeclarable": isDeclarable,
"IsFirstIn": isFirstIn,
"IsLastIn": isLastIn,
"BuiltinTable": g.builtinTable,
"Permute": g.permute,
"Eval": g.eval,
"WriteFile": func(relpath, content string) (string, error) { return "", writeFile(relpath, content) },
}).Option("missingkey=error").
Parse(tmpl)
if err != nil {
return err
}
g.t = t
return t.Execute(w, map[string]interface{}{
"Sem": g.s,
})
}
// eval executes the sub-template with the given name and argument, returning
// the generated output
func (g *generator) eval(template string, args ...interface{}) (string, error) {
target := g.t.Lookup(template)
if target == nil {
return "", fmt.Errorf("template '%v' not found", template)
}
sb := strings.Builder{}
var err error
if len(args) == 1 {
err = target.Execute(&sb, args[0])
} else {
m := newMap()
if len(args)%2 != 0 {
return "", fmt.Errorf("Eval expects a single argument or list name-value pairs")
}
for i := 0; i < len(args); i += 2 {
name, ok := args[i].(string)
if !ok {
return "", fmt.Errorf("Eval argument %v is not a string", i)
}
m.Put(name, args[i+1])
}
err = target.Execute(&sb, m)
}
if err != nil {
return "", fmt.Errorf("while evaluating '%v': %v", template, err)
}
return sb.String(), nil
}
// builtinTable lazily calls and returns the result of buildBuiltinTable(),
// caching the result for repeated calls.
func (g *generator) builtinTable() (*BuiltinTable, error) {
if g.cached.builtinTable == nil {
var err error
g.cached.builtinTable, err = buildBuiltinTable(g.s)
if err != nil {
return nil, err
}
}
return g.cached.builtinTable, nil
}
// permute lazily calls buildPermuter(), caching the result for repeated
// calls, then passes the argument to Permutator.Permute()
func (g *generator) permute(overload *sem.Overload) ([]Permutation, error) {
if g.cached.permuter == nil {
var err error
g.cached.permuter, err = buildPermuter(g.s)
if err != nil {
return nil, err
}
}
return g.cached.permuter.Permute(overload)
}
// Map is a simple generic key-value map, which can be used in the template
type Map map[interface{}]interface{}
func newMap() Map { return Map{} }
// Put adds the key-value pair into the map.
// Put always returns an empty string so nothing is printed in the template.
func (m Map) Put(key, value interface{}) string {
m[key] = value
return ""
}
// Get looks up and returns the value with the given key. If the map does not
// contain the given key, then nil is returned.
func (m Map) Get(key interface{}) interface{} {
return m[key]
}
// is returns a function that returns true if the value passed to the function
// matches the type of 'ty'.
func is(ty interface{}) func(interface{}) bool {
rty := reflect.TypeOf(ty)
return func(v interface{}) bool {
ty := reflect.TypeOf(v)
return ty == rty || ty == reflect.PtrTo(rty)
}
}
// isFirstIn returns true if v is the first element of the given slice.
func isFirstIn(v, slice interface{}) bool {
s := reflect.ValueOf(slice)
count := s.Len()
if count == 0 {
return false
}
return s.Index(0).Interface() == v
}
// isFirstIn returns true if v is the last element of the given slice.
func isLastIn(v, slice interface{}) bool {
s := reflect.ValueOf(slice)
count := s.Len()
if count == 0 {
return false
}
return s.Index(count-1).Interface() == v
}
// iterate returns a slice of length 'n', with each element equal to its index.
// Useful for: {{- range Iterate $n -}}<this will be looped $n times>{{end}}
func iterate(n int) []int {
out := make([]int, n)
for i := range out {
out[i] = i
}
return out
}
// isDeclarable returns false if the FullyQualifiedName starts with a
// leading underscore. These are undeclarable as WGSL does not allow identifers
// to have a leading underscore.
func isDeclarable(fqn sem.FullyQualifiedName) bool {
return !strings.HasPrefix(fqn.Target.GetName(), "_")
}
// pascalCase returns the snake-case string s transformed into 'PascalCase',
// Rules:
// * The first letter of the string is capitalized
// * Characters following an underscore or number are capitalized
// * Underscores are removed from the returned string
// See: https://en.wikipedia.org/wiki/Camel_case
func pascalCase(s string) string {
b := strings.Builder{}
upper := true
for _, r := range s {
if r == '_' {
upper = true
continue
}
if upper {
b.WriteRune(unicode.ToUpper(r))
upper = false
} else {
b.WriteRune(r)
}
if unicode.IsNumber(r) {
upper = true
}
}
return b.String()
}
// splitDisplayName splits displayName into parts, where text wrapped in {}
// braces are not quoted and the rest is quoted. This is used to help process
// the string value of the [[display()]] decoration. For example:
// splitDisplayName("vec{N}<{T}>")
// would return the strings:
// [`"vec"`, `N`, `"<"`, `T`, `">"`]
func splitDisplayName(displayName string) []string {
parts := []string{}
pending := strings.Builder{}
for _, r := range displayName {
switch r {
case '{':
if pending.Len() > 0 {
parts = append(parts, fmt.Sprintf(`"%v"`, pending.String()))
pending.Reset()
}
case '}':
if pending.Len() > 0 {
parts = append(parts, pending.String())
pending.Reset()
}
default:
pending.WriteRune(r)
}
}
if pending.Len() > 0 {
parts = append(parts, fmt.Sprintf(`"%v"`, pending.String()))
}
return parts
}

380
tools/src/cmd/builtin-gen/gen/permutate.go Normal file
View File

@@ -0,0 +1,380 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gen
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"strings"
"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/sem"
"dawn.googlesource.com/tint/tools/src/fileutils"
)
// Permuter generates permutations of intrinsic overloads
type Permuter struct {
sem *sem.Sem
allTypes []sem.FullyQualifiedName
}
// buildPermuter returns a new initialized Permuter
func buildPermuter(s *sem.Sem) (*Permuter, error) {
// allTypes are the list of FQNs that are used for open, unconstrained types
allTypes := []sem.FullyQualifiedName{}
for _, ty := range s.Types {
if len(ty.TemplateParams) > 0 {
// Ignore aggregate types for now.
// TODO(bclayton): Support a limited set of aggregate types
continue
}
allTypes = append(allTypes, sem.FullyQualifiedName{Target: ty})
}
return &Permuter{
sem: s,
allTypes: allTypes,
}, nil
}
// Permutation describes a single permutation of an overload
type Permutation struct {
sem.Overload // The permutated overload signature
Desc string // Description of the overload
Hash string // Hash of the overload
}
// Permute generates a set of permutations for the given intrinsic overload
func (p *Permuter) Permute(overload *sem.Overload) ([]Permutation, error) {
state := permutationState{
Permuter: p,
closedTypes: map[sem.TemplateParam]sem.FullyQualifiedName{},
closedNumbers: map[sem.TemplateParam]interface{}{},
parameters: map[int]sem.FullyQualifiedName{},
}
out := []Permutation{}
// Map of hash to permutation description. Used to detect collisions.
hashes := map[string]string{}
// permutate appends a permutation to out.
// permutate may be chained to generate N-dimensional permutations.
permutate := func() error {
o := sem.Overload{
Decl: overload.Decl,
Function: overload.Function,
CanBeUsedInStage: overload.CanBeUsedInStage,
}
for i, p := range overload.Parameters {
ty := state.parameters[i]
if !validate(ty, &o.CanBeUsedInStage) {
return nil
}
o.Parameters = append(o.Parameters, sem.Parameter{
Name: p.Name,
Type: ty,
})
}
if overload.ReturnType != nil {
retTys, err := state.permutateFQN(*overload.ReturnType)
if err != nil {
return fmt.Errorf("while permutating return type: %w", err)
}
if len(retTys) != 1 {
return fmt.Errorf("result type not pinned")
}
o.ReturnType = &retTys[0]
}
desc := fmt.Sprint(o)
hash := sha256.Sum256([]byte(desc))
const hashLength = 6
shortHash := hex.EncodeToString(hash[:])[:hashLength]
out = append(out, Permutation{
Overload: o,
Desc: desc,
Hash: shortHash,
})
// Check for hash collisions
if existing, collision := hashes[shortHash]; collision {
return fmt.Errorf("hash '%v' collision between %v and %v\nIncrease hashLength in %v",
shortHash, existing, desc, fileutils.GoSourcePath())
}
hashes[shortHash] = desc
return nil
}
for i, param := range overload.Parameters {
i, param := i, param // Capture iterator values for anonymous function
next := permutate // Permutation chaining
permutate = func() error {
permutations, err := state.permutateFQN(param.Type)
if err != nil {
return fmt.Errorf("while processing parameter %v: %w", i, err)
}
if len(permutations) == 0 {
return fmt.Errorf("parameter %v has no permutations", i)
}
for _, fqn := range permutations {
state.parameters[i] = fqn
if err := next(); err != nil {
return err
}
}
return nil
}
}
for _, t := range overload.TemplateParams {
next := permutate // Permutation chaining
switch t := t.(type) {
case *sem.TemplateTypeParam:
types := p.allTypes
if t.Type != nil {
var err error
types, err = state.permutateFQN(sem.FullyQualifiedName{Target: t.Type})
if err != nil {
return nil, fmt.Errorf("while permutating open types: %w", err)
}
}
if len(types) == 0 {
return nil, fmt.Errorf("open type %v has no permutations", t.Name)
}
permutate = func() error {
for _, ty := range types {
state.closedTypes[t] = ty
if err := next(); err != nil {
return err
}
}
return nil
}
case *sem.TemplateEnumParam:
var permutations []sem.FullyQualifiedName
var err error
if t.Matcher != nil {
permutations, err = state.permutateFQN(sem.FullyQualifiedName{Target: t.Matcher})
} else {
permutations, err = state.permutateFQN(sem.FullyQualifiedName{Target: t.Enum})
}
if err != nil {
return nil, fmt.Errorf("while permutating open numbers: %w", err)
}
if len(permutations) == 0 {
return nil, fmt.Errorf("open type %v has no permutations", t.Name)
}
permutate = func() error {
for _, n := range permutations {
state.closedNumbers[t] = n
if err := next(); err != nil {
return err
}
}
return nil
}
case *sem.TemplateNumberParam:
// Currently all open numbers are used for vector / matrices
permutations := []int{2, 3, 4}
permutate = func() error {
for _, n := range permutations {
state.closedNumbers[t] = n
if err := next(); err != nil {
return err
}
}
return nil
}
}
}
if err := permutate(); err != nil {
return nil, fmt.Errorf("%v %v %w\nState: %v", overload.Decl.Source, overload.Decl, err, state)
}
return out, nil
}
type permutationState struct {
*Permuter
closedTypes map[sem.TemplateParam]sem.FullyQualifiedName
closedNumbers map[sem.TemplateParam]interface{}
parameters map[int]sem.FullyQualifiedName
}
func (s permutationState) String() string {
sb := &strings.Builder{}
sb.WriteString("Closed types:\n")
for ct, ty := range s.closedTypes {
fmt.Fprintf(sb, " %v: %v\n", ct.GetName(), ty)
}
sb.WriteString("Closed numbers:\n")
for cn, v := range s.closedNumbers {
fmt.Fprintf(sb, " %v: %v\n", cn.GetName(), v)
}
return sb.String()
}
func (s *permutationState) permutateFQN(in sem.FullyQualifiedName) ([]sem.FullyQualifiedName, error) {
args := append([]interface{}{}, in.TemplateArguments...)
out := []sem.FullyQualifiedName{}
// permutate appends a permutation to out.
// permutate may be chained to generate N-dimensional permutations.
var permutate func() error
switch target := in.Target.(type) {
case *sem.Type:
permutate = func() error {
out = append(out, sem.FullyQualifiedName{Target: in.Target, TemplateArguments: args})
args = append([]interface{}{}, in.TemplateArguments...)
return nil
}
case sem.TemplateParam:
if ty, ok := s.closedTypes[target]; ok {
permutate = func() error {
out = append(out, ty)
return nil
}
} else {
return nil, fmt.Errorf("'%v' was not found in closedTypes", target.GetName())
}
case *sem.TypeMatcher:
permutate = func() error {
for _, ty := range target.Types {
out = append(out, sem.FullyQualifiedName{Target: ty})
}
return nil
}
case *sem.EnumMatcher:
permutate = func() error {
for _, o := range target.Options {
if !o.IsInternal {
out = append(out, sem.FullyQualifiedName{Target: o})
}
}
return nil
}
case *sem.Enum:
permutate = func() error {
for _, e := range target.Entries {
if !e.IsInternal {
out = append(out, sem.FullyQualifiedName{Target: e})
}
}
return nil
}
default:
return nil, fmt.Errorf("unhandled target type: %T", in.Target)
}
for i, arg := range in.TemplateArguments {
i := i // Capture iterator value for anonymous functions
next := permutate // Permutation chaining
switch arg := arg.(type) {
case sem.FullyQualifiedName:
switch target := arg.Target.(type) {
case sem.TemplateParam:
if ty, ok := s.closedTypes[target]; ok {
args[i] = ty
} else if num, ok := s.closedNumbers[target]; ok {
args[i] = num
} else {
return nil, fmt.Errorf("'%v' was not found in closedTypes or closedNumbers", target.GetName())
}
default:
perms, err := s.permutateFQN(arg)
if err != nil {
return nil, fmt.Errorf("while processing template argument %v: %v", i, err)
}
if len(perms) == 0 {
return nil, fmt.Errorf("template argument %v has no permutations", i)
}
permutate = func() error {
for _, f := range perms {
args[i] = f
if err := next(); err != nil {
return err
}
}
return nil
}
}
default:
return nil, fmt.Errorf("permutateFQN() unhandled template argument type: %T", arg)
}
}
if err := permutate(); err != nil {
return nil, fmt.Errorf("while processing fully qualified name '%v': %w", in.Target.GetName(), err)
}
return out, nil
}
func validate(fqn sem.FullyQualifiedName, uses *sem.StageUses) bool {
switch fqn.Target.GetName() {
case "array":
elTy := fqn.TemplateArguments[0].(sem.FullyQualifiedName)
elTyName := elTy.Target.GetName()
switch {
case elTyName == "bool" ||
strings.Contains(elTyName, "sampler"),
strings.Contains(elTyName, "texture"):
return false // Not storable
}
case "ptr":
// https://gpuweb.github.io/gpuweb/wgsl/#storage-class
access := fqn.TemplateArguments[2].(sem.FullyQualifiedName).Target.(*sem.EnumEntry).Name
storageClass := fqn.TemplateArguments[0].(sem.FullyQualifiedName).Target.(*sem.EnumEntry).Name
switch storageClass {
case "function", "private":
if access != "read_write" {
return false
}
case "workgroup":
uses.Vertex = false
uses.Fragment = false
if access != "read_write" {
return false
}
case "uniform":
if access != "read" {
return false
}
case "storage":
if access != "read_write" && access != "read" {
return false
}
case "handle":
if access != "read" {
return false
}
default:
return false
}
}
if !isDeclarable(fqn) {
return false
}
for _, arg := range fqn.TemplateArguments {
if argFQN, ok := arg.(sem.FullyQualifiedName); ok {
if !validate(argFQN, uses) {
return false
}
}
}
return true
}

200
tools/src/cmd/builtin-gen/lexer/lexer.go Normal file
View File

@@ -0,0 +1,200 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package lexer provides a basic lexer for the Tint intrinsic definition
// language
package lexer
import (
"fmt"
"unicode"
"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/tok"
)
// Lex produces a list of tokens for the given source code
func Lex(src []rune, filepath string) ([]tok.Token, error) {
l := lexer{
tok.Location{Line: 1, Column: 1, Rune: 0, Filepath: filepath},
src,
[]tok.Token{},
}
if err := l.lex(); err != nil {
return nil, err
}
return l.tokens, nil
}
type lexer struct {
loc tok.Location
runes []rune
tokens []tok.Token
}
// lex() lexes the source, populating l.tokens
func (l *lexer) lex() error {
for {
switch l.peek(0) {
case 0:
return nil
case ' ', '\t':
l.next()
case '\n':
l.next()
case '<':
l.tok(1, tok.Lt)
case '>':
l.tok(1, tok.Gt)
case '(':
l.tok(1, tok.Lparen)
case ')':
l.tok(1, tok.Rparen)
case '{':
l.tok(1, tok.Lbrace)
case '}':
l.tok(1, tok.Rbrace)
case ':':
l.tok(1, tok.Colon)
case ',':
l.tok(1, tok.Comma)
case '|':
l.tok(1, tok.Or)
case '"':
start := l.loc
l.next() // Skip opening quote
n := l.count(toFirst('\n', '"'))
if l.peek(n) != '"' {
return fmt.Errorf("%v unterminated string", start)
}
l.tok(n, tok.String)
l.next() // Skip closing quote
default:
switch {
case l.peek(1) == '/':
l.skip(l.count(toFirst('\n')))
l.next() // Consume newline
case l.match("[[", tok.Ldeco):
case l.match("]]", tok.Rdeco):
case l.match("->", tok.Arrow):
case l.match("fn", tok.Function):
case l.match("enum", tok.Enum):
case l.match("type", tok.Type):
case l.match("match", tok.Match):
case unicode.IsLetter(l.peek(0)) || l.peek(0) == '_':
l.tok(l.count(alphaNumericOrUnderscore), tok.Identifier)
case unicode.IsNumber(l.peek(0)):
l.tok(l.count(unicode.IsNumber), tok.Integer)
default:
return fmt.Errorf("%v: unexpected '%v'", l.loc, string(l.runes[0]))
}
}
}
}
// next() consumes and returns the next rune in the source, or 0 if reached EOF
func (l *lexer) next() rune {
if len(l.runes) > 0 {
r := l.runes[0]
l.runes = l.runes[1:]
l.loc.Rune++
if r == '\n' {
l.loc.Line++
l.loc.Column = 1
} else {
l.loc.Column++
}
return r
}
return 0
}
// skip() consumes the next `n` runes in the source
func (l *lexer) skip(n int) {
for i := 0; i < n; i++ {
l.next()
}
}
// peek() returns the rune `i` runes ahead of the current position
func (l *lexer) peek(i int) rune {
if i >= len(l.runes) {
return 0
}
return l.runes[i]
}
// predicate is a function that can be passed to count()
type predicate func(r rune) bool
// count() returns the number of sequential runes from the current position that
// match the predicate `p`
func (l *lexer) count(p predicate) int {
for i := 0; i < len(l.runes); i++ {
if !p(l.peek(i)) {
return i
}
}
return len(l.runes)
}
// tok() appends a new token of kind `k` using the next `n` runes.
// The next `n` runes are consumed by tok().
func (l *lexer) tok(n int, k tok.Kind) {
start := l.loc
runes := l.runes[:n]
l.skip(n)
end := l.loc
src := tok.Source{S: start, E: end}
l.tokens = append(l.tokens, tok.Token{Kind: k, Source: src, Runes: runes})
}
// match() checks whether the next runes are equal to `s`. If they are, then
// these runes are used to append a new token of kind `k`, and match() returns
// true. If the next runes are not equal to `s` then false is returned, and no
// runes are consumed.
func (l *lexer) match(s string, kind tok.Kind) bool {
runes := []rune(s)
if len(l.runes) < len(runes) {
return false
}
for i, r := range runes {
if l.runes[i] != r {
return false
}
}
l.tok(len(runes), kind)
return true
}
// toFirst() returns a predicate that returns true if the rune is not in `runes`
// toFirst() is intended to be used with count(), so `count(toFirst('x'))` will
// count up to, but not including the number of consecutive runes that are not
// 'x'.
func toFirst(runes ...rune) predicate {
return func(r rune) bool {
for _, t := range runes {
if t == r {
return false
}
}
return true
}
}
// alphaNumericOrUnderscore() returns true if the rune `r` is a number, letter
// or underscore.
func alphaNumericOrUnderscore(r rune) bool {
return r == '_' || unicode.IsLetter(r) || unicode.IsNumber(r)
}

147
tools/src/cmd/builtin-gen/lexer/lexer_test.go Normal file
View File

@@ -0,0 +1,147 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package lexer_test
import (
"fmt"
"testing"
"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/lexer"
"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/tok"
)
func TestLexTokens(t *testing.T) {
type test struct {
src string
expect tok.Token
}
filepath := "test.txt"
loc := func(l, c, r int) tok.Location {
return tok.Location{Line: l, Column: c, Rune: r, Filepath: filepath}
}
for _, test := range []test{
{"ident", tok.Token{Kind: tok.Identifier, Runes: []rune("ident"), Source: tok.Source{
S: loc(1, 1, 0), E: loc(1, 6, 5),
}}},
{"ident_123", tok.Token{Kind: tok.Identifier, Runes: []rune("ident_123"), Source: tok.Source{
S: loc(1, 1, 0), E: loc(1, 10, 9),
}}},
{"_ident_", tok.Token{Kind: tok.Identifier, Runes: []rune("_ident_"), Source: tok.Source{
S: loc(1, 1, 0), E: loc(1, 8, 7),
}}},
{"123456789", tok.Token{Kind: tok.Integer, Runes: []rune("123456789"), Source: tok.Source{
S: loc(1, 1, 0), E: loc(1, 10, 9),
}}},
{"match", tok.Token{Kind: tok.Match, Runes: []rune("match"), Source: tok.Source{
S: loc(1, 1, 0), E: loc(1, 6, 5),
}}},
{"fn", tok.Token{Kind: tok.Function, Runes: []rune("fn"), Source: tok.Source{
S: loc(1, 1, 0), E: loc(1, 3, 2),
}}},
{"type", tok.Token{Kind: tok.Type, Runes: []rune("type"), Source: tok.Source{
S: loc(1, 1, 0), E: loc(1, 5, 4),
}}},
{"enum", tok.Token{Kind: tok.Enum, Runes: []rune("enum"), Source: tok.Source{
S: loc(1, 1, 0), E: loc(1, 5, 4),
}}},
{":", tok.Token{Kind: tok.Colon, Runes: []rune(":"), Source: tok.Source{
S: loc(1, 1, 0), E: loc(1, 2, 1),
}}},
{",", tok.Token{Kind: tok.Comma, Runes: []rune(","), Source: tok.Source{
S: loc(1, 1, 0), E: loc(1, 2, 1),
}}},
{"<", tok.Token{Kind: tok.Lt, Runes: []rune("<"), Source: tok.Source{
S: loc(1, 1, 0), E: loc(1, 2, 1),
}}},
{">", tok.Token{Kind: tok.Gt, Runes: []rune(">"), Source: tok.Source{
S: loc(1, 1, 0), E: loc(1, 2, 1),
}}},
{"{", tok.Token{Kind: tok.Lbrace, Runes: []rune("{"), Source: tok.Source{
S: loc(1, 1, 0), E: loc(1, 2, 1),
}}},
{"}", tok.Token{Kind: tok.Rbrace, Runes: []rune("}"), Source: tok.Source{
S: loc(1, 1, 0), E: loc(1, 2, 1),
}}},
{"[[", tok.Token{Kind: tok.Ldeco, Runes: []rune("[["), Source: tok.Source{
S: loc(1, 1, 0), E: loc(1, 3, 2),
}}},
{"]]", tok.Token{Kind: tok.Rdeco, Runes: []rune("]]"), Source: tok.Source{
S: loc(1, 1, 0), E: loc(1, 3, 2),
}}},
{"(", tok.Token{Kind: tok.Lparen, Runes: []rune("("), Source: tok.Source{
S: loc(1, 1, 0), E: loc(1, 2, 1),
}}},
{")", tok.Token{Kind: tok.Rparen, Runes: []rune(")"), Source: tok.Source{
S: loc(1, 1, 0), E: loc(1, 2, 1),
}}},
{"|", tok.Token{Kind: tok.Or, Runes: []rune("|"), Source: tok.Source{
S: loc(1, 1, 0), E: loc(1, 2, 1),
}}},
{"->", tok.Token{Kind: tok.Arrow, Runes: []rune("->"), Source: tok.Source{
S: loc(1, 1, 0), E: loc(1, 3, 2),
}}},
{"x // y ", tok.Token{Kind: tok.Identifier, Runes: []rune("x"), Source: tok.Source{
S: loc(1, 1, 0), E: loc(1, 2, 1),
}}},
{`"abc"`, tok.Token{Kind: tok.String, Runes: []rune("abc"), Source: tok.Source{
S: loc(1, 2, 1), E: loc(1, 5, 4),
}}},
{`
//
ident
`, tok.Token{Kind: tok.Identifier, Runes: []rune("ident"), Source: tok.Source{
S: loc(3, 4, 10), E: loc(3, 9, 15),
}}},
} {
got, err := lexer.Lex([]rune(test.src), filepath)
name := fmt.Sprintf(`Lex("%v")`, test.src)
switch {
case err != nil:
t.Errorf("%v returned error: %v", name, err)
case len(got) != 1:
t.Errorf("%v returned %d tokens: %v", name, len(got), got)
case got[0].Kind != test.expect.Kind:
t.Errorf(`%v returned unexpected token kind: got "%+v", expected "%+v"`, name, got[0], test.expect)
case string(got[0].Runes) != string(test.expect.Runes):
t.Errorf(`%v returned unexpected token runes: got "%+v", expected "%+v"`, name, string(got[0].Runes), string(test.expect.Runes))
case got[0].Source != test.expect.Source:
t.Errorf(`%v returned unexpected token source: got %+v, expected %+v`, name, got[0].Source, test.expect.Source)
}
}
}
func TestErrors(t *testing.T) {
type test struct {
src string
expect string
}
for _, test := range []test{
{" \"abc", "test.txt:1:2 unterminated string"},
{" \"abc\n", "test.txt:1:2 unterminated string"},
{"*", "test.txt:1:1: unexpected '*'"},
} {
got, err := lexer.Lex([]rune(test.src), "test.txt")
if gotErr := err.Error(); test.expect != gotErr {
t.Errorf(`Lex() returned error "%+v", expected error "%+v"`, gotErr, test.expect)
}
if got != nil {
t.Errorf("Lex() returned non-nil for error")
}
}
}

173
tools/src/cmd/builtin-gen/main.go Normal file
View File

@@ -0,0 +1,173 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// builtin-gen parses the <tint>/src/tint/builtins.def file, then scans the
// project directory for '<file>.tmpl' files, to produce '<file>' source code
// files.
package main
import (
"flag"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/gen"
"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/parser"
"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/resolver"
"dawn.googlesource.com/tint/tools/src/fileutils"
"dawn.googlesource.com/tint/tools/src/glob"
)
const defProjectRelPath = "src/tint/builtins.def"
func main() {
if err := run(); err != nil {
fmt.Println(err)
os.Exit(1)
}
}
func showUsage() {
fmt.Println(`
builtin-gen generates the builtin table for the Tint compiler
builtin-gen parses the <tint>/src/tint/builtins.def file, then scans the project
directory for '<file>.tmpl' files, to produce '<file>' source code files.
usage:
builtin-gen
optional flags:`)
flag.PrintDefaults()
fmt.Println(``)
os.Exit(1)
}
func run() error {
// Load the builtins definition file
projectRoot := fileutils.ProjectRoot()
defPath := filepath.Join(projectRoot, defProjectRelPath)
defSource, err := ioutil.ReadFile(defPath)
if err != nil {
return err
}
// Parse the definition file to produce an AST
ast, err := parser.Parse(string(defSource), defProjectRelPath)
if err != nil {
return err
}
// Resolve the AST to produce the semantic info
sem, err := resolver.Resolve(ast)
if err != nil {
return err
}
// Recursively find all the template files in the <tint>/src directory
files, err := glob.Scan(projectRoot, glob.MustParseConfig(`{
"paths": [{"include": [
"src/**.tmpl",
"test/**.tmpl"
]}]
}`))
if err != nil {
return err
}
// For each template file...
for _, relTmplPath := range files {
// Make tmplPath absolute
tmplPath := filepath.Join(projectRoot, relTmplPath)
// Read the template file
tmpl, err := ioutil.ReadFile(tmplPath)
if err != nil {
return fmt.Errorf("failed to open '%v': %w", tmplPath, err)
}
// Create or update the file at relpath if the file content has changed
// relpath is a path relative to the template
writeFile := func(relpath, body string) error {
// Write the common file header
sb := strings.Builder{}
sb.WriteString(fmt.Sprintf(header, filepath.ToSlash(relTmplPath), filepath.ToSlash(defProjectRelPath)))
sb.WriteString(body)
content := sb.String()
abspath := filepath.Join(filepath.Dir(tmplPath), relpath)
return writeFileIfChanged(abspath, content)
}
// Write the content generated using the template and semantic info
sb := strings.Builder{}
if err := gen.Generate(sem, string(tmpl), &sb, writeFile); err != nil {
return fmt.Errorf("while processing '%v': %w", tmplPath, err)
}
if body := sb.String(); body != "" {
_, tmplFileName := filepath.Split(tmplPath)
outFileName := strings.TrimSuffix(tmplFileName, ".tmpl")
if err := writeFile(outFileName, body); err != nil {
return err
}
}
}
return nil
}
// writes content to path if the file has changed
func writeFileIfChanged(path, content string) error {
existing, err := ioutil.ReadFile(path)
if err == nil && string(existing) == content {
return nil // Not changed
}
if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
return fmt.Errorf("failed to create directory for '%v': %w", path, err)
}
if err := ioutil.WriteFile(path, []byte(content), 0666); err != nil {
return fmt.Errorf("failed to write file '%v': %w", path, err)
}
return nil
}
const header = `// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
////////////////////////////////////////////////////////////////////////////////
// File generated by tools/builtin-gen
// using the template:
// %v
// and the builtin defintion file:
// %v
//
// Do not modify this file directly
////////////////////////////////////////////////////////////////////////////////
`

312
tools/src/cmd/builtin-gen/parser/parser.go Normal file
View File

@@ -0,0 +1,312 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package parser provides a basic parser for the Tint builtin definition
// language
package parser
import (
"fmt"
"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/ast"
"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/lexer"
"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/tok"
)
// Parse produces a list of tokens for the given source code
func Parse(source, filepath string) (*ast.AST, error) {
runes := []rune(source)
tokens, err := lexer.Lex(runes, filepath)
if err != nil {
return nil, err
}
p := parser{tokens: tokens}
return p.parse()
}
type parser struct {
tokens []tok.Token
err error
}
func (p *parser) parse() (*ast.AST, error) {
out := ast.AST{}
var decorations ast.Decorations
for p.err == nil {
t := p.peek(0)
if t == nil {
break
}
switch t.Kind {
case tok.Ldeco:
decorations = append(decorations, p.decorations()...)
case tok.Enum:
if len(decorations) > 0 {
p.err = fmt.Errorf("%v unexpected decoration", decorations[0].Source)
}
out.Enums = append(out.Enums, p.enumDecl())
case tok.Match:
if len(decorations) > 0 {
p.err = fmt.Errorf("%v unexpected decoration", decorations[0].Source)
}
out.Matchers = append(out.Matchers, p.matcherDecl())
case tok.Type:
out.Types = append(out.Types, p.typeDecl(decorations))
decorations = nil
case tok.Function:
out.Functions = append(out.Functions, p.functionDecl(decorations))
decorations = nil
default:
p.err = fmt.Errorf("%v unexpected token '%v'", t.Source, t.Kind)
}
if p.err != nil {
return nil, p.err
}
}
return &out, nil
}
func (p *parser) enumDecl() ast.EnumDecl {
p.expect(tok.Enum, "enum declaration")
name := p.expect(tok.Identifier, "enum name")
e := ast.EnumDecl{Source: name.Source, Name: string(name.Runes)}
p.expect(tok.Lbrace, "enum declaration")
for p.err == nil && p.match(tok.Rbrace) == nil {
e.Entries = append(e.Entries, p.enumEntry())
}
return e
}
func (p *parser) enumEntry() ast.EnumEntry {
decos := p.decorations()
name := p.expect(tok.Identifier, "enum entry")
return ast.EnumEntry{Source: name.Source, Decorations: decos, Name: string(name.Runes)}
}
func (p *parser) matcherDecl() ast.MatcherDecl {
p.expect(tok.Match, "matcher declaration")
name := p.expect(tok.Identifier, "matcher name")
m := ast.MatcherDecl{Source: name.Source, Name: string(name.Runes)}
p.expect(tok.Colon, "matcher declaration")
for p.err == nil {
m.Options = append(m.Options, p.templatedName())
if p.match(tok.Or) == nil {
break
}
}
return m
}
func (p *parser) typeDecl(decos ast.Decorations) ast.TypeDecl {
p.expect(tok.Type, "type declaration")
name := p.expect(tok.Identifier, "type name")
m := ast.TypeDecl{
Source: name.Source,
Decorations: decos,
Name: string(name.Runes),
}
if p.peekIs(0, tok.Lt) {
m.TemplateParams = p.templateParams()
}
return m
}
func (p *parser) decorations() ast.Decorations {
if p.match(tok.Ldeco) == nil {
return nil
}
out := ast.Decorations{}
for p.err == nil {
name := p.expect(tok.Identifier, "decoration name")
values := []string{}
if p.match(tok.Lparen) != nil {
for p.err == nil {
values = append(values, p.string())
if p.match(tok.Comma) == nil {
break
}
}
p.expect(tok.Rparen, "decoration values")
}
out = append(out, ast.Decoration{
Source: name.Source,
Name: string(name.Runes),
Values: values,
})
if !p.peekIs(0, tok.Comma) {
break
}
}
p.expect(tok.Rdeco, "decoration list")
return out
}
func (p *parser) functionDecl(decos ast.Decorations) ast.FunctionDecl {
p.expect(tok.Function, "function declaration")
name := p.expect(tok.Identifier, "function name")
f := ast.FunctionDecl{
Source: name.Source,
Decorations: decos,
Name: string(name.Runes),
}
if p.peekIs(0, tok.Lt) {
f.TemplateParams = p.templateParams()
}
f.Parameters = p.parameters()
if p.match(tok.Arrow) != nil {
ret := p.templatedName()
f.ReturnType = &ret
}
return f
}
func (p *parser) parameters() ast.Parameters {
l := ast.Parameters{}
p.expect(tok.Lparen, "function parameter list")
if p.match(tok.Rparen) == nil {
for p.err == nil {
l = append(l, p.parameter())
if p.match(tok.Comma) == nil {
break
}
}
p.expect(tok.Rparen, "function parameter list")
}
return l
}
func (p *parser) parameter() ast.Parameter {
if p.peekIs(1, tok.Colon) {
// name type
name := p.expect(tok.Identifier, "parameter name")
p.expect(tok.Colon, "parameter type")
return ast.Parameter{
Source: name.Source,
Name: string(name.Runes),
Type: p.templatedName(),
}
}
// type
ty := p.templatedName()
return ast.Parameter{
Source: ty.Source,
Type: ty,
}
}
func (p *parser) string() string {
s := p.expect(tok.String, "string")
return string(s.Runes)
}
func (p *parser) templatedName() ast.TemplatedName {
name := p.expect(tok.Identifier, "type name")
m := ast.TemplatedName{Source: name.Source, Name: string(name.Runes)}
if p.match(tok.Lt) != nil {
for p.err == nil {
m.TemplateArgs = append(m.TemplateArgs, p.templatedName())
if p.match(tok.Comma) == nil {
break
}
}
p.expect(tok.Gt, "template argument type list")
}
return m
}
func (p *parser) templateParams() ast.TemplateParams {
t := ast.TemplateParams{}
p.expect(tok.Lt, "template parameter list")
for p.err == nil && p.peekIs(0, tok.Identifier) {
t = append(t, p.templateParam())
}
p.expect(tok.Gt, "template parameter list")
return t
}
func (p *parser) templateParam() ast.TemplateParam {
name := p.match(tok.Identifier)
t := ast.TemplateParam{
Source: name.Source,
Name: string(name.Runes),
}
if p.match(tok.Colon) != nil {
t.Type = p.templatedName()
}
p.match(tok.Comma)
return t
}
func (p *parser) expect(kind tok.Kind, use string) tok.Token {
if p.err != nil {
return tok.Invalid
}
t := p.match(kind)
if t == nil {
if len(p.tokens) > 0 {
p.err = fmt.Errorf("%v expected '%v' for %v, got '%v'",
p.tokens[0].Source, kind, use, p.tokens[0].Kind)
} else {
p.err = fmt.Errorf("expected '%v' for %v, but reached end of file", kind, use)
}
return tok.Invalid
}
return *t
}
func (p *parser) ident(use string) string {
return string(p.expect(tok.Identifier, use).Runes)
}
// TODO(bclayton): Currently unused, but will be needed for integer bounds
// func (p *parser) integer(use string) int {
// t := p.expect(tok.Integer, use)
// if t.Kind != tok.Integer {
// return 0
// }
// i, err := strconv.Atoi(string(t.Runes))
// if err != nil {
// p.err = err
// return 0
// }
// return i
// }
func (p *parser) match(kind tok.Kind) *tok.Token {
if p.err != nil || len(p.tokens) == 0 {
return nil
}
t := p.tokens[0]
if t.Kind != kind {
return nil
}
p.tokens = p.tokens[1:]
return &t
}
func (p *parser) peekIs(i int, kind tok.Kind) bool {
t := p.peek(i)
if t == nil {
return false
}
return t.Kind == kind
}
func (p *parser) peek(i int) *tok.Token {
if len(p.tokens) <= i {
return nil
}
return &p.tokens[i]
}

210
tools/src/cmd/builtin-gen/parser/parser_test.go Normal file
View File

@@ -0,0 +1,210 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package parser_test
import (
"testing"
"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/ast"
"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/parser"
)
func TestParser(t *testing.T) {
type test struct {
src string
expect ast.AST
}
for _, test := range []test{
{"enum E {}", ast.AST{
Enums: []ast.EnumDecl{{Name: "E"}},
}},
{"enum E { A [[deco]] B C }", ast.AST{
Enums: []ast.EnumDecl{{
Name: "E",
Entries: []ast.EnumEntry{
{Name: "A"},
{
Decorations: ast.Decorations{{Name: "deco"}},
Name: "B",
},
{Name: "C"},
},
}},
}},
{"type T", ast.AST{
Types: []ast.TypeDecl{{Name: "T"}},
}},
{"type T<A, B, C>", ast.AST{
Types: []ast.TypeDecl{{
Name: "T",
TemplateParams: ast.TemplateParams{
{Name: "A"},
{Name: "B"},
{Name: "C"},
},
}},
}},
{"[[deco]] type T", ast.AST{
Types: []ast.TypeDecl{{
Decorations: ast.Decorations{
{Name: "deco"},
},
Name: "T",
}},
}},
{`[[deco("a", "b")]] type T`, ast.AST{
Types: []ast.TypeDecl{{
Decorations: ast.Decorations{
{Name: "deco", Values: []string{"a", "b"}},
},
Name: "T",
}},
}},
{"match M : A", ast.AST{
Matchers: []ast.MatcherDecl{{
Name: "M",
Options: ast.MatcherOptions{
ast.TemplatedName{Name: "A"},
},
}},
}},
{"match M : A | B", ast.AST{
Matchers: []ast.MatcherDecl{{
Name: "M",
Options: ast.MatcherOptions{
ast.TemplatedName{Name: "A"},
ast.TemplatedName{Name: "B"},
},
}},
}},
{"fn F()", ast.AST{
Functions: []ast.FunctionDecl{{
Name: "F",
}},
}},
{"[[deco]] fn F()", ast.AST{
Functions: []ast.FunctionDecl{{
Name: "F",
Decorations: ast.Decorations{
{Name: "deco"},
},
}},
}},
{"fn F(a)", ast.AST{
Functions: []ast.FunctionDecl{{
Name: "F",
Parameters: ast.Parameters{
{Type: ast.TemplatedName{Name: "a"}},
},
}},
}},
{"fn F(a: T)", ast.AST{
Functions: []ast.FunctionDecl{{
Name: "F",
Parameters: ast.Parameters{
{Name: "a", Type: ast.TemplatedName{Name: "T"}},
},
}},
}},
{"fn F(a, b)", ast.AST{
Functions: []ast.FunctionDecl{{
Name: "F",
Parameters: ast.Parameters{
{Type: ast.TemplatedName{Name: "a"}},
{Type: ast.TemplatedName{Name: "b"}},
},
}},
}},
{"fn F<A : B<C>>()", ast.AST{
Functions: []ast.FunctionDecl{{
Name: "F",
TemplateParams: ast.TemplateParams{
{
Name: "A", Type: ast.TemplatedName{
Name: "B",
TemplateArgs: ast.TemplatedNames{
{Name: "C"},
},
},
},
},
}},
}},
{"fn F<T>(a: X, b: Y<T>)", ast.AST{
Functions: []ast.FunctionDecl{{
Name: "F",
TemplateParams: ast.TemplateParams{
{Name: "T"},
},
Parameters: ast.Parameters{
{Name: "a", Type: ast.TemplatedName{Name: "X"}},
{Name: "b", Type: ast.TemplatedName{
Name: "Y",
TemplateArgs: []ast.TemplatedName{{Name: "T"}},
}},
},
}},
}},
{"fn F() -> X", ast.AST{
Functions: []ast.FunctionDecl{{
Name: "F",
ReturnType: &ast.TemplatedName{Name: "X"},
}},
}},
{"fn F() -> X<T>", ast.AST{
Functions: []ast.FunctionDecl{{
Name: "F",
ReturnType: &ast.TemplatedName{
Name: "X",
TemplateArgs: []ast.TemplatedName{{Name: "T"}},
},
}},
}},
} {
got, err := parser.Parse(test.src, "file.txt")
if err != nil {
t.Errorf("While parsing:\n%s\nParse() returned error: %v", test.src, err)
continue
}
gotStr, expectStr := got.String(), test.expect.String()
if gotStr != expectStr {
t.Errorf("While parsing:\n%s\nGot:\n%s\nExpected:\n%s", test.src, gotStr, expectStr)
}
}
}
func TestErrors(t *testing.T) {
type test struct {
src string
expect string
}
for _, test := range []test{
{"+", "test.txt:1:1: unexpected '+'"},
{"123", "test.txt:1:1 unexpected token 'integer'"},
{"[[123]]", "test.txt:1:3 expected 'ident' for decoration name, got 'integer'"},
{"[[abc", "expected ']]' for decoration list, but reached end of file"},
} {
got, err := parser.Parse(test.src, "test.txt")
if gotErr := err.Error(); test.expect != gotErr {
t.Errorf(`Parse() returned error "%+v", expected error "%+v"`, gotErr, test.expect)
}
if got != nil {
t.Errorf("Lex() returned non-nil for error")
}
}
}

653
tools/src/cmd/builtin-gen/resolver/resolve.go Normal file
View File

@@ -0,0 +1,653 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package resolver
import (
"fmt"
"sort"
"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/ast"
"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/sem"
"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/tok"
)
type resolver struct {
a *ast.AST
s *sem.Sem
globals scope
functions map[string]*sem.Function
enumEntryMatchers map[*sem.EnumEntry]*sem.EnumMatcher
}
// Resolve processes the AST
func Resolve(a *ast.AST) (*sem.Sem, error) {
r := resolver{
a: a,
s: sem.New(),
globals: newScope(nil),
functions: map[string]*sem.Function{},
enumEntryMatchers: map[*sem.EnumEntry]*sem.EnumMatcher{},
}
// Declare and resolve all the enumerators
for _, e := range a.Enums {
if err := r.enum(e); err != nil {
return nil, err
}
}
// Declare and resolve all the ty types
for _, p := range a.Types {
if err := r.ty(p); err != nil {
return nil, err
}
}
// Declare and resolve the type matchers
for _, m := range a.Matchers {
if err := r.matcher(m); err != nil {
return nil, err
}
}
// Declare and resolve the functions
for _, f := range a.Functions {
if err := r.function(f); err != nil {
return nil, err
}
}
// Calculate the unique parameter names
r.s.UniqueParameterNames = r.calculateUniqueParameterNames()
return r.s, nil
}
// enum() resolves an enum declaration.
// The resulting sem.Enum is appended to Sem.Enums, and the enum and all its
// entries are registered with the global scope.
func (r *resolver) enum(e ast.EnumDecl) error {
s := &sem.Enum{
Decl: e,
Name: e.Name,
}
// Register the enum
r.s.Enums = append(r.s.Enums, s)
if err := r.globals.declare(s, e.Source); err != nil {
return err
}
// Register each of the enum entries
for _, ast := range e.Entries {
entry := &sem.EnumEntry{
Name: ast.Name,
Enum: s,
}
if internal := ast.Decorations.Take("internal"); internal != nil {
entry.IsInternal = true
if len(internal.Values) != 0 {
return fmt.Errorf("%v unexpected value for internal decoration", ast.Source)
}
}
if len(ast.Decorations) != 0 {
return fmt.Errorf("%v unknown decoration", ast.Decorations[0].Source)
}
if err := r.globals.declare(entry, e.Source); err != nil {
return err
}
s.Entries = append(s.Entries, entry)
}
return nil
}
// ty() resolves a type declaration.
// The resulting sem.Type is appended to Sem.Types, and the type is registered
// with the global scope.
func (r *resolver) ty(a ast.TypeDecl) error {
t := &sem.Type{
Decl: a,
Name: a.Name,
}
// Register the type
r.s.Types = append(r.s.Types, t)
if err := r.globals.declare(t, a.Source); err != nil {
return err
}
// Create a new scope for resolving template parameters
s := newScope(&r.globals)
// Resolve the type template parameters
templateParams, err := r.templateParams(&s, a.TemplateParams)
if err != nil {
return err
}
t.TemplateParams = templateParams
// Scan for decorations
if d := a.Decorations.Take("display"); d != nil {
if len(d.Values) != 1 {
return fmt.Errorf("%v expected a single value for 'display' decoration", d.Source)
}
t.DisplayName = d.Values[0]
}
if len(a.Decorations) != 0 {
return fmt.Errorf("%v unknown decoration", a.Decorations[0].Source)
}
return nil
}
// matcher() resolves a match declaration to either a sem.TypeMatcher or
// sem.EnumMatcher.
// The resulting matcher is appended to either Sem.TypeMatchers or
// Sem.EnumMatchers, and is registered with the global scope.
func (r *resolver) matcher(a ast.MatcherDecl) error {
// Determine whether this is a type matcher or enum matcher by resolving the
// first option
firstOption, err := r.lookupNamed(&r.globals, a.Options[0])
if err != nil {
return err
}
// Resolve to a sem.TypeMatcher or a sem.EnumMatcher
switch firstOption := firstOption.(type) {
case *sem.Type:
options := map[sem.Named]tok.Source{}
m := &sem.TypeMatcher{
Decl: a,
Name: a.Name,
}
// Register the matcher
r.s.TypeMatchers = append(r.s.TypeMatchers, m)
if err := r.globals.declare(m, a.Source); err != nil {
return err
}
// Resolve each of the types in the options list
for _, ast := range m.Decl.Options {
ty, err := r.lookupType(&r.globals, ast)
if err != nil {
return err
}
m.Types = append(m.Types, ty)
if s, dup := options[ty]; dup {
return fmt.Errorf("%v duplicate option '%v' in matcher\nFirst declared here: %v", ast.Source, ast.Name, s)
}
options[ty] = ast.Source
}
return nil
case *sem.EnumEntry:
enum := firstOption.Enum
m := &sem.EnumMatcher{
Decl: a,
Name: a.Name,
Enum: enum,
}
// Register the matcher
r.s.EnumMatchers = append(r.s.EnumMatchers, m)
if err := r.globals.declare(m, a.Source); err != nil {
return err
}
// Resolve each of the enums in the options list
for _, ast := range m.Decl.Options {
entry := enum.FindEntry(ast.Name)
if entry == nil {
return fmt.Errorf("%v enum '%v' does not contain '%v'", ast.Source, enum.Name, ast.Name)
}
m.Options = append(m.Options, entry)
}
return nil
}
return fmt.Errorf("'%v' cannot be used for matcher", a.Name)
}
// function() resolves a function overload declaration.
// The the first overload for the function creates and appends the sem.Function
// to Sem.Functions. Subsequent overloads append their resolved overload to the
// sem.Function.Overloads list.
func (r *resolver) function(a ast.FunctionDecl) error {
// If this is the first overload of the function, create and register the
// semantic function.
f := r.functions[a.Name]
if f == nil {
f = &sem.Function{Name: a.Name}
r.functions[a.Name] = f
r.s.Functions = append(r.s.Functions, f)
}
// Create a new scope for resolving template parameters
s := newScope(&r.globals)
// Resolve the declared template parameters
templateParams, err := r.templateParams(&s, a.TemplateParams)
if err != nil {
return err
}
// Construct the semantic overload
overload := &sem.Overload{
Decl: a,
Function: f,
Parameters: make([]sem.Parameter, len(a.Parameters)),
TemplateParams: templateParams,
}
// Process overload decorations
if stageDeco := a.Decorations.Take("stage"); stageDeco != nil {
for stageDeco != nil {
for _, stage := range stageDeco.Values {
switch stage {
case "vertex":
overload.CanBeUsedInStage.Vertex = true
case "fragment":
overload.CanBeUsedInStage.Fragment = true
case "compute":
overload.CanBeUsedInStage.Compute = true
default:
return fmt.Errorf("%v unknown stage '%v'", stageDeco.Source, stage)
}
}
stageDeco = a.Decorations.Take("stage")
}
} else {
overload.CanBeUsedInStage = sem.StageUses{
Vertex: true,
Fragment: true,
Compute: true,
}
}
if deprecated := a.Decorations.Take("deprecated"); deprecated != nil {
overload.IsDeprecated = true
if len(deprecated.Values) != 0 {
return fmt.Errorf("%v unexpected value for deprecated decoration", deprecated.Source)
}
}
if len(a.Decorations) != 0 {
return fmt.Errorf("%v unknown decoration", a.Decorations[0].Source)
}
// Append the overload to the function
f.Overloads = append(f.Overloads, overload)
// Sort the template parameters by resolved type. Append these to
// sem.Overload.OpenTypes or sem.Overload.OpenNumbers based on their kind.
for _, param := range templateParams {
switch param := param.(type) {
case *sem.TemplateTypeParam:
overload.OpenTypes = append(overload.OpenTypes, param)
case *sem.TemplateEnumParam, *sem.TemplateNumberParam:
overload.OpenNumbers = append(overload.OpenNumbers, param)
}
}
// Update high-water marks of open types / numbers
if r.s.MaxOpenTypes < len(overload.OpenTypes) {
r.s.MaxOpenTypes = len(overload.OpenTypes)
}
if r.s.MaxOpenNumbers < len(overload.OpenNumbers) {
r.s.MaxOpenNumbers = len(overload.OpenNumbers)
}
// Resolve the parameters
for i, p := range a.Parameters {
usage, err := r.fullyQualifiedName(&s, p.Type)
if err != nil {
return err
}
overload.Parameters[i] = sem.Parameter{
Name: p.Name,
Type: usage,
}
}
// Resolve the return type
if a.ReturnType != nil {
usage, err := r.fullyQualifiedName(&s, *a.ReturnType)
if err != nil {
return err
}
switch usage.Target.(type) {
case *sem.Type, *sem.TemplateTypeParam:
overload.ReturnType = &usage
default:
return fmt.Errorf("%v cannot use '%v' as return type. Must be a type or template type", a.ReturnType.Source, a.ReturnType.Name)
}
}
return nil
}
// fullyQualifiedName() resolves the ast.TemplatedName to a sem.FullyQualifiedName.
func (r *resolver) fullyQualifiedName(s *scope, arg ast.TemplatedName) (sem.FullyQualifiedName, error) {
target, err := r.lookupNamed(s, arg)
if err != nil {
return sem.FullyQualifiedName{}, err
}
if entry, ok := target.(*sem.EnumEntry); ok {
// The target resolved to an enum entry.
// Automagically transform this into a synthetic matcher with a single
// option. i.e.
// This:
// enum E{ a b c }
// fn F(b)
// Becomes:
// enum E{ a b c }
// matcher b
// fn F(b)
// We don't really care right now that we have a symbol collision
// between E.b and b, as the generators return different names for
// these.
matcher, ok := r.enumEntryMatchers[entry]
if !ok {
matcher = &sem.EnumMatcher{
Name: entry.Name,
Enum: entry.Enum,
Options: []*sem.EnumEntry{entry},
}
r.enumEntryMatchers[entry] = matcher
r.s.EnumMatchers = append(r.s.EnumMatchers, matcher)
}
target = matcher
}
fqn := sem.FullyQualifiedName{
Target: target,
TemplateArguments: make([]interface{}, len(arg.TemplateArgs)),
}
for i, a := range arg.TemplateArgs {
arg, err := r.fullyQualifiedName(s, a)
if err != nil {
return sem.FullyQualifiedName{}, err
}
fqn.TemplateArguments[i] = arg
}
return fqn, nil
}
// templateParams() resolves the ast.TemplateParams into list of sem.TemplateParam.
// Each sem.TemplateParam is registered with the scope s.
func (r *resolver) templateParams(s *scope, l ast.TemplateParams) ([]sem.TemplateParam, error) {
out := []sem.TemplateParam{}
for _, ast := range l {
param, err := r.templateParam(ast)
if err != nil {
return nil, err
}
s.declare(param, ast.Source)
out = append(out, param)
}
return out, nil
}
// templateParams() resolves the ast.TemplateParam into sem.TemplateParam, which
// is either a sem.TemplateEnumParam or a sem.TemplateTypeParam.
func (r *resolver) templateParam(a ast.TemplateParam) (sem.TemplateParam, error) {
if a.Type.Name == "num" {
return &sem.TemplateNumberParam{Name: a.Name}, nil
}
if a.Type.Name != "" {
resolved, err := r.lookupNamed(&r.globals, a.Type)
if err != nil {
return nil, err
}
switch r := resolved.(type) {
case *sem.Enum:
return &sem.TemplateEnumParam{Name: a.Name, Enum: r}, nil
case *sem.EnumMatcher:
return &sem.TemplateEnumParam{Name: a.Name, Enum: r.Enum, Matcher: r}, nil
case *sem.TypeMatcher:
return &sem.TemplateTypeParam{Name: a.Name, Type: r}, nil
default:
return nil, fmt.Errorf("%v invalid template parameter type '%v'", a.Source, a.Type.Name)
}
}
return &sem.TemplateTypeParam{Name: a.Name}, nil
}
// lookupType() searches the scope `s` and its ancestors for the sem.Type with
// the given name.
func (r *resolver) lookupType(s *scope, a ast.TemplatedName) (*sem.Type, error) {
resolved, err := r.lookupNamed(s, a)
if err != nil {
return nil, err
}
// Something with the given name was found...
if ty, ok := resolved.(*sem.Type); ok {
return ty, nil
}
// ... but that something was not a sem.Type
return nil, fmt.Errorf("%v '%v' resolves to %v but type is expected", a.Source, a.Name, describe(resolved))
}
// lookupNamed() searches `s` and its ancestors for the sem.Named object with
// the given name. If there are template arguments for the name `a`, then
// lookupNamed() performs basic validation that those arguments can be passed
// to the named object.
func (r *resolver) lookupNamed(s *scope, a ast.TemplatedName) (sem.Named, error) {
target := s.lookup(a.Name)
if target == nil {
return nil, fmt.Errorf("%v cannot resolve '%v'", a.Source, a.Name)
}
// Something with the given name was found...
var params []sem.TemplateParam
var ty sem.ResolvableType
switch target := target.object.(type) {
case *sem.Type:
ty = target
params = target.TemplateParams
case *sem.TypeMatcher:
ty = target
params = target.TemplateParams
case sem.TemplateParam:
if len(a.TemplateArgs) != 0 {
return nil, fmt.Errorf("%v '%v' template parameters do not accept template arguments", a.Source, a.Name)
}
return target.(sem.Named), nil
case sem.Named:
return target, nil
default:
panic(fmt.Errorf("Unknown resolved type %T", target))
}
// ... and that something takes template parameters
// Check the number of templated name template arguments match the number of
// templated parameters for the target.
args := a.TemplateArgs
if len(params) != len(args) {
return nil, fmt.Errorf("%v '%v' requires %d template arguments, but %d were provided", a.Source, a.Name, len(params), len(args))
}
// Check templated name template argument kinds match the parameter kinds
for i, ast := range args {
param := params[i]
arg, err := r.lookupNamed(s, args[i])
if err != nil {
return nil, err
}
if err := checkCompatible(arg, param); err != nil {
return nil, fmt.Errorf("%v %w", ast.Source, err)
}
}
return ty, nil
}
// calculateUniqueParameterNames() iterates over all the parameters of all
// overloads, calculating the list of unique parameter names
func (r *resolver) calculateUniqueParameterNames() []string {
set := map[string]struct{}{"": {}}
names := []string{}
for _, f := range r.s.Functions {
for _, o := range f.Overloads {
for _, p := range o.Parameters {
if _, dup := set[p.Name]; !dup {
set[p.Name] = struct{}{}
names = append(names, p.Name)
}
}
}
}
sort.Strings(names)
return names
}
// describe() returns a string describing a sem.Named
func describe(n sem.Named) string {
switch n := n.(type) {
case *sem.Type:
return "type '" + n.Name + "'"
case *sem.TypeMatcher:
return "type matcher '" + n.Name + "'"
case *sem.Enum:
return "enum '" + n.Name + "'"
case *sem.EnumMatcher:
return "enum matcher '" + n.Name + "'"
case *sem.TemplateTypeParam:
return "template type"
case *sem.TemplateEnumParam:
return "template enum '" + n.Enum.Name + "'"
case *sem.EnumEntry:
return "enum entry '" + n.Enum.Name + "." + n.Name + "'"
case *sem.TemplateNumberParam:
return "template number"
default:
panic(fmt.Errorf("unhandled type %T", n))
}
}
// checkCompatible() returns an error if `arg` cannot be used as an argument for
// a parameter of `param`.
func checkCompatible(arg, param sem.Named) error {
// asEnum() returns the underlying sem.Enum if n is a enum matcher,
// templated enum parameter or an enum entry, otherwise nil
asEnum := func(n sem.Named) *sem.Enum {
switch n := n.(type) {
case *sem.EnumMatcher:
return n.Enum
case *sem.TemplateEnumParam:
return n.Enum
case *sem.EnumEntry:
return n.Enum
default:
return nil
}
}
if arg := asEnum(arg); arg != nil {
param := asEnum(param)
if arg == param {
return nil
}
}
anyNumber := "any number"
// asNumber() returns anyNumber if n is a TemplateNumberParam.
// TODO(bclayton): Once we support number ranges [e.g.: fn F<N: 1..4>()], we
// should check number ranges are compatible
asNumber := func(n sem.Named) interface{} {
switch n.(type) {
case *sem.TemplateNumberParam:
return anyNumber
default:
return nil
}
}
if arg := asNumber(arg); arg != nil {
param := asNumber(param)
if arg == param {
return nil
}
}
anyType := &sem.Type{}
// asNumber() returns the sem.Type, sem.TypeMatcher if the named object
// resolves to one of these, or anyType if n is a unconstrained template
// type parameter.
asResolvableType := func(n sem.Named) sem.ResolvableType {
switch n := n.(type) {
case *sem.TemplateTypeParam:
if n.Type != nil {
return n.Type
}
return anyType
case *sem.Type:
return n
case *sem.TypeMatcher:
return n
default:
return nil
}
}
if arg := asResolvableType(arg); arg != nil {
param := asResolvableType(param)
if arg == param || param == anyType {
return nil
}
}
return fmt.Errorf("cannot use %v as %v", describe(arg), describe(param))
}
// scope is a basic hierarchical name to object table
type scope struct {
objects map[string]objectAndSource
parent *scope
}
// objectAndSource is a sem.Named object with a source
type objectAndSource struct {
object sem.Named
source tok.Source
}
// newScope returns a newly initalized scope
func newScope(parent *scope) scope {
return scope{objects: map[string]objectAndSource{}, parent: parent}
}
// lookup() searches the scope and then its parents for the symbol with the
// given name.
func (s *scope) lookup(name string) *objectAndSource {
if o, found := s.objects[name]; found {
return &o
}
if s.parent == nil {
return nil
}
return s.parent.lookup(name)
}
// declare() declares the symbol with the given name, erroring on symbol
// collision.
func (s *scope) declare(object sem.Named, source tok.Source) error {
name := object.GetName()
if existing := s.lookup(name); existing != nil {
return fmt.Errorf("%v '%v' already declared\nFirst declared here: %v", source, name, existing.source)
}
s.objects[name] = objectAndSource{object, source}
return nil
}

330
tools/src/cmd/builtin-gen/resolver/resolver_test.go Normal file
View File

@@ -0,0 +1,330 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package resolver_test
import (
"fmt"
"strings"
"testing"
"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/parser"
"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/resolver"
)
func TestResolver(t *testing.T) {
type test struct {
src string
err string
}
success := ""
for _, test := range []test{
{
`type X`,
success,
}, {
`enum E {}`,
success,
}, {
`enum E {A B C}`,
success,
}, {
`type X`,
success,
}, {
`[[display("Y")]] type X`,
success,
}, {
`
type x
match y: x`,
success,
}, {
`
enum e {a b c}
match y: c | a | b`,
success,
}, {
`fn f()`,
success,
}, {
`fn f<T>()`,
success,
}, {
`
type f32
fn f<N: num>()`,
success,
}, {
`
enum e { a b c }
fn f<N: e>()`,
success,
}, {
`
type f32
fn f<T>(T) -> f32`,
success,
}, {
`
type f32
type P<T>
match m: f32
fn f<T: m>(P<T>) -> T`,
success,
}, {
`
type f32
type P<T>
match m: f32
fn f(P<m>)`,
success,
}, {
`
enum e { a }
fn f(a)`,
success,
}, {
`
enum e { a b }
type T<E: e>
match m: a
fn f<E: m>(T<E>)`,
success,
}, {
`
enum e { a b }
type T<E: e>
match m: a
fn f(T<m>)`,
success,
}, {
`
enum e { a }
type T<E: e>
fn f(T<a>)`,
success,
}, {
`
type T<E: num>
fn f<E: num>(T<E>)`,
success,
}, {
`fn f<T>(T)`,
success,
}, {
`
enum e { a b }
fn f<E: e>()`,
success,
}, {
`
enum e { a b }
match m: a | b
fn f<E: m>()`,
success,
}, {
`
type f32
type T<x>
fn f(T<T<f32>>)`,
success,
}, {
`enum E {A A}`,
`
file.txt:1:6 'A' already declared
First declared here: file.txt:1:6
`,
},
{
`type X type X`,
`
file.txt:1:13 'X' already declared
First declared here: file.txt:1:6`,
}, {
`[[meow]] type X`,
`
file.txt:1:3 unknown decoration
`,
}, {
`[[display("Y", "Z")]] type X`,
`
file.txt:1:3 expected a single value for 'display' decoration`,
}, {
`
enum e { a }
enum e { b }`,
`
file.txt:2:6 'e' already declared
First declared here: file.txt:1:6`,
}, {
`
type X
match X : X`,
`
file.txt:2:7 'X' already declared
First declared here: file.txt:1:6`,
}, {
`type T<X>
match M : T`,
`file.txt:2:11 'T' requires 1 template arguments, but 0 were provided`,
}, {
`
match x: y`,
`
file.txt:1:10 cannot resolve 'y'
`,
}, {
`
type a
match x: a | b`,
`
file.txt:2:14 cannot resolve 'b'
`,
}, {
`
type a
enum e { b }
match x: a | b`,
`
file.txt:3:14 'b' resolves to enum entry 'e.b' but type is expected
`,
}, {
`
type a
type b
match x: a | b | a`,
`
file.txt:3:18 duplicate option 'a' in matcher
First declared here: file.txt:3:10
`,
}, {
`
enum e { a c }
match x: a | b | c`,
`
file.txt:2:14 enum 'e' does not contain 'b'
`,
}, {
`
enum e { a }
match x: a
match x: a`,
`
file.txt:3:7 'x' already declared
First declared here: file.txt:2:7
`,
}, {
`
type t
match x: t
match y: x`,
`
'y' cannot be used for matcher
`,
}, {
`fn f(u)`,
`file.txt:1:6 cannot resolve 'u'`,
}, {
`fn f() -> u`,
`file.txt:1:11 cannot resolve 'u'`,
}, {
`fn f<T: u>()`,
`file.txt:1:9 cannot resolve 'u'`,
}, {
`
enum e { a }
fn f() -> e`,
`file.txt:2:11 cannot use 'e' as return type. Must be a type or template type`,
}, {
`
type T<x>
fn f(T<u>)`,
`file.txt:2:8 cannot resolve 'u'`,
}, {
`
type x
fn f<T>(T<x>)`,
`file.txt:2:9 'T' template parameters do not accept template arguments`,
}, {
`
type A<N: num>
type B
fn f(A<B>)`,
`file.txt:3:8 cannot use type 'B' as template number`,
}, {
`
type A<N>
enum E { b }
fn f(A<b>)`,
`file.txt:3:8 cannot use enum entry 'E.b' as template type`,
}, {
`
type T
type P<N: num>
match m: T
fn f(P<m>)`,
`file.txt:4:8 cannot use type matcher 'm' as template number`,
}, {
`
type P<N: num>
enum E { b }
fn f(P<E>)`,
`file.txt:3:8 cannot use enum 'E' as template number`,
}, {
`
type P<N: num>
enum E { a b }
match m: a | b
fn f(P<m>)`,
`file.txt:4:8 cannot use enum matcher 'm' as template number`,
}, {
`
type P<N: num>
enum E { a b }
match m: a | b
fn f<M: m>(P<M>)`,
`file.txt:4:14 cannot use template enum 'E' as template number`,
}, {
`
enum E { a }
type T<X: a>`,
`file.txt:2:8 invalid template parameter type 'a'`,
}, {
`
enum E { a }
fn f<M: a>()`,
`file.txt:2:6 invalid template parameter type 'a'`,
},
} {
ast, err := parser.Parse(strings.TrimSpace(string(test.src)), "file.txt")
if err != nil {
t.Errorf("Unexpected parser error: %v", err)
continue
}
expectErr := strings.TrimSpace(test.err)
_, err = resolver.Resolve(ast)
if err != nil {
gotErr := strings.TrimSpace(fmt.Sprint(err))
if gotErr != expectErr {
t.Errorf("While parsing:\n%s\nGot error:\n%s\nExpected:\n%s", test.src, gotErr, expectErr)
}
} else if expectErr != success {
t.Errorf("While parsing:\n%s\nGot no error, expected error:\n%s", test.src, expectErr)
}
}
}

283
tools/src/cmd/builtin-gen/sem/sem.go Normal file
View File

@@ -0,0 +1,283 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sem
import (
"fmt"
"dawn.googlesource.com/tint/tools/src/cmd/builtin-gen/ast"
)
// Sem is the root of the semantic tree
type Sem struct {
Enums []*Enum
Types []*Type
TypeMatchers []*TypeMatcher
EnumMatchers []*EnumMatcher
Functions []*Function
// Maximum number of open-types used across all builtins
MaxOpenTypes int
// Maximum number of open-numbers used across all builtins
MaxOpenNumbers int
// The alphabetically sorted list of unique parameter names
UniqueParameterNames []string
}
// New returns a new Sem
func New() *Sem {
return &Sem{
Enums: []*Enum{},
Types: []*Type{},
TypeMatchers: []*TypeMatcher{},
EnumMatchers: []*EnumMatcher{},
Functions: []*Function{},
}
}
// Enum describes an enumerator
type Enum struct {
Decl ast.EnumDecl
Name string
Entries []*EnumEntry
}
// FindEntry returns the enum entry with the given name
func (e *Enum) FindEntry(name string) *EnumEntry {
for _, entry := range e.Entries {
if entry.Name == name {
return entry
}
}
return nil
}
// EnumEntry is an entry in an enumerator
type EnumEntry struct {
Enum *Enum
Name string
IsInternal bool // True if this entry is not part of the WGSL grammar
}
// Format implements the fmt.Formatter interface
func (e EnumEntry) Format(w fmt.State, verb rune) {
if e.IsInternal {
fmt.Fprint(w, "[[internal]] ")
}
fmt.Fprint(w, e.Name)
}
// Type declares a type
type Type struct {
TemplateParams []TemplateParam
Decl ast.TypeDecl
Name string
DisplayName string
}
// TypeMatcher declares a type matcher
type TypeMatcher struct {
TemplateParams []TemplateParam
Decl ast.MatcherDecl
Name string
Types []*Type
}
// EnumMatcher declares a enum matcher
type EnumMatcher struct {
TemplateParams []TemplateParam
Decl ast.MatcherDecl
Name string
Enum *Enum
Options []*EnumEntry
}
// TemplateEnumParam is a template enum parameter
type TemplateEnumParam struct {
Name string
Enum *Enum
Matcher *EnumMatcher // Optional
}
// TemplateTypeParam is a template type parameter
type TemplateTypeParam struct {
Name string
Type ResolvableType
}
// TemplateNumberParam is a template type parameter
type TemplateNumberParam struct {
Name string
}
// Function describes the overloads of a builtin function
type Function struct {
Name string
Overloads []*Overload
}
// Overload describes a single overload of a function
type Overload struct {
Decl ast.FunctionDecl
Function *Function
TemplateParams []TemplateParam
OpenTypes []*TemplateTypeParam
OpenNumbers []TemplateParam
ReturnType *FullyQualifiedName
Parameters []Parameter
CanBeUsedInStage StageUses
IsDeprecated bool // True if this overload is deprecated
}
// StageUses describes the stages an overload can be used in
type StageUses struct {
Vertex bool
Fragment bool
Compute bool
}
// List returns the stage uses as a string list
func (u StageUses) List() []string {
out := []string{}
if u.Vertex {
out = append(out, "vertex")
}
if u.Fragment {
out = append(out, "fragment")
}
if u.Compute {
out = append(out, "compute")
}
return out
}
// Format implements the fmt.Formatter interface
func (o Overload) Format(w fmt.State, verb rune) {
fmt.Fprintf(w, "fn %v", o.Function.Name)
if len(o.TemplateParams) > 0 {
fmt.Fprintf(w, "<")
for i, t := range o.TemplateParams {
if i > 0 {
fmt.Fprint(w, ", ")
}
fmt.Fprintf(w, "%v", t)
}
fmt.Fprintf(w, ">")
}
fmt.Fprint(w, "(")
for i, p := range o.Parameters {
if i > 0 {
fmt.Fprint(w, ", ")
}
fmt.Fprintf(w, "%v", p)
}
fmt.Fprint(w, ")")
if o.ReturnType != nil {
fmt.Fprintf(w, " -> %v", o.ReturnType)
}
}
// Parameter describes a single parameter of a function overload
type Parameter struct {
Name string
Type FullyQualifiedName
}
// Format implements the fmt.Formatter interface
func (p Parameter) Format(w fmt.State, verb rune) {
if p.Name != "" {
fmt.Fprintf(w, "%v: ", p.Name)
}
fmt.Fprintf(w, "%v", p.Type)
}
// FullyQualifiedName is the usage of a Type, TypeMatcher or TemplateTypeParam
type FullyQualifiedName struct {
Target Named
TemplateArguments []interface{}
}
// Format implements the fmt.Formatter interface
func (f FullyQualifiedName) Format(w fmt.State, verb rune) {
fmt.Fprint(w, f.Target.GetName())
if len(f.TemplateArguments) > 0 {
fmt.Fprintf(w, "<")
for i, t := range f.TemplateArguments {
if i > 0 {
fmt.Fprint(w, ", ")
}
fmt.Fprintf(w, "%v", t)
}
fmt.Fprintf(w, ">")
}
}
// TemplateParam is a TemplateEnumParam, TemplateTypeParam or TemplateNumberParam
type TemplateParam interface {
Named
isTemplateParam()
}
func (*TemplateEnumParam) isTemplateParam() {}
func (*TemplateTypeParam) isTemplateParam() {}
func (*TemplateNumberParam) isTemplateParam() {}
// ResolvableType is a Type, TypeMatcher or TemplateTypeParam
type ResolvableType interface {
Named
isResolvableType()
}
func (*Type) isResolvableType() {}
func (*TypeMatcher) isResolvableType() {}
func (*TemplateTypeParam) isResolvableType() {}
// Named is something that can be looked up by name
type Named interface {
isNamed()
GetName() string
}
func (*Enum) isNamed() {}
func (*EnumEntry) isNamed() {}
func (*Type) isNamed() {}
func (*TypeMatcher) isNamed() {}
func (*EnumMatcher) isNamed() {}
func (*TemplateTypeParam) isNamed() {}
func (*TemplateEnumParam) isNamed() {}
func (*TemplateNumberParam) isNamed() {}
// GetName returns the name of the Enum
func (e *Enum) GetName() string { return e.Name }
// GetName returns the name of the EnumEntry
func (e *EnumEntry) GetName() string { return e.Name }
// GetName returns the name of the Type
func (t *Type) GetName() string { return t.Name }
// GetName returns the name of the TypeMatcher
func (t *TypeMatcher) GetName() string { return t.Name }
// GetName returns the name of the EnumMatcher
func (e *EnumMatcher) GetName() string { return e.Name }
// GetName returns the name of the TemplateTypeParam
func (t *TemplateTypeParam) GetName() string { return t.Name }
// GetName returns the name of the TemplateEnumParam
func (t *TemplateEnumParam) GetName() string { return t.Name }
// GetName returns the name of the TemplateNumberParam
func (t *TemplateNumberParam) GetName() string { return t.Name }

119
tools/src/cmd/builtin-gen/tok/tok.go Normal file
View File

@@ -0,0 +1,119 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package tok defines tokens that are produced by the Tint intrinsic definition
// lexer
package tok
import "fmt"
// Kind is an enumerator of token kinds
type Kind string
// Token enumerator types
const (
InvalidToken Kind = "<invalid>"
Identifier Kind = "ident"
Integer Kind = "integer"
String Kind = "string"
Match Kind = "match"
Function Kind = "fn"
Type Kind = "type"
Enum Kind = "enum"
Colon Kind = ":"
Comma Kind = ","
Lt Kind = "<"
Gt Kind = ">"
Lbrace Kind = "{"
Rbrace Kind = "}"
Ldeco Kind = "[["
Rdeco Kind = "]]"
Lparen Kind = "("
Rparen Kind = ")"
Or Kind = "|"
Arrow Kind = "->"
)
// Invalid represents an invalid token
var Invalid = Token{Kind: InvalidToken}
// Location describes a rune location in the source code
type Location struct {
// 1-based line index
Line int
// 1-based column index
Column int
// 0-based rune index
Rune int
// Optional file path
Filepath string
}
// Format implements the fmt.Formatter interface
func (l Location) Format(w fmt.State, verb rune) {
if w.Flag('+') {
if l.Filepath != "" {
fmt.Fprintf(w, "%v:%v:%v[%v]", l.Filepath, l.Line, l.Column, l.Rune)
} else {
fmt.Fprintf(w, "%v:%v[%v]", l.Line, l.Column, l.Rune)
}
} else {
if l.Filepath != "" {
fmt.Fprintf(w, "%v:%v:%v", l.Filepath, l.Line, l.Column)
} else {
fmt.Fprintf(w, "%v:%v", l.Line, l.Column)
}
}
}
// Source describes a start and end range in the source code
type Source struct {
S, E Location
}
// IsValid returns true if the source is valid
func (s Source) IsValid() bool {
return s.S.Line != 0 && s.S.Column != 0 && s.E.Line != 0 && s.E.Column != 0
}
// Format implements the fmt.Formatter interface
func (s Source) Format(w fmt.State, verb rune) {
if w.Flag('+') {
fmt.Fprint(w, "[")
s.S.Format(w, verb)
fmt.Fprint(w, " - ")
s.E.Format(w, verb)
fmt.Fprint(w, "]")
} else {
s.S.Format(w, verb)
}
}
// Token describes a parsed token
type Token struct {
Kind Kind
Runes []rune
Source Source
}
// Format implements the fmt.Formatter interface
func (t Token) Format(w fmt.State, verb rune) {
fmt.Fprint(w, "[")
t.Source.Format(w, verb)
fmt.Fprint(w, " ")
fmt.Fprint(w, t.Kind)
fmt.Fprint(w, " ")
fmt.Fprint(w, string(t.Runes))
fmt.Fprint(w, "]")
}

314
tools/src/cmd/check-spec-examples/main.go Normal file
View File

@@ -0,0 +1,314 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// check-spec-examples tests that WGSL specification examples compile as
// expected.
//
// The tool parses the WGSL HTML specification from the web or from a local file
// and then runs the WGSL compiler for all examples annotated with the 'wgsl'
// and 'global-scope' or 'function-scope' HTML class types.
//
// To run:
// go get golang.org/x/net/html # Only required once
// go run tools/check-spec-examples/main.go --compiler=<path-to-tint>
package main
import (
"errors"
"flag"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
"os/exec"
"path/filepath"
"strings"
"golang.org/x/net/html"
)
const (
toolName = "check-spec-examples"
defaultSpecPath = "https://gpuweb.github.io/gpuweb/wgsl/"
)
var (
errInvalidArg = errors.New("Invalid arguments")
)
func main() {
flag.Usage = func() {
out := flag.CommandLine.Output()
fmt.Fprintf(out, "%v tests that WGSL specification examples compile as expected.\n", toolName)
fmt.Fprintf(out, "\n")
fmt.Fprintf(out, "Usage:\n")
fmt.Fprintf(out, " %s [spec] [flags]\n", toolName)
fmt.Fprintf(out, "\n")
fmt.Fprintf(out, "spec is an optional local file path or URL to the WGSL specification.\n")
fmt.Fprintf(out, "If spec is omitted then the specification is fetched from %v\n", defaultSpecPath)
fmt.Fprintf(out, "\n")
fmt.Fprintf(out, "flags may be any combination of:\n")
flag.PrintDefaults()
}
err := run()
switch err {
case nil:
return
case errInvalidArg:
fmt.Fprintf(os.Stderr, "Error: %v\n\n", err)
flag.Usage()
default:
fmt.Fprintf(os.Stderr, "%v\n", err)
}
os.Exit(1)
}
func run() error {
// Parse flags
compilerPath := flag.String("compiler", "tint", "path to compiler executable")
verbose := flag.Bool("verbose", false, "print examples that pass")
flag.Parse()
// Try to find the WGSL compiler
compiler, err := exec.LookPath(*compilerPath)
if err != nil {
return fmt.Errorf("Failed to find WGSL compiler: %w", err)
}
if compiler, err = filepath.Abs(compiler); err != nil {
return fmt.Errorf("Failed to find WGSL compiler: %w", err)
}
// Check for explicit WGSL spec path
args := flag.Args()
specURL, _ := url.Parse(defaultSpecPath)
switch len(args) {
case 0:
case 1:
var err error
specURL, err = url.Parse(args[0])
if err != nil {
return err
}
default:
if len(args) > 1 {
return errInvalidArg
}
}
// The specURL might just be a local file path, in which case automatically
// add the 'file' URL scheme
if specURL.Scheme == "" {
specURL.Scheme = "file"
}
// Open the spec from HTTP(S) or from a local file
var specContent io.ReadCloser
switch specURL.Scheme {
case "http", "https":
response, err := http.Get(specURL.String())
if err != nil {
return fmt.Errorf("Failed to load the WGSL spec from '%v': %w", specURL, err)
}
specContent = response.Body
case "file":
specURL.Path, err = filepath.Abs(specURL.Path)
if err != nil {
return fmt.Errorf("Failed to load the WGSL spec from '%v': %w", specURL, err)
}
file, err := os.Open(specURL.Path)
if err != nil {
return fmt.Errorf("Failed to load the WGSL spec from '%v': %w", specURL, err)
}
specContent = file
default:
return fmt.Errorf("Unsupported URL scheme: %v", specURL.Scheme)
}
defer specContent.Close()
// Create the HTML parser
doc, err := html.Parse(specContent)
if err != nil {
return err
}
// Parse all the WGSL examples
examples := []example{}
if err := gatherExamples(doc, &examples); err != nil {
return err
}
if len(examples) == 0 {
return fmt.Errorf("no examples found")
}
// Create a temporary directory to hold the examples as separate files
tmpDir, err := ioutil.TempDir("", "wgsl-spec-examples")
if err != nil {
return err
}
if err := os.MkdirAll(tmpDir, 0666); err != nil {
return fmt.Errorf("Failed to create temporary directory: %w", err)
}
defer os.RemoveAll(tmpDir)
// For each compilable WGSL example...
for _, e := range examples {
exampleURL := specURL.String() + "#" + e.name
if err := tryCompile(compiler, tmpDir, e); err != nil {
if !e.expectError {
fmt.Printf("✘ %v ✘\n%v\n", exampleURL, err)
continue
}
} else if e.expectError {
fmt.Printf("✘ %v ✘\nCompiled even though it was marked with 'expect-error'\n", exampleURL)
}
if *verbose {
fmt.Printf("✔ %v ✔\n", exampleURL)
}
}
return nil
}
// Holds all the information about a single, compilable WGSL example in the spec
type example struct {
name string // The name (typically hash generated by bikeshed)
code string // The example source
globalScope bool // Annotated with 'global-scope' ?
functionScope bool // Annotated with 'function-scope' ?
expectError bool // Annotated with 'expect-error' ?
}
// tryCompile attempts to compile the example e in the directory wd, using the
// compiler at the given path. If the example is annotated with 'function-scope'
// then the code is wrapped with a basic vertex-stage-entry function.
// If the first compile fails then a placeholder vertex-state-entry
// function is appended to the source, and another attempt to compile
// the shader is made.
func tryCompile(compiler, wd string, e example) error {
code := e.code
if e.functionScope {
code = "\n@stage(vertex) fn main() -> @builtin(position) vec4<f32> {\n" + code + " return vec4<f32>();}\n"
}
addedStubFunction := false
for {
err := compile(compiler, wd, e.name, code)
if err == nil {
return nil
}
if !addedStubFunction {
code += "\n@stage(vertex) fn main() {}\n"
addedStubFunction = true
continue
}
return err
}
}
// compile creates a file in wd and uses the compiler to attempt to compile it.
func compile(compiler, wd, name, code string) error {
filename := name + ".wgsl"
path := filepath.Join(wd, filename)
if err := ioutil.WriteFile(path, []byte(code), 0666); err != nil {
return fmt.Errorf("Failed to write example file '%v'", path)
}
cmd := exec.Command(compiler, filename)
cmd.Dir = wd
out, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("%v\n%v", err, string(out))
}
return nil
}
// gatherExamples scans the HTML node and its children for blocks that contain
// WGSL example code, populating the examples slice.
func gatherExamples(node *html.Node, examples *[]example) error {
if hasClass(node, "example") && hasClass(node, "wgsl") {
e := example{
name: nodeID(node),
code: exampleCode(node),
globalScope: hasClass(node, "global-scope"),
functionScope: hasClass(node, "function-scope"),
expectError: hasClass(node, "expect-error"),
}
// If the example is annotated with a scope, then it can be compiled.
if e.globalScope || e.functionScope {
*examples = append(*examples, e)
}
}
for child := node.FirstChild; child != nil; child = child.NextSibling {
if err := gatherExamples(child, examples); err != nil {
return err
}
}
return nil
}
// exampleCode returns a string formed from all the TextNodes found in <pre>
// blocks that are children of node.
func exampleCode(node *html.Node) string {
sb := strings.Builder{}
for child := node.FirstChild; child != nil; child = child.NextSibling {
if child.Data == "pre" {
printNodeText(child, &sb)
}
}
return sb.String()
}
// printNodeText traverses node and its children, writing the Data of all
// TextNodes to sb.
func printNodeText(node *html.Node, sb *strings.Builder) {
if node.Type == html.TextNode {
sb.WriteString(node.Data)
}
for child := node.FirstChild; child != nil; child = child.NextSibling {
printNodeText(child, sb)
}
}
// hasClass returns true iff node is has the given "class" attribute.
func hasClass(node *html.Node, class string) bool {
for _, attr := range node.Attr {
if attr.Key == "class" {
classes := strings.Split(attr.Val, " ")
for _, c := range classes {
if c == class {
return true
}
}
}
}
return false
}
// nodeID returns the given "id" attribute of node, or an empty string if there
// is no "id" attribute.
func nodeID(node *html.Node) string {
for _, attr := range node.Attr {
if attr.Key == "id" {
return attr.Val
}
}
return ""
}

340
tools/src/cmd/fix-tests/main.go Normal file
View File

@@ -0,0 +1,340 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// fix-tests is a tool to update tests with new expected output.
package main
import (
"encoding/json"
"flag"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"
"dawn.googlesource.com/tint/tools/src/substr"
)
func main() {
if err := run(); err != nil {
fmt.Println(err)
os.Exit(1)
}
}
func showUsage() {
fmt.Println(`
fix-tests is a tool to update tests with new expected output.
fix-tests performs string matching and heuristics to fix up expected results of
tests that use EXPECT_EQ(a, b) and EXPECT_THAT(a, HasSubstr(b))
WARNING: Always thoroughly check the generated output for mistakes.
This may produce incorrect output
Usage:
fix-tests <executable>
executable - the path to the test executable to run.`)
os.Exit(1)
}
func run() error {
flag.Parse()
args := flag.Args()
if len(args) < 1 {
showUsage()
}
exe := args[0] // The path to the test executable
wd := filepath.Dir(exe) // The directory holding the test exe
// Create a temporary directory to hold the 'test-results.json' file
tmpDir, err := ioutil.TempDir("", "fix-tests")
if err != nil {
return err
}
if err := os.MkdirAll(tmpDir, 0666); err != nil {
return fmt.Errorf("Failed to create temporary directory: %w", err)
}
defer os.RemoveAll(tmpDir)
// Full path to the 'test-results.json' in the temporary directory
testResultsPath := filepath.Join(tmpDir, "test-results.json")
// Run the tests
testArgs := []string{"--gtest_output=json:" + testResultsPath}
if len(args) > 1 {
testArgs = append(testArgs, args[1:]...)
}
switch err := exec.Command(exe, testArgs...).Run().(type) {
default:
return err
case nil:
fmt.Println("All tests passed")
case *exec.ExitError:
}
// Read the 'test-results.json' file
testResultsFile, err := os.Open(testResultsPath)
if err != nil {
return err
}
var testResults Results
if err := json.NewDecoder(testResultsFile).Decode(&testResults); err != nil {
return err
}
// For each failing test...
seen := map[string]bool{}
numFixed, numFailed := 0, 0
for _, group := range testResults.Groups {
for _, suite := range group.Testsuites {
for _, failure := range suite.Failures {
// .. attempt to fix the problem
test := testName(group, suite)
if seen[test] {
continue
}
seen[test] = true
if err := processFailure(test, wd, failure.Failure); err != nil {
fmt.Println(fmt.Errorf("%v: %w", test, err))
numFailed++
} else {
numFixed++
}
}
}
}
fmt.Println()
if numFailed > 0 {
fmt.Println(numFailed, "tests could not be fixed")
}
if numFixed > 0 {
fmt.Println(numFixed, "tests fixed")
}
return nil
}
func testName(group TestsuiteGroup, suite Testsuite) string {
groupParts := strings.Split(group.Name, "/")
suiteParts := strings.Split(suite.Name, "/")
return groupParts[len(groupParts)-1] + "." + suiteParts[0]
}
var (
// Regular expression to match a test declaration
reTests = regexp.MustCompile(`TEST(?:_[FP])?\([ \n]*(\w+),[ \n]*(\w+)\)`)
// Regular expression to match a `EXPECT_EQ(a, b)` failure for strings
reExpectEq = regexp.MustCompile(`([./\\\w_\-:]*):(\d+).*\nExpected equality of these values:\n(?:.|\n)*?(?:Which is: | )"((?:.|\n)*?[^\\])"\n(?:.|\n)*?(?:Which is: | )"((?:.|\n)*?[^\\])"`)
// Regular expression to match a `EXPECT_THAT(a, HasSubstr(b))` failure for strings
reExpectHasSubstr = regexp.MustCompile(`([./\\\w_\-:]*):(\d+).*\nValue of: .*\nExpected: has substring "((?:.|\n)*?[^\\])"\n Actual: "((?:.|\n)*?[^\\])"`)
)
func processFailure(test, wd, failure string) error {
// Start by un-escaping newlines in the failure message
failure = strings.ReplaceAll(failure, "\\n", "\n")
// Matched regex strings will also need to be un-escaped, but do this after
// the match, as unescaped quotes may upset the regex patterns
unescape := func(s string) string {
return strings.ReplaceAll(s, `\"`, `"`)
}
escape := func(s string) string {
s = strings.ReplaceAll(s, "\n", `\n`)
s = strings.ReplaceAll(s, "\"", `\"`)
return s
}
// Look for a EXPECT_EQ failure pattern
var file string
var fix func(testSource string) (string, error)
if parts := reExpectEq.FindStringSubmatch(failure); len(parts) == 5 {
// EXPECT_EQ(a, b)
a, b := unescape(parts[3]), unescape(parts[4])
file = parts[1]
fix = func(testSource string) (string, error) {
// We don't know if a or b is the expected, so just try flipping the string
// to the other form.
if len(b) > len(a) { // Go with the longer match, in case both are found
a, b = b, a
}
switch {
case strings.Contains(testSource, a):
testSource = strings.ReplaceAll(testSource, a, b)
case strings.Contains(testSource, b):
testSource = strings.ReplaceAll(testSource, b, a)
default:
// Try escaping for R"(...)" strings
a, b = escape(a), escape(b)
switch {
case strings.Contains(testSource, a):
testSource = strings.ReplaceAll(testSource, a, b)
case strings.Contains(testSource, b):
testSource = strings.ReplaceAll(testSource, b, a)
default:
return "", fmt.Errorf("Could not fix 'EXPECT_EQ' pattern in '%v'", file)
}
}
return testSource, nil
}
} else if parts := reExpectHasSubstr.FindStringSubmatch(failure); len(parts) == 5 {
// EXPECT_THAT(a, HasSubstr(b))
a, b := unescape(parts[4]), unescape(parts[3])
file = parts[1]
fix = func(testSource string) (string, error) {
if fix := substr.Fix(a, b); fix != "" {
if !strings.Contains(testSource, b) {
// Try escaping for R"(...)" strings
b, fix = escape(b), escape(fix)
}
if strings.Contains(testSource, b) {
testSource = strings.Replace(testSource, b, fix, -1)
return testSource, nil
}
return "", fmt.Errorf("Could apply fix for 'HasSubstr' pattern in '%v'", file)
}
return "", fmt.Errorf("Could find fix for 'HasSubstr' pattern in '%v'", file)
}
} else {
return fmt.Errorf("Cannot fix this type of failure")
}
// Get the absolute source path
sourcePath := file
if !filepath.IsAbs(sourcePath) {
sourcePath = filepath.Join(wd, file)
}
// Parse the source file, split into tests
sourceFile, err := parseSourceFile(sourcePath)
if err != nil {
return fmt.Errorf("Couldn't parse tests from file '%v': %w", file, err)
}
// Find the test
testIdx, ok := sourceFile.tests[test]
if !ok {
return fmt.Errorf("Test not found in '%v'", file)
}
// Grab the source for the particular test
testSource := sourceFile.parts[testIdx]
if testSource, err = fix(testSource); err != nil {
return err
}
// Replace the part of the source file
sourceFile.parts[testIdx] = testSource
// Write out the source file
return writeSourceFile(sourcePath, sourceFile)
}
// parseSourceFile() reads the file at path, splitting the content into chunks
// for each TEST.
func parseSourceFile(path string) (sourceFile, error) {
fileBytes, err := ioutil.ReadFile(path)
if err != nil {
return sourceFile{}, err
}
fileContent := string(fileBytes)
out := sourceFile{
tests: map[string]int{},
}
pos := 0
for _, span := range reTests.FindAllStringIndex(fileContent, -1) {
out.parts = append(out.parts, fileContent[pos:span[0]])
pos = span[0]
match := reTests.FindStringSubmatch(fileContent[span[0]:span[1]])
group := match[1]
suite := match[2]
out.tests[group+"."+suite] = len(out.parts)
}
out.parts = append(out.parts, fileContent[pos:])
return out, nil
}
// writeSourceFile() joins the chunks of the file, and writes the content out to
// path.
func writeSourceFile(path string, file sourceFile) error {
body := strings.Join(file.parts, "")
return ioutil.WriteFile(path, []byte(body), 0666)
}
type sourceFile struct {
parts []string
tests map[string]int // "X.Y" -> part index
}
// Results is the root JSON structure of the JSON --gtest_output file .
type Results struct {
Tests int `json:"tests"`
Failures int `json:"failures"`
Disabled int `json:"disabled"`
Errors int `json:"errors"`
Timestamp string `json:"timestamp"`
Time string `json:"time"`
Name string `json:"name"`
Groups []TestsuiteGroup `json:"testsuites"`
}
// TestsuiteGroup is a group of test suites in the JSON --gtest_output file .
type TestsuiteGroup struct {
Name string `json:"name"`
Tests int `json:"tests"`
Failures int `json:"failures"`
Disabled int `json:"disabled"`
Errors int `json:"errors"`
Timestamp string `json:"timestamp"`
Time string `json:"time"`
Testsuites []Testsuite `json:"testsuite"`
}
// Testsuite is a suite of tests in the JSON --gtest_output file.
type Testsuite struct {
Name string `json:"name"`
ValueParam string `json:"value_param,omitempty"`
Status Status `json:"status"`
Result Result `json:"result"`
Timestamp string `json:"timestamp"`
Time string `json:"time"`
Classname string `json:"classname"`
Failures []Failure `json:"failures,omitempty"`
}
// Failure is a reported test failure in the JSON --gtest_output file.
type Failure struct {
Failure string `json:"failure"`
Type string `json:"type"`
}
// Status is a status code in the JSON --gtest_output file.
type Status string
// Result is a result code in the JSON --gtest_output file.
type Result string

152
tools/src/cmd/gerrit-stats/main.go Normal file
View File

@@ -0,0 +1,152 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// gerrit-stats gathers statistics about changes made to Tint.
package main
import (
"flag"
"fmt"
"net/url"
"os"
"regexp"
"time"
"dawn.googlesource.com/tint/tools/src/gerrit"
)
const yyyymmdd = "2006-01-02"
var (
// See https://dawn-review.googlesource.com/new-password for obtaining
// username and password for gerrit.
gerritUser = flag.String("gerrit-user", "", "gerrit authentication username")
gerritPass = flag.String("gerrit-pass", "", "gerrit authentication password")
repoFlag = flag.String("repo", "tint", "the project (tint or dawn)")
userFlag = flag.String("user", "", "user name / email")
afterFlag = flag.String("after", "", "start date")
beforeFlag = flag.String("before", "", "end date")
daysFlag = flag.Int("days", 182, "interval in days (used if --after is not specified)")
verboseFlag = flag.Bool("v", false, "verbose mode - lists all the changes")
)
func main() {
flag.Parse()
if err := run(); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
func run() error {
var after, before time.Time
var err error
user := *userFlag
if user == "" {
return fmt.Errorf("Missing required 'user' flag")
}
if *beforeFlag != "" {
before, err = time.Parse(yyyymmdd, *beforeFlag)
if err != nil {
return fmt.Errorf("Couldn't parse before date: %w", err)
}
} else {
before = time.Now()
}
if *afterFlag != "" {
after, err = time.Parse(yyyymmdd, *afterFlag)
if err != nil {
return fmt.Errorf("Couldn't parse after date: %w", err)
}
} else {
after = before.Add(-time.Hour * time.Duration(24**daysFlag))
}
g, err := gerrit.New(gerrit.Config{Username: *gerritUser, Password: *gerritPass})
if err != nil {
return err
}
submitted, submittedQuery, err := g.QueryChanges(
"status:merged",
"owner:"+user,
"after:"+date(after),
"before:"+date(before),
"repo:"+*repoFlag)
if err != nil {
return fmt.Errorf("Query failed: %w", err)
}
reviewed, reviewQuery, err := g.QueryChanges(
"commentby:"+user,
"-owner:"+user,
"after:"+date(after),
"before:"+date(before),
"repo:"+*repoFlag)
if err != nil {
return fmt.Errorf("Query failed: %w", err)
}
ignorelist := []*regexp.Regexp{
regexp.MustCompile("Revert .*"),
}
ignore := func(s string) bool {
for _, re := range ignorelist {
if re.MatchString(s) {
return true
}
}
return false
}
insertions, deletions := 0, 0
for _, change := range submitted {
if ignore(change.Subject) {
continue
}
insertions += change.Insertions
deletions += change.Deletions
}
fmt.Printf("Between %v and %v, %v:\n", date(after), date(before), user)
fmt.Printf(" Submitted %v changes (LOC: %v+, %v-) \n", len(submitted), insertions, deletions)
fmt.Printf(" Reviewed %v changes\n", len(reviewed))
fmt.Printf("\n")
if *verboseFlag {
fmt.Printf("Submitted changes:\n")
for i, change := range submitted {
fmt.Printf("%3.1v: %6.v %v (LOC: %v+, %v-)\n", i, change.Number, change.Subject, change.Insertions, change.Deletions)
}
fmt.Printf("\n")
fmt.Printf("Reviewed changes:\n")
for i, change := range reviewed {
fmt.Printf("%3.1v: %6.v %v (LOC: %v+, %v-)\n", i, change.Number, change.Subject, change.Insertions, change.Deletions)
}
}
fmt.Printf("\n")
fmt.Printf("Submitted query: %vq/%v\n", gerrit.URL, url.QueryEscape(submittedQuery))
fmt.Printf("Review query: %vq/%v\n", gerrit.URL, url.QueryEscape(reviewQuery))
return nil
}
func today() time.Time {
return time.Now()
}
func date(t time.Time) string {
return t.Format(yyyymmdd)
}

1016
tools/src/cmd/get-test-plan/main.go Normal file
View File

File diff suppressed because it is too large Load Diff

937
tools/src/cmd/perfmon/main.go Normal file
View File

@@ -0,0 +1,937 @@
// Copyright 2022 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"log"
"os"
"os/exec"
"path/filepath"
"reflect"
"sort"
"strings"
"time"
"dawn.googlesource.com/tint/tools/src/bench"
"dawn.googlesource.com/tint/tools/src/git"
"github.com/andygrunwald/go-gerrit"
"github.com/go-git/go-git/v5/plumbing/transport"
"github.com/go-git/go-git/v5/plumbing/transport/http"
"github.com/shirou/gopsutil/cpu"
)
// main entry point
func main() {
var cfgPath string
flag.StringVar(&cfgPath, "c", "~/.config/perfmon/config.json", "the config file")
flag.Parse()
if err := run(cfgPath); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
// run starts the perfmon tool with the given config path
func run(cfgPath string) error {
cfgPath, err := expandHomeDir(cfgPath)
if err != nil {
return err
}
if err := findTools(); err != nil {
return err
}
g, err := git.New(tools.git)
if err != nil {
return err
}
cfg, err := loadConfig(cfgPath)
if err != nil {
return err
}
tintDir, resultsDir, err := makeWorkingDirs(cfg)
if err != nil {
return err
}
tintRepo, err := createOrOpenGitRepo(g, tintDir, cfg.Tint)
if err != nil {
return err
}
resultsRepo, err := createOrOpenGitRepo(g, resultsDir, cfg.Results)
if err != nil {
return err
}
gerritClient, err := gerrit.NewClient(cfg.Gerrit.URL, nil)
if err != nil {
return err
}
gerritClient.Authentication.SetBasicAuth(cfg.Gerrit.Username, cfg.Gerrit.Password)
sysInfo, err := cpu.Info()
if err != nil {
return fmt.Errorf("failed to obtain system info:\n %v", err)
}
e := env{
cfg: cfg,
git: g,
system: sysInfo,
systemID: hash(sysInfo)[:8],
tintDir: tintDir,
buildDir: filepath.Join(tintDir, "out"),
resultsDir: resultsDir,
tintRepo: tintRepo,
resultsRepo: resultsRepo,
gerrit: gerritClient,
benchmarkCache: map[git.Hash]*bench.Run{},
}
for true {
didSomething, err := e.doSomeWork()
if err != nil {
log.Printf("ERROR: %v", err)
time.Sleep(time.Minute * 10)
continue
}
if !didSomething {
log.Println("nothing to do. Sleeping...")
time.Sleep(time.Minute * 5)
}
}
return nil
}
// Config holds the root configuration options for the perfmon tool
type Config struct {
WorkingDir string
RootChange git.Hash
Tint GitConfig
Results GitConfig
Gerrit GerritConfig
Timeouts TimeoutsConfig
ExternalAccounts []string
BenchmarkRepetitions int
}
// GitConfig holds the configuration options for accessing a git repo
type GitConfig struct {
URL string
Branch string
Auth git.Auth
}
// GerritConfig holds the configuration options for accessing gerrit
type GerritConfig struct {
URL string
Username string
Email string
Password string
}
// TimeoutsConfig holds the configuration options for timeouts
type TimeoutsConfig struct {
Sync time.Duration
Build time.Duration
Benchmark time.Duration
}
// HistoricResults contains the full set of historic benchmark results for a
// given system
type HistoricResults struct {
System []cpu.InfoStat
Commits []CommitResults
}
// CommitResults holds the results of a single tint commit
type CommitResults struct {
Commit string
CommitTime time.Time
CommitDescription string
Benchmarks []Benchmark
}
// Benchmark holds the benchmark results for a single test
type Benchmark struct {
Name string
Mean float64
Median float64
Stddev float64
}
// setDefaults assigns default values to unassigned fields of cfg
func (cfg *Config) setDefaults() {
if cfg.RootChange.IsZero() {
cfg.RootChange, _ = git.ParseHash("be2362b18c792364c6bf5744db6d3837fbc655a0")
}
cfg.Tint.setDefaults()
cfg.Results.setDefaults()
cfg.Timeouts.setDefaults()
if cfg.BenchmarkRepetitions < 2 {
cfg.BenchmarkRepetitions = 2
}
}
// setDefaults assigns default values to unassigned fields of cfg
func (cfg *GitConfig) setDefaults() {
if cfg.Branch == "" {
cfg.Branch = "main"
}
}
// setDefaults assigns default values to unassigned fields of cfg
func (cfg *TimeoutsConfig) setDefaults() {
if cfg.Sync == 0 {
cfg.Sync = time.Minute * 10
}
if cfg.Build == 0 {
cfg.Build = time.Minute * 10
}
if cfg.Benchmark == 0 {
cfg.Benchmark = time.Minute * 30
}
}
// AuthConfig holds the authentication options for accessing a git repo
type AuthConfig struct {
Username string
Password string
}
// authMethod returns a http.BasicAuth constructed from the AuthConfig
func (cfg AuthConfig) authMethod() transport.AuthMethod {
if cfg.Username != "" || cfg.Password != "" {
return &http.BasicAuth{Username: cfg.Username, Password: cfg.Password}
}
return nil
}
// env holds the perfmon main environment state
type env struct {
cfg Config
git *git.Git
system []cpu.InfoStat
systemID string
tintDir string
buildDir string
resultsDir string
tintRepo *git.Repository
resultsRepo *git.Repository
gerrit *gerrit.Client
benchmarkCache map[git.Hash]*bench.Run
}
// doSomeWork scans gerrit for changes up for review and submitted changes to
// benchmark. If something was found to do, then returns true.
func (e env) doSomeWork() (bool, error) {
{
log.Println("scanning for review changes to benchmark...")
change, err := e.findGerritChangeToBenchmark()
if err != nil {
return true, err
}
if change != nil {
if err := e.benchmarkGerritChange(*change); err != nil {
return true, err
}
return true, nil
}
}
{
log.Println("scanning for submitted changes to benchmark...")
changesToBenchmark, err := e.changesToBenchmark()
if err != nil {
return true, err
}
if len(changesToBenchmark) > 0 {
log.Printf("benchmarking %v changes...", len(changesToBenchmark))
for i, c := range changesToBenchmark {
log.Printf("benchmarking %v/%v....", i+1, len(changesToBenchmark))
benchRes, err := e.benchmarkTintChange(c)
if err != nil {
return true, err
}
commitRes, err := e.benchmarksToCommitResults(c, *benchRes)
if err != nil {
return true, err
}
log.Printf("pushing results...")
if err := e.pushUpdatedResults(*commitRes); err != nil {
return true, err
}
}
return true, nil
}
}
return false, nil
}
// changesToBenchmark fetches the list of changes that do not currently have
// benchmark results, which should be benchmarked.
func (e env) changesToBenchmark() ([]git.Hash, error) {
log.Println("syncing tint repo...")
latest, err := e.tintRepo.Fetch(e.cfg.Tint.Branch, &git.FetchOptions{Auth: e.cfg.Tint.Auth})
if err != nil {
return nil, err
}
allChanges, err := e.tintRepo.Log(&git.LogOptions{
From: e.cfg.RootChange.String(),
To: latest.String(),
})
if err != nil {
return nil, fmt.Errorf("failed to obtain tint log:\n %w", err)
}
changesWithBenchmarks, err := e.changesWithBenchmarks()
if err != nil {
return nil, fmt.Errorf("failed to gather changes with existing benchmarks:\n %w", err)
}
changesToBenchmark := make([]git.Hash, 0, len(allChanges))
for _, c := range allChanges {
if _, exists := changesWithBenchmarks[c.Hash]; !exists {
changesToBenchmark = append(changesToBenchmark, c.Hash)
}
}
// Reverse the order of changesToBenchmark, so that the oldest comes first.
for i := len(changesToBenchmark)/2 - 1; i >= 0; i-- {
j := len(changesToBenchmark) - 1 - i
changesToBenchmark[i], changesToBenchmark[j] = changesToBenchmark[j], changesToBenchmark[i]
}
return changesToBenchmark, nil
}
// benchmarkTintChange checks out the given commit, fetches the tint third party
// dependencies, builds tint, then runs the benchmarks, returning the results.
func (e env) benchmarkTintChange(hash git.Hash) (*bench.Run, error) {
if cached, ok := e.benchmarkCache[hash]; ok {
log.Printf("reusing cached benchmark results of '%v'...", hash)
return cached, nil
}
log.Printf("checking out tint at '%v'...", hash)
if err := checkout(hash, e.tintRepo); err != nil {
return nil, err
}
log.Println("fetching tint dependencies...")
if err := e.fetchTintDeps(); err != nil {
return nil, err
}
log.Println("building tint...")
if err := e.buildTint(); err != nil {
return nil, err
}
log.Println("benchmarking tint...")
run, err := e.benchmarkTint()
if err != nil {
return nil, err
}
e.benchmarkCache[hash] = run
return run, nil
}
// benchmarksToCommitResults converts the benchmarks in the provided bench.Run
// to a CommitResults.
func (e env) benchmarksToCommitResults(hash git.Hash, results bench.Run) (*CommitResults, error) {
commits, err := e.tintRepo.Log(&git.LogOptions{
From: hash.String(),
Count: 1,
})
if err != nil || len(commits) != 1 {
return nil, fmt.Errorf("failed to get commit object '%v' of tint repo:\n %w", hash, err)
}
commit := commits[0]
m := map[string]Benchmark{}
for _, b := range results.Benchmarks {
benchmark := m[b.Name]
benchmark.Name = b.Name
switch b.AggregateType {
case bench.Mean:
benchmark.Mean = float64(b.Duration) / float64(time.Second)
case bench.Median:
benchmark.Median = float64(b.Duration) / float64(time.Second)
case bench.Stddev:
benchmark.Stddev = float64(b.Duration) / float64(time.Second)
}
m[b.Name] = benchmark
}
sorted := make([]Benchmark, 0, len(m))
for _, b := range m {
sorted = append(sorted, b)
}
sort.Slice(sorted, func(i, j int) bool { return sorted[i].Name < sorted[i].Name })
return &CommitResults{
Commit: commit.Hash.String(),
CommitDescription: commit.Subject,
CommitTime: commit.Date,
Benchmarks: sorted,
}, nil
}
// changesWithBenchmarks returns a set of tint changes that we already have
// benchmarks for.
func (e env) changesWithBenchmarks() (map[git.Hash]struct{}, error) {
log.Println("syncing results repo...")
if err := fetchAndCheckoutLatest(e.resultsRepo, e.cfg.Results); err != nil {
return nil, err
}
_, absPath, err := e.resultsFilePaths()
if err != nil {
return nil, err
}
results, err := e.loadHistoricResults(absPath)
if err != nil {
log.Println(fmt.Errorf("WARNING: failed to open result file '%v':\n %w", absPath, err))
return nil, nil
}
m := make(map[git.Hash]struct{}, len(results.Commits))
for _, c := range results.Commits {
hash, err := git.ParseHash(c.Commit)
if err != nil {
return nil, err
}
m[hash] = struct{}{}
}
return m, nil
}
func (e env) pushUpdatedResults(res CommitResults) error {
log.Println("syncing results repo...")
if err := fetchAndCheckoutLatest(e.resultsRepo, e.cfg.Results); err != nil {
return err
}
relPath, absPath, err := e.resultsFilePaths()
if err != nil {
return err
}
h, err := e.loadHistoricResults(absPath)
if err != nil {
log.Println(fmt.Errorf("failed to open result file '%v'. Creating new file\n %w", absPath, err))
h = &HistoricResults{System: e.system}
}
h.Commits = append(h.Commits, res)
// Sort the commits by timestamp
sort.Slice(h.Commits, func(i, j int) bool { return h.Commits[i].CommitTime.Before(h.Commits[j].CommitTime) })
// Write the new results to the file
f, err := os.Create(absPath)
if err != nil {
return fmt.Errorf("failed to create updated results file '%v':\n %w", absPath, err)
}
defer f.Close()
enc := json.NewEncoder(f)
enc.SetIndent("", " ")
if err := enc.Encode(h); err != nil {
return fmt.Errorf("failed to encode updated results file '%v':\n %w", absPath, err)
}
// Stage the file
if err := e.resultsRepo.Add(relPath, nil); err != nil {
return fmt.Errorf("failed to stage updated results file '%v':\n %w", relPath, err)
}
// Commit the change
msg := fmt.Sprintf("Add benchmark results for '%v'", res.Commit[:6])
hash, err := e.resultsRepo.Commit(msg, &git.CommitOptions{
AuthorName: "tint perfmon bot",
AuthorEmail: "tint-perfmon-bot@gmail.com",
})
if err != nil {
return fmt.Errorf("failed to commit updated results file '%v':\n %w", absPath, err)
}
// Push the change
log.Println("pushing updated results to results repo...")
if err := e.resultsRepo.Push(hash.String(), e.cfg.Results.Branch, &git.PushOptions{Auth: e.cfg.Results.Auth}); err != nil {
return fmt.Errorf("failed to push updated results file '%v':\n %w", absPath, err)
}
return nil
}
// resultsFilePaths returns the paths to the results.json file, holding the
// benchmarks for the given system.
func (e env) resultsFilePaths() (relPath string, absPath string, err error) {
dir := filepath.Join(e.resultsDir, "results")
if err = os.MkdirAll(dir, 0777); err != nil {
err = fmt.Errorf("failed to create results directory '%v':\n %w", dir, err)
return
}
relPath = filepath.Join("results", e.systemID+".json")
absPath = filepath.Join(dir, e.systemID+".json")
return
}
// loadHistoricResults loads and returns the results.json file for the given
// system.
func (e env) loadHistoricResults(path string) (*HistoricResults, error) {
file, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("failed to open result file '%v':\n %w", path, err)
}
defer file.Close()
res := &HistoricResults{}
if err := json.NewDecoder(file).Decode(res); err != nil {
return nil, fmt.Errorf("failed to parse result file '%v':\n %w", path, err)
}
if !reflect.DeepEqual(res.System, e.system) {
log.Printf(`WARNING: results file '%v' has different system information!
File: %+v
System: %+v`, path, res.System, e.system)
}
return res, nil
}
// fetchTintDeps fetches the third party tint dependencies using gclient.
func (e env) fetchTintDeps() error {
gclientConfig := filepath.Join(e.tintDir, ".gclient")
if _, err := os.Stat(gclientConfig); errors.Is(err, os.ErrNotExist) {
standalone := filepath.Join(e.tintDir, "standalone.gclient")
if err := copyFile(gclientConfig, standalone); err != nil {
return fmt.Errorf("failed to copy '%v' to '%v':\n %w", standalone, gclientConfig, err)
}
}
if _, err := call(tools.gclient, e.tintDir, e.cfg.Timeouts.Sync,
"sync",
"--force",
); err != nil {
return fmt.Errorf("failed to fetch tint dependencies:\n %w", err)
}
return nil
}
// buildTint builds the tint benchmarks.
func (e env) buildTint() error {
if err := os.MkdirAll(e.buildDir, 0777); err != nil {
return fmt.Errorf("failed to create build directory at '%v':\n %w", e.buildDir, err)
}
if _, err := call(tools.cmake, e.buildDir, e.cfg.Timeouts.Build,
e.tintDir,
"-GNinja",
"-DCMAKE_CXX_COMPILER_LAUNCHER=ccache",
"-DCMAKE_BUILD_TYPE=Release",
"-DTINT_BUILD_SPV_READER=1",
"-DTINT_BUILD_WGSL_READER=1",
"-DTINT_BUILD_GLSL_WRITER=1",
"-DTINT_BUILD_HLSL_WRITER=1",
"-DTINT_BUILD_MSL_WRITER=1",
"-DTINT_BUILD_SPV_WRITER=1",
"-DTINT_BUILD_WGSL_WRITER=1",
"-DTINT_BUILD_BENCHMARKS=1",
); err != nil {
return errFailedToBuild{fmt.Errorf("failed to generate tint build config:\n %w", err)}
}
if _, err := call(tools.ninja, e.buildDir, e.cfg.Timeouts.Build); err != nil {
return errFailedToBuild{err}
}
return nil
}
// errFailedToBuild is the error returned by buildTint() if the build failed
type errFailedToBuild struct {
// The reason
reason error
}
func (e errFailedToBuild) Error() string {
return fmt.Sprintf("failed to build: %v", e.reason)
}
// benchmarkTint runs the tint benchmarks, returning the results.
func (e env) benchmarkTint() (*bench.Run, error) {
exe := filepath.Join(e.buildDir, "tint-benchmark")
out, err := call(exe, e.buildDir, e.cfg.Timeouts.Benchmark,
"--benchmark_format=json",
fmt.Sprintf("--benchmark_repetitions=%v", e.cfg.BenchmarkRepetitions),
)
if err != nil {
return nil, fmt.Errorf("failed to benchmark tint:\n %w", err)
}
results, err := bench.Parse(out)
if err != nil {
return nil, fmt.Errorf("failed to parse benchmark results:\n %w", err)
}
return &results, nil
}
// findGerritChangeToBenchmark queries gerrit for a change to benchmark.
func (e env) findGerritChangeToBenchmark() (*gerrit.ChangeInfo, error) {
log.Println("querying gerrit for changes...")
results, _, err := e.gerrit.Changes.QueryChanges(&gerrit.QueryChangeOptions{
QueryOptions: gerrit.QueryOptions{
Query: []string{"project:tint status:open+-age:3d"},
Limit: 100,
},
ChangeOptions: gerrit.ChangeOptions{
AdditionalFields: []string{"CURRENT_REVISION", "CURRENT_COMMIT", "MESSAGES", "LABELS", "DETAILED_ACCOUNTS"},
},
})
if err != nil {
return nil, fmt.Errorf("failed to get list of changes:\n %w", err)
}
type candidate struct {
change gerrit.ChangeInfo
priority int
}
candidates := make([]candidate, 0, len(*results))
for _, change := range *results {
kokoroApproved := change.Labels["Kokoro"].Approved.AccountID != 0
codeReviewScore := change.Labels["Code-Review"].Value
codeReviewApproved := change.Labels["Code-Review"].Approved.AccountID != 0
presubmitReady := change.Labels["Presubmit-Ready"].Approved.AccountID != 0
verifiedScore := change.Labels["Verified"].Value
current, ok := change.Revisions[change.CurrentRevision]
if !ok {
log.Printf("WARNING: couldn't find current revision for change '%s'", change.ChangeID)
}
canBenchmark := func() bool {
// Is the change from a Googler, reviewed by a Googler or is from a allow-listed external developer?
if !(strings.HasSuffix(current.Commit.Committer.Email, "@google.com") ||
strings.HasSuffix(change.Labels["Code-Review"].Approved.Email, "@google.com") ||
strings.HasSuffix(change.Labels["Code-Review"].Recommended.Email, "@google.com") ||
strings.HasSuffix(change.Labels["Presubmit-Ready"].Approved.Email, "@google.com")) {
permitted := false
for _, email := range e.cfg.ExternalAccounts {
if strings.ToLower(current.Commit.Committer.Email) == strings.ToLower(email) {
permitted = true
break
}
}
if !permitted {
return false
}
}
// Don't benchmark if the change has negative scores.
if codeReviewScore < 0 || verifiedScore < 0 {
return false
}
// Has the latest patchset already been benchmarked?
for _, msg := range change.Messages {
if msg.RevisionNumber == current.Number &&
msg.Author.Email == e.cfg.Gerrit.Email {
return false
}
}
return true
}()
if !canBenchmark {
continue
}
priority := 0
if presubmitReady {
priority += 10
}
priority += codeReviewScore
if codeReviewApproved {
priority += 2
}
if kokoroApproved {
priority++
}
candidates = append(candidates, candidate{change, priority})
}
// Sort the candidates
sort.Slice(candidates, func(i, j int) bool {
return candidates[i].priority > candidates[j].priority
})
if len(candidates) > 0 {
log.Printf("%d gerrit changes to benchmark", len(candidates))
return &candidates[0].change, nil
}
return nil, nil
}
// benchmarks the gerrit change, posting the findings to the change
func (e env) benchmarkGerritChange(change gerrit.ChangeInfo) error {
current := change.Revisions[change.CurrentRevision]
log.Printf("fetching '%v'...", current.Ref)
currentHash, err := e.tintRepo.Fetch(current.Ref, &git.FetchOptions{Auth: e.cfg.Tint.Auth})
if err != nil {
return err
}
parent := current.Commit.Parents[0].Commit
parentHash, err := git.ParseHash(parent)
if err != nil {
return fmt.Errorf("failed to parse parent hash '%v':\n %v", parent, err)
}
postMsg := func(notify, msg string) error {
_, _, err = e.gerrit.Changes.SetReview(change.ChangeID, currentHash.String(), &gerrit.ReviewInput{
Message: msg,
Tag: "autogenerated:perfmon",
Notify: notify,
})
if err != nil {
return fmt.Errorf("failed to post message to gerrit change:\n %v", err)
}
return nil
}
newRun, err := e.benchmarkTintChange(currentHash)
if err != nil {
var ftb errFailedToBuild
if errors.As(err, &ftb) {
return postMsg("OWNER", fmt.Sprintf("patchset %v failed to build", current.Number))
}
return err
}
if _, err := e.tintRepo.Fetch(parent, &git.FetchOptions{Auth: e.cfg.Tint.Auth}); err != nil {
return err
}
parentRun, err := e.benchmarkTintChange(parentHash)
if err != nil {
return err
}
// filters the benchmark results to only the mean aggregate values
meanBenchmarkResults := func(in []bench.Benchmark) []bench.Benchmark {
out := make([]bench.Benchmark, 0, len(in))
for _, b := range in {
if b.AggregateType == bench.Mean {
out = append(out, b)
}
}
return out
}
newResults := meanBenchmarkResults(newRun.Benchmarks)
parentResults := meanBenchmarkResults(parentRun.Benchmarks)
const minDiff = time.Microsecond * 50 // Ignore time diffs less than this duration
const minRelDiff = 0.01 // Ignore absolute relative diffs between [1, 1+x]
diff := bench.Compare(parentResults, newResults, minDiff, minRelDiff)
diffFmt := bench.DiffFormat{
TestName: true,
Delta: true,
PercentChangeAB: true,
TimeA: true,
TimeB: true,
}
msg := &strings.Builder{}
fmt.Fprintf(msg, "Tint perfmon analysis:\n")
fmt.Fprintf(msg, " \n")
fmt.Fprintf(msg, " A: parent change (%v) -> B: patchset %v\n", parent[:7], current.Number)
fmt.Fprintf(msg, " \n")
for _, line := range strings.Split(diff.Format(diffFmt), "\n") {
fmt.Fprintf(msg, " %v\n", line)
}
notify := "OWNER"
if len(diff) > 0 {
notify = "OWNER_REVIEWERS"
}
return postMsg(notify, msg.String())
}
// createOrOpenGitRepo creates a new local repo by cloning cfg.URL into
// filepath, or opens the existing repo at filepath.
func createOrOpenGitRepo(g *git.Git, filepath string, cfg GitConfig) (*git.Repository, error) {
repo, err := g.Open(filepath)
if errors.Is(err, git.ErrRepositoryDoesNotExist) {
log.Printf("cloning '%v' branch '%v' to '%v'...", cfg.URL, cfg.Branch, filepath)
repo, err = g.Clone(filepath, cfg.URL, &git.CloneOptions{
Branch: cfg.Branch,
Auth: cfg.Auth,
})
}
if err != nil {
return nil, fmt.Errorf("failed to open git repository '%v':\n %w", filepath, err)
}
return repo, err
}
// loadConfig loads the perfmon config file.
func loadConfig(path string) (Config, error) {
f, err := os.Open(path)
if err != nil {
return Config{}, fmt.Errorf("failed to open config file at '%v':\n %w", path, err)
}
cfg := Config{}
if err := json.NewDecoder(f).Decode(&cfg); err != nil {
return Config{}, fmt.Errorf("failed to load config file at '%v':\n %w", path, err)
}
cfg.setDefaults()
return cfg, nil
}
// makeWorkingDirs builds the tint repo and results repo directories.
func makeWorkingDirs(cfg Config) (tintDir, resultsDir string, err error) {
wd, err := expandHomeDir(cfg.WorkingDir)
if err != nil {
return "", "", err
}
if err := os.MkdirAll(wd, 0777); err != nil {
return "", "", fmt.Errorf("failed to create working directory '%v':\n %w", wd, err)
}
tintDir = filepath.Join(wd, "tint")
if err := os.MkdirAll(tintDir, 0777); err != nil {
return "", "", fmt.Errorf("failed to create working tint directory '%v':\n %w", tintDir, err)
}
resultsDir = filepath.Join(wd, "results")
if err := os.MkdirAll(resultsDir, 0777); err != nil {
return "", "", fmt.Errorf("failed to create working results directory '%v':\n %w", resultsDir, err)
}
return tintDir, resultsDir, nil
}
// fetchAndCheckoutLatest calls fetch(cfg.Branch) followed by checkoutLatest().
func fetchAndCheckoutLatest(repo *git.Repository, cfg GitConfig) error {
hash, err := repo.Fetch(cfg.Branch, &git.FetchOptions{Auth: cfg.Auth})
if err != nil {
return err
}
if err := repo.Checkout(hash.String(), nil); err != nil {
return err
}
return checkout(hash, repo)
}
// checkout checks out the change with the given hash.
// Note: call fetch() to ensure that this is the latest change on the
// branch.
func checkout(hash git.Hash, repo *git.Repository) error {
if err := repo.Checkout(hash.String(), nil); err != nil {
return fmt.Errorf("failed to checkout '%v':\n %w", hash, err)
}
return nil
}
// expandHomeDir returns path with all occurrences of '~' replaced with the user
// home directory.
func expandHomeDir(path string) (string, error) {
if strings.ContainsRune(path, '~') {
home, err := os.UserHomeDir()
if err != nil {
return "", fmt.Errorf("failed to expand home dir:\n %w", err)
}
path = strings.ReplaceAll(path, "~", home)
}
return path, nil
}
// tools holds the file paths to the executables used by this tool
var tools struct {
ccache string
cmake string
gclient string
git string
ninja string
}
// findTools looks for the file paths for executables used by this tool,
// returning an error if any could not be found.
func findTools() error {
for _, tool := range []struct {
name string
path *string
}{
{"ccache", &tools.ccache},
{"cmake", &tools.cmake},
{"gclient", &tools.gclient},
{"git", &tools.git},
{"ninja", &tools.ninja},
} {
path, err := exec.LookPath(tool.name)
if err != nil {
return fmt.Errorf("failed to find path to '%v':\n %w", tool.name, err)
}
*tool.path = path
}
return nil
}
// copyFile copies the file at srcPath to dstPath.
func copyFile(dstPath, srcPath string) error {
src, err := os.Open(srcPath)
if err != nil {
return fmt.Errorf("failed to open file '%v':\n %w", srcPath, err)
}
defer src.Close()
dst, err := os.Create(dstPath)
if err != nil {
return fmt.Errorf("failed to create file '%v':\n %w", dstPath, err)
}
defer dst.Close()
_, err = io.Copy(dst, src)
return err
}
// call invokes the executable exe in the current working directory wd, with
// the provided arguments.
// If the executable does not complete within the timeout duration, then an
// error is returned.
func call(exe, wd string, timeout time.Duration, args ...string) (string, error) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
cmd := exec.CommandContext(ctx, exe, args...)
cmd.Dir = wd
out, err := cmd.CombinedOutput()
if err != nil {
return string(out), fmt.Errorf("'%v %v' failed:\n %w\n%v", exe, args, err, string(out))
}
return string(out), nil
}
// hash returns a hash of the string representation of 'o'.
func hash(o interface{}) string {
str := fmt.Sprintf("%+v", o)
hash := sha256.New()
hash.Write([]byte(str))
return hex.EncodeToString(hash.Sum(nil))[:8]
}

40
tools/src/cmd/remote-compile/CMakeLists.txt Normal file
View File

@@ -0,0 +1,40 @@
# Copyright 2021 The Tint Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set(SRC
main.cc
rwmutex.h
socket.cc
socket.h
)
find_package (Threads REQUIRED)
add_executable(tint-remote-compile ${SRC})
target_link_libraries (tint-remote-compile Threads::Threads)
target_include_directories(tint-remote-compile PRIVATE "${TINT_ROOT_SOURCE_DIR}")
# If we're building on mac / ios and we have CoreGraphics, then we can use the
# metal API to validate our shaders. This is roughly 4x faster than invoking
# the metal shader compiler executable.
if(APPLE)
find_library(LIB_CORE_GRAPHICS CoreGraphics)
if(LIB_CORE_GRAPHICS)
target_sources(tint-remote-compile PRIVATE "msl_metal.mm")
target_compile_definitions(tint-remote-compile PRIVATE "-DTINT_ENABLE_MSL_COMPILATION_USING_METAL_API=1")
target_compile_options(tint-remote-compile PRIVATE "-fmodules" "-fcxx-modules")
target_link_options(tint-remote-compile PRIVATE "-framework" "CoreGraphics")
endif()
endif()

30
tools/src/cmd/remote-compile/compile.h Normal file
View File

@@ -0,0 +1,30 @@
// Copyright 2021 The Tint Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef TOOLS_SRC_CMD_REMOTE_COMPILE_COMPILE_H_
#define TOOLS_SRC_CMD_REMOTE_COMPILE_COMPILE_H_
#include <string>
/// The return structure of a compile function
struct CompileResult {
/// True if shader compiled
bool success = false;
/// Output of the compiler
std::string output;
};
CompileResult CompileMslUsingMetalAPI(const std::string& src);
#endif // TOOLS_SRC_CMD_REMOTE_COMPILE_COMPILE_H_

445
tools/src/cmd/remote-compile/main.cc Normal file
View File

@@ -0,0 +1,445 @@
// Copyright 2021 The Tint Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdio.h>
#include <stdlib.h>
#include <fstream>
#include <sstream>
#include <string>
#include <type_traits>
#include <vector>
#include <thread> // NOLINT
#include "tools/src/cmd/remote-compile/compile.h"
#include "tools/src/cmd/remote-compile/socket.h"
namespace {
#if 0
#define DEBUG(msg, ...) printf(msg "\n", ##__VA_ARGS__)
#else
#define DEBUG(...)
#endif
/// Print the tool usage, and exit with 1.
void ShowUsage() {
const char* name = "tint-remote-compile";
printf(R"(%s is a tool for compiling a shader on a remote machine
usage as server:
%s -s [-p port-number]
usage as client:
%s [-p port-number] [server-address] shader-file-path
[server-address] can be omitted if the TINT_REMOTE_COMPILE_ADDRESS environment
variable is set.
Alternatively, you can pass xcrun arguments so %s can be used as a
drop-in replacement.
)",
name, name, name, name);
exit(1);
}
/// The protocol version code. Bump each time the protocol changes
constexpr uint32_t kProtocolVersion = 1;
/// Supported shader source languages
enum SourceLanguage {
MSL,
};
/// Stream is a serialization wrapper around a socket
struct Stream {
/// The underlying socket
Socket* const socket;
/// Error state
std::string error;
/// Writes a uint32_t to the socket
Stream operator<<(uint32_t v) {
if (error.empty()) {
Write(&v, sizeof(v));
}
return *this;
}
/// Reads a uint32_t from the socket
Stream operator>>(uint32_t& v) {
if (error.empty()) {
Read(&v, sizeof(v));
}
return *this;
}
/// Writes a std::string to the socket
Stream operator<<(const std::string& v) {
if (error.empty()) {
uint32_t count = static_cast<uint32_t>(v.size());
*this << count;
if (count) {
Write(v.data(), count);
}
}
return *this;
}
/// Reads a std::string from the socket
Stream operator>>(std::string& v) {
uint32_t count = 0;
*this >> count;
if (count) {
std::vector<char> buf(count);
if (Read(buf.data(), count)) {
v = std::string(buf.data(), buf.size());
}
} else {
v.clear();
}
return *this;
}
/// Writes an enum value to the socket
template <typename T>
std::enable_if_t<std::is_enum<T>::value, Stream> operator<<(T e) {
return *this << static_cast<uint32_t>(e);
}
/// Reads an enum value from the socket
template <typename T>
std::enable_if_t<std::is_enum<T>::value, Stream> operator>>(T& e) {
uint32_t v;
*this >> v;
e = static_cast<T>(v);
return *this;
}
private:
bool Write(const void* data, size_t size) {
if (error.empty()) {
if (!socket->Write(data, size)) {
error = "Socket::Write() failed";
}
}
return error.empty();
}
bool Read(void* data, size_t size) {
auto buf = reinterpret_cast<uint8_t*>(data);
while (size > 0 && error.empty()) {
if (auto n = socket->Read(buf, size)) {
if (n > size) {
error = "Socket::Read() returned more bytes than requested";
return false;
}
size -= n;
buf += n;
}
}
return error.empty();
}
};
////////////////////////////////////////////////////////////////////////////////
// Messages
////////////////////////////////////////////////////////////////////////////////
/// Base class for all messages
struct Message {
/// The type of the message
enum class Type {
ConnectionRequest,
ConnectionResponse,
CompileRequest,
CompileResponse,
};
explicit Message(Type ty) : type(ty) {}
const Type type;
};
struct ConnectionResponse : Message { // Server -> Client
ConnectionResponse() : Message(Type::ConnectionResponse) {}
template <typename T>
void Serialize(T&& f) {
f(error);
}
std::string error;
};
struct ConnectionRequest : Message { // Client -> Server
using Response = ConnectionResponse;
explicit ConnectionRequest(uint32_t proto_ver = kProtocolVersion)
: Message(Type::ConnectionRequest), protocol_version(proto_ver) {}
template <typename T>
void Serialize(T&& f) {
f(protocol_version);
}
uint32_t protocol_version;
};
struct CompileResponse : Message { // Server -> Client
CompileResponse() : Message(Type::CompileResponse) {}
template <typename T>
void Serialize(T&& f) {
f(error);
}
std::string error;
};
struct CompileRequest : Message { // Client -> Server
using Response = CompileResponse;
CompileRequest() : Message(Type::CompileRequest) {}
CompileRequest(SourceLanguage lang, std::string src)
: Message(Type::CompileRequest), language(lang), source(src) {}
template <typename T>
void Serialize(T&& f) {
f(language);
f(source);
}
SourceLanguage language;
std::string source;
};
/// Writes the message `m` to the stream `s`
template <typename MESSAGE>
std::enable_if_t<std::is_base_of<Message, MESSAGE>::value, Stream>& operator<<(
Stream& s,
const MESSAGE& m) {
s << m.type;
const_cast<MESSAGE&>(m).Serialize([&s](const auto& value) { s << value; });
return s;
}
/// Reads the message `m` from the stream `s`
template <typename MESSAGE>
std::enable_if_t<std::is_base_of<Message, MESSAGE>::value, Stream>& operator>>(
Stream& s,
MESSAGE& m) {
Message::Type ty;
s >> ty;
if (ty == m.type) {
m.Serialize([&s](auto& value) { s >> value; });
} else {
std::stringstream ss;
ss << "expected message type " << static_cast<int>(m.type) << ", got "
<< static_cast<int>(ty);
s.error = ss.str();
}
return s;
}
/// Writes the request message `req` to the stream `s`, then reads and returns
/// the response message from the same stream.
template <typename REQUEST, typename RESPONSE = typename REQUEST::Response>
RESPONSE Send(Stream& s, const REQUEST& req) {
s << req;
if (s.error.empty()) {
RESPONSE resp;
s >> resp;
if (s.error.empty()) {
return resp;
}
}
return {};
}
} // namespace
bool RunServer(std::string port);
bool RunClient(std::string address, std::string port, std::string file);
int main(int argc, char* argv[]) {
bool run_server = false;
std::string port = "19000";
std::vector<std::string> args;
for (int i = 1; i < argc; i++) {
std::string arg = argv[i];
if (arg == "-s" || arg == "--server") {
run_server = true;
continue;
}
if (arg == "-p" || arg == "--port") {
if (i < argc - 1) {
i++;
port = argv[i];
} else {
printf("expected port number");
exit(1);
}
continue;
}
// xcrun flags are ignored so this executable can be used as a replacement
// for xcrun.
if ((arg == "-x" || arg == "-sdk") && (i < argc - 1)) {
i++;
continue;
}
if (arg == "metal") {
for (; i < argc; i++) {
if (std::string(argv[i]) == "-c") {
break;
}
}
continue;
}
args.emplace_back(arg);
}
bool success = false;
if (run_server) {
success = RunServer(port);
} else {
std::string address;
std::string file;
switch (args.size()) {
case 1:
if (auto* addr = getenv("TINT_REMOTE_COMPILE_ADDRESS")) {
address = addr;
}
file = args[0];
break;
case 2:
address = args[0];
file = args[1];
break;
}
if (address.empty() || file.empty()) {
ShowUsage();
}
success = RunClient(address, port, file);
}
if (!success) {
exit(1);
}
return 0;
}
bool RunServer(std::string port) {
auto server_socket = Socket::Listen("", port.c_str());
if (!server_socket) {
printf("Failed to listen on port %s\n", port.c_str());
return false;
}
printf("Listening on port %s...\n", port.c_str());
while (auto conn = server_socket->Accept()) {
std::thread([=] {
DEBUG("Client connected...");
Stream stream{conn.get()};
{
ConnectionRequest req;
stream >> req;
if (!stream.error.empty()) {
printf("%s\n", stream.error.c_str());
return;
}
ConnectionResponse resp;
if (req.protocol_version != kProtocolVersion) {
DEBUG("Protocol version mismatch");
resp.error = "Protocol version mismatch";
stream << resp;
return;
}
stream << resp;
}
DEBUG("Connection established");
{
CompileRequest req;
stream >> req;
if (!stream.error.empty()) {
printf("%s\n", stream.error.c_str());
return;
}
#ifdef TINT_ENABLE_MSL_COMPILATION_USING_METAL_API
if (req.language == SourceLanguage::MSL) {
auto result = CompileMslUsingMetalAPI(req.source);
CompileResponse resp;
if (!result.success) {
resp.error = result.output;
}
stream << resp;
return;
}
#endif
CompileResponse resp;
resp.error = "server cannot compile this type of shader";
stream << resp;
}
}).detach();
}
return true;
}
bool RunClient(std::string address, std::string port, std::string file) {
// Read the file
std::ifstream input(file, std::ios::binary);
if (!input) {
printf("Couldn't open '%s'\n", file.c_str());
return false;
}
std::string source((std::istreambuf_iterator<char>(input)),
std::istreambuf_iterator<char>());
constexpr const int timeout_ms = 10000;
DEBUG("Connecting to %s:%s...", address.c_str(), port.c_str());
auto conn = Socket::Connect(address.c_str(), port.c_str(), timeout_ms);
if (!conn) {
printf("Connection failed\n");
return false;
}
Stream stream{conn.get()};
DEBUG("Sending connection request...");
auto conn_resp = Send(stream, ConnectionRequest{kProtocolVersion});
if (!stream.error.empty()) {
printf("%s\n", stream.error.c_str());
return false;
}
if (!conn_resp.error.empty()) {
printf("%s\n", conn_resp.error.c_str());
return false;
}
DEBUG("Connection established. Requesting compile...");
auto comp_resp = Send(stream, CompileRequest{SourceLanguage::MSL, source});
if (!stream.error.empty()) {
printf("%s\n", stream.error.c_str());
return false;
}
if (!comp_resp.error.empty()) {
printf("%s\n", comp_resp.error.c_str());
return false;
}
DEBUG("Compilation successful");
return true;
}

56
tools/src/cmd/remote-compile/msl_metal.mm Normal file
View File

@@ -0,0 +1,56 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifdef TINT_ENABLE_MSL_COMPILATION_USING_METAL_API
@import Metal;
// Disable: error: treating #include as an import of module 'std.string'
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wauto-import"
#include "compile.h"
#pragma clang diagnostic pop
CompileResult CompileMslUsingMetalAPI(const std::string& src) {
CompileResult result;
result.success = false;
NSError* error = nil;
id<MTLDevice> device = MTLCreateSystemDefaultDevice();
if (!device) {
result.output = "MTLCreateSystemDefaultDevice returned null";
result.success = false;
return result;
}
NSString* source = [NSString stringWithCString:src.c_str()
encoding:NSUTF8StringEncoding];
MTLCompileOptions* compileOptions = [MTLCompileOptions new];
compileOptions.languageVersion = MTLLanguageVersion1_2;
id<MTLLibrary> library = [device newLibraryWithSource:source
options:compileOptions
error:&error];
if (!library) {
NSString* output = [error localizedDescription];
result.output = [output UTF8String];
result.success = false;
}
return result;
}
#endif

190
tools/src/cmd/remote-compile/rwmutex.h Normal file
View File

@@ -0,0 +1,190 @@
// Copyright 2020 The Tint Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef TOOLS_SRC_CMD_REMOTE_COMPILE_RWMUTEX_H_
#define TOOLS_SRC_CMD_REMOTE_COMPILE_RWMUTEX_H_
#include <condition_variable> // NOLINT
#include <mutex> // NOLINT
////////////////////////////////////////////////////////////////////////////////
// RWMutex
////////////////////////////////////////////////////////////////////////////////
/// A RWMutex is a reader/writer mutual exclusion lock.
/// The lock can be held by an arbitrary number of readers or a single writer.
/// Also known as a shared mutex.
class RWMutex {
public:
inline RWMutex() = default;
/// lockReader() locks the mutex for reading.
/// Multiple read locks can be held while there are no writer locks.
inline void lockReader();
/// unlockReader() unlocks the mutex for reading.
inline void unlockReader();
/// lockWriter() locks the mutex for writing.
/// If the lock is already locked for reading or writing, lockWriter blocks
/// until the lock is available.
inline void lockWriter();
/// unlockWriter() unlocks the mutex for writing.
inline void unlockWriter();
private:
RWMutex(const RWMutex&) = delete;
RWMutex& operator=(const RWMutex&) = delete;
int readLocks = 0;
int pendingWriteLocks = 0;
std::mutex mutex;
std::condition_variable cv;
};
void RWMutex::lockReader() {
std::unique_lock<std::mutex> lock(mutex);
readLocks++;
}
void RWMutex::unlockReader() {
std::unique_lock<std::mutex> lock(mutex);
readLocks--;
if (readLocks == 0 && pendingWriteLocks > 0) {
cv.notify_one();
}
}
void RWMutex::lockWriter() {
std::unique_lock<std::mutex> lock(mutex);
if (readLocks > 0) {
pendingWriteLocks++;
cv.wait(lock, [&] { return readLocks == 0; });
pendingWriteLocks--;
}
lock.release(); // Keep lock held
}
void RWMutex::unlockWriter() {
if (pendingWriteLocks > 0) {
cv.notify_one();
}
mutex.unlock();
}
////////////////////////////////////////////////////////////////////////////////
// RLock
////////////////////////////////////////////////////////////////////////////////
/// RLock is a RAII read lock helper for a RWMutex.
class RLock {
public:
/// Constructor.
/// Locks `mutex` with a read-lock for the lifetime of the WLock.
/// @param mutex the mutex
explicit inline RLock(RWMutex& mutex);
/// Destructor.
/// Unlocks the RWMutex.
inline ~RLock();
/// Move constructor
/// @param other the other RLock to move into this RLock.
inline RLock(RLock&& other);
/// Move assignment operator
/// @param other the other RLock to move into this RLock.
/// @returns this RLock so calls can be chained
inline RLock& operator=(RLock&& other);
private:
RLock(const RLock&) = delete;
RLock& operator=(const RLock&) = delete;
RWMutex* m;
};
RLock::RLock(RWMutex& mutex) : m(&mutex) {
m->lockReader();
}
RLock::~RLock() {
if (m != nullptr) {
m->unlockReader();
}
}
RLock::RLock(RLock&& other) {
m = other.m;
other.m = nullptr;
}
RLock& RLock::operator=(RLock&& other) {
m = other.m;
other.m = nullptr;
return *this;
}
////////////////////////////////////////////////////////////////////////////////
// WLock
////////////////////////////////////////////////////////////////////////////////
/// WLock is a RAII write lock helper for a RWMutex.
class WLock {
public:
/// Constructor.
/// Locks `mutex` with a write-lock for the lifetime of the WLock.
/// @param mutex the mutex
explicit inline WLock(RWMutex& mutex);
/// Destructor.
/// Unlocks the RWMutex.
inline ~WLock();
/// Move constructor
/// @param other the other WLock to move into this WLock.
inline WLock(WLock&& other);
/// Move assignment operator
/// @param other the other WLock to move into this WLock.
/// @returns this WLock so calls can be chained
inline WLock& operator=(WLock&& other);
private:
WLock(const WLock&) = delete;
WLock& operator=(const WLock&) = delete;
RWMutex* m;
};
WLock::WLock(RWMutex& mutex) : m(&mutex) {
m->lockWriter();
}
WLock::~WLock() {
if (m != nullptr) {
m->unlockWriter();
}
}
WLock::WLock(WLock&& other) {
m = other.m;
other.m = nullptr;
}
WLock& WLock::operator=(WLock&& other) {
m = other.m;
other.m = nullptr;
return *this;
}
#endif // TOOLS_SRC_CMD_REMOTE_COMPILE_RWMUTEX_H_

310
tools/src/cmd/remote-compile/socket.cc Normal file
View File

@@ -0,0 +1,310 @@
// Copyright 2021 The Tint Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "tools/src/cmd/remote-compile/socket.h"
#include "tools/src/cmd/remote-compile/rwmutex.h"
#if defined(_WIN32)
#include <winsock2.h>
#include <ws2tcpip.h>
#else
#include <netdb.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <sys/select.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <unistd.h>
#endif
#if defined(_WIN32)
#include <atomic>
namespace {
std::atomic<int> wsaInitCount = {0};
} // anonymous namespace
#else
#include <fcntl.h>
namespace {
using SOCKET = int;
} // anonymous namespace
#endif
namespace {
constexpr SOCKET InvalidSocket = static_cast<SOCKET>(-1);
void init() {
#if defined(_WIN32)
if (wsaInitCount++ == 0) {
WSADATA winsockData;
(void)WSAStartup(MAKEWORD(2, 2), &winsockData);
}
#endif
}
void term() {
#if defined(_WIN32)
if (--wsaInitCount == 0) {
WSACleanup();
}
#endif
}
bool setBlocking(SOCKET s, bool blocking) {
#if defined(_WIN32)
u_long mode = blocking ? 0 : 1;
return ioctlsocket(s, FIONBIO, &mode) == NO_ERROR;
#else
auto arg = fcntl(s, F_GETFL, nullptr);
if (arg < 0) {
return false;
}
arg = blocking ? (arg & ~O_NONBLOCK) : (arg | O_NONBLOCK);
return fcntl(s, F_SETFL, arg) >= 0;
#endif
}
bool errored(SOCKET s) {
if (s == InvalidSocket) {
return true;
}
char error = 0;
socklen_t len = sizeof(error);
getsockopt(s, SOL_SOCKET, SO_ERROR, &error, &len);
return error != 0;
}
class Impl : public Socket {
public:
static std::shared_ptr<Impl> create(const char* address, const char* port) {
init();
addrinfo hints = {};
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
hints.ai_protocol = IPPROTO_TCP;
hints.ai_flags = AI_PASSIVE;
addrinfo* info = nullptr;
auto err = getaddrinfo(address, port, &hints, &info);
#if !defined(_WIN32)
if (err) {
printf("getaddrinfo(%s, %s) error: %s\n", address, port,
gai_strerror(err));
}
#endif
if (info) {
auto socket =
::socket(info->ai_family, info->ai_socktype, info->ai_protocol);
auto out = std::make_shared<Impl>(info, socket);
out->setOptions();
return out;
}
freeaddrinfo(info);
term();
return nullptr;
}
explicit Impl(SOCKET socket) : info(nullptr), s(socket) {}
Impl(addrinfo* info, SOCKET socket) : info(info), s(socket) {}
~Impl() {
freeaddrinfo(info);
Close();
term();
}
template <typename FUNCTION>
void lock(FUNCTION&& f) {
RLock l(mutex);
f(s, info);
}
void setOptions() {
RLock l(mutex);
if (s == InvalidSocket) {
return;
}
int enable = 1;
#if !defined(_WIN32)
// Prevent sockets lingering after process termination, causing
// reconnection issues on the same port.
setsockopt(s, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast<char*>(&enable),
sizeof(enable));
struct {
int l_onoff; /* linger active */
int l_linger; /* how many seconds to linger for */
} linger = {false, 0};
setsockopt(s, SOL_SOCKET, SO_LINGER, reinterpret_cast<char*>(&linger),
sizeof(linger));
#endif // !defined(_WIN32)
// Enable TCP_NODELAY.
setsockopt(s, IPPROTO_TCP, TCP_NODELAY, reinterpret_cast<char*>(&enable),
sizeof(enable));
}
bool IsOpen() override {
{
RLock l(mutex);
if ((s != InvalidSocket) && !errored(s)) {
return true;
}
}
WLock lock(mutex);
s = InvalidSocket;
return false;
}
void Close() override {
{
RLock l(mutex);
if (s != InvalidSocket) {
#if defined(_WIN32)
closesocket(s);
#else
::shutdown(s, SHUT_RDWR);
#endif
}
}
WLock l(mutex);
if (s != InvalidSocket) {
#if !defined(_WIN32)
::close(s);
#endif
s = InvalidSocket;
}
}
size_t Read(void* buffer, size_t bytes) override {
RLock lock(mutex);
if (s == InvalidSocket) {
return 0;
}
auto len =
recv(s, reinterpret_cast<char*>(buffer), static_cast<int>(bytes), 0);
return (len < 0) ? 0 : len;
}
bool Write(const void* buffer, size_t bytes) override {
RLock lock(mutex);
if (s == InvalidSocket) {
return false;
}
if (bytes == 0) {
return true;
}
return ::send(s, reinterpret_cast<const char*>(buffer),
static_cast<int>(bytes), 0) > 0;
}
std::shared_ptr<Socket> Accept() override {
std::shared_ptr<Impl> out;
lock([&](SOCKET socket, const addrinfo*) {
if (socket != InvalidSocket) {
init();
out = std::make_shared<Impl>(::accept(socket, 0, 0));
out->setOptions();
}
});
return out;
}
private:
addrinfo* const info;
SOCKET s = InvalidSocket;
RWMutex mutex;
};
} // anonymous namespace
std::shared_ptr<Socket> Socket::Listen(const char* address, const char* port) {
auto impl = Impl::create(address, port);
if (!impl) {
return nullptr;
}
impl->lock([&](SOCKET socket, const addrinfo* info) {
if (bind(socket, info->ai_addr, static_cast<int>(info->ai_addrlen)) != 0) {
impl.reset();
return;
}
if (listen(socket, 0) != 0) {
impl.reset();
return;
}
});
return impl;
}
std::shared_ptr<Socket> Socket::Connect(const char* address,
const char* port,
uint32_t timeoutMillis) {
auto impl = Impl::create(address, port);
if (!impl) {
return nullptr;
}
std::shared_ptr<Socket> out;
impl->lock([&](SOCKET socket, const addrinfo* info) {
if (socket == InvalidSocket) {
return;
}
if (timeoutMillis == 0) {
if (::connect(socket, info->ai_addr,
static_cast<int>(info->ai_addrlen)) == 0) {
out = impl;
}
return;
}
if (!setBlocking(socket, false)) {
return;
}
auto res =
::connect(socket, info->ai_addr, static_cast<int>(info->ai_addrlen));
if (res == 0) {
if (setBlocking(socket, true)) {
out = impl;
}
} else {
const auto microseconds = timeoutMillis * 1000;
fd_set fdset;
FD_ZERO(&fdset);
FD_SET(socket, &fdset);
timeval tv;
tv.tv_sec = microseconds / 1000000;
tv.tv_usec = microseconds - static_cast<uint32_t>(tv.tv_sec * 1000000);
res = select(static_cast<int>(socket + 1), nullptr, &fdset, nullptr, &tv);
if (res > 0 && !errored(socket) && setBlocking(socket, true)) {
out = impl;
}
}
});
if (!out) {
return nullptr;
}
return out->IsOpen() ? out : nullptr;
}

71
tools/src/cmd/remote-compile/socket.h Normal file
View File

@@ -0,0 +1,71 @@
// Copyright 2021 The Tint Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef TOOLS_SRC_CMD_REMOTE_COMPILE_SOCKET_H_
#define TOOLS_SRC_CMD_REMOTE_COMPILE_SOCKET_H_
#include <atomic>
#include <memory>
/// Socket provides an OS abstraction to a TCP socket.
class Socket {
public:
/// Connects to the given TCP address and port.
/// @param address the target socket address
/// @param port the target socket port
/// @param timeoutMillis the timeout for the connection attempt.
/// If timeoutMillis is non-zero and no connection was made before
/// timeoutMillis milliseconds, then nullptr is returned.
/// @returns the connected Socket, or nullptr on failure
static std::shared_ptr<Socket> Connect(const char* address,
const char* port,
uint32_t timeoutMillis);
/// Begins listening for connections on the given TCP address and port.
/// Call Accept() on the returned Socket to block and wait for a connection.
/// @param address the socket address to listen on. Use "localhost" for
/// connections from only this machine, or an empty string to allow
/// connections from any incoming address.
/// @param port the socket port to listen on
/// @returns the Socket that listens for connections
static std::shared_ptr<Socket> Listen(const char* address, const char* port);
/// Attempts to read at most `n` bytes into buffer, returning the actual
/// number of bytes read.
/// read() will block until the socket is closed or at least one byte is read.
/// @param buffer the output buffer. Must be at least `n` bytes in size.
/// @param n the maximum number of bytes to read
/// @return the number of bytes read, or 0 if the socket was closed
virtual size_t Read(void* buffer, size_t n) = 0;
/// Writes `n` bytes from buffer into the socket.
/// @param buffer the source data buffer. Must be at least `n` bytes in size.
/// @param n the number of bytes to read from `buffer`
/// @returns true on success, or false if there was an error or the socket was
/// closed.
virtual bool Write(const void* buffer, size_t n) = 0;
/// @returns true if the socket has not been closed.
virtual bool IsOpen() = 0;
/// Closes the socket.
virtual void Close() = 0;
/// Blocks for a connection to be made to the listening port, or for the
/// Socket to be closed.
/// @returns a pointer to the next established incoming connection
virtual std::shared_ptr<Socket> Accept() = 0;
};
#endif // TOOLS_SRC_CMD_REMOTE_COMPILE_SOCKET_H_

133
tools/src/cmd/run-parallel/main.go Normal file
View File

@@ -0,0 +1,133 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// run-parallel is a tool to run an executable with the provided templated
// arguments across all the hardware threads.
package main
import (
"flag"
"fmt"
"os"
"os/exec"
"runtime"
"strings"
"sync"
)
func main() {
if err := run(); err != nil {
fmt.Println(err)
os.Exit(1)
}
}
func showUsage() {
fmt.Println(`
run-parallel is a tool to run an executable with the provided templated
arguments across all the hardware threads.
Usage:
run-parallel <executable> [arguments...] -- [per-instance-value...]
executable - the path to the executable to run.
arguments - a list of arguments to pass to the executable.
Any occurrance of $ will be substituted with the
per-instance-value for the given invocation.
per-instance-value - a list of values. The executable will be invoked for each
value in this list.`)
os.Exit(1)
}
func run() error {
onlyPrintFailures := flag.Bool("only-print-failures", false, "Omit output for processes that did not fail")
flag.Parse()
args := flag.Args()
if len(args) < 2 {
showUsage()
}
exe := args[0]
args = args[1:]
var perInstanceValues []string
for i, arg := range args {
if arg == "--" {
perInstanceValues = args[i+1:]
args = args[:i]
break
}
}
if perInstanceValues == nil {
showUsage()
}
taskIndices := make(chan int, 64)
type result struct {
msg string
failed bool
}
results := make([]result, len(perInstanceValues))
numCPU := runtime.NumCPU()
wg := sync.WaitGroup{}
wg.Add(numCPU)
for i := 0; i < numCPU; i++ {
go func() {
defer wg.Done()
for idx := range taskIndices {
taskArgs := make([]string, len(args))
for i, arg := range args {
taskArgs[i] = strings.ReplaceAll(arg, "$", perInstanceValues[idx])
}
success, out := invoke(exe, taskArgs)
if !success || !*onlyPrintFailures {
results[idx] = result{out, !success}
}
}
}()
}
for i := range perInstanceValues {
taskIndices <- i
}
close(taskIndices)
wg.Wait()
failed := false
for _, result := range results {
if result.msg != "" {
fmt.Println(result.msg)
}
failed = failed || result.failed
}
if failed {
os.Exit(1)
}
return nil
}
func invoke(exe string, args []string) (ok bool, output string) {
cmd := exec.Command(exe, args...)
out, err := cmd.CombinedOutput()
str := string(out)
if err != nil {
if str != "" {
return false, str
}
return false, err.Error()
}
return true, str
}

112
tools/src/cmd/snippets/main.go Normal file
View File

@@ -0,0 +1,112 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// snippets gathers information about changes merged for weekly reports (snippets).
package main
import (
"flag"
"fmt"
"os"
"strings"
"time"
"dawn.googlesource.com/tint/tools/src/gerrit"
)
const yyyymmdd = "2006-01-02"
var (
// See https://dawn-review.googlesource.com/new-password for obtaining
// username and password for gerrit.
gerritUser = flag.String("gerrit-user", "", "gerrit authentication username")
gerritPass = flag.String("gerrit-pass", "", "gerrit authentication password")
userFlag = flag.String("user", "", "user name / email")
afterFlag = flag.String("after", "", "start date")
beforeFlag = flag.String("before", "", "end date")
daysFlag = flag.Int("days", 7, "interval in days (used if --after is not specified)")
)
func main() {
flag.Parse()
if err := run(); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
func run() error {
var after, before time.Time
var err error
user := *userFlag
if user == "" {
return fmt.Errorf("Missing required 'user' flag")
}
if *beforeFlag != "" {
before, err = time.Parse(yyyymmdd, *beforeFlag)
if err != nil {
return fmt.Errorf("Couldn't parse before date: %w", err)
}
} else {
before = time.Now()
}
if *afterFlag != "" {
after, err = time.Parse(yyyymmdd, *afterFlag)
if err != nil {
return fmt.Errorf("Couldn't parse after date: %w", err)
}
} else {
after = before.Add(-time.Hour * time.Duration(24**daysFlag))
}
g, err := gerrit.New(gerrit.Config{Username: *gerritUser, Password: *gerritPass})
if err != nil {
return err
}
submitted, _, err := g.QueryChanges(
"status:merged",
"owner:"+user,
"after:"+date(after),
"before:"+date(before))
if err != nil {
return fmt.Errorf("Query failed: %w", err)
}
changesByProject := map[string][]string{}
for _, change := range submitted {
str := fmt.Sprintf(`* [%s](%sc/%s/+/%d)`, change.Subject, gerrit.URL, change.Project, change.Number)
changesByProject[change.Project] = append(changesByProject[change.Project], str)
}
for _, project := range []string{"tint", "dawn"} {
if changes := changesByProject[project]; len(changes) > 0 {
fmt.Println("##", strings.Title(project))
for _, change := range changes {
fmt.Println(change)
}
fmt.Println()
}
}
return nil
}
func today() time.Time {
return time.Now()
}
func date(t time.Time) string {
return t.Format(yyyymmdd)
}

796
tools/src/cmd/test-runner/main.go Normal file
View File

@@ -0,0 +1,796 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// test-runner runs tint against a number of test shaders checking for expected behavior
package main
import (
"context"
"flag"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"regexp"
"runtime"
"sort"
"strings"
"time"
"unicode/utf8"
"dawn.googlesource.com/tint/tools/src/fileutils"
"dawn.googlesource.com/tint/tools/src/glob"
"github.com/fatih/color"
"github.com/sergi/go-diff/diffmatchpatch"
)
type outputFormat string
const (
testTimeout = 30 * time.Second
glsl = outputFormat("glsl")
hlsl = outputFormat("hlsl")
msl = outputFormat("msl")
spvasm = outputFormat("spvasm")
wgsl = outputFormat("wgsl")
)
// Directories we don't generate expected PASS result files for.
// These directories contain large corpora of tests for which the generated code
// is uninteresting.
var dirsWithNoPassExpectations = []string{
"/test/tint/benchmark/",
"/test/tint/unittest/",
"/test/tint/vk-gl-cts/",
}
func main() {
if err := run(); err != nil {
fmt.Println(err)
os.Exit(1)
}
}
func showUsage() {
fmt.Println(`
test-runner runs tint against a number of test shaders checking for expected behavior
usage:
test-runner [flags...] <executable> [<directory>]
<executable> the path to the tint executable
<directory> the root directory of the test files
optional flags:`)
flag.PrintDefaults()
fmt.Println(``)
os.Exit(1)
}
func run() error {
var formatList, filter, dxcPath, xcrunPath string
var maxFilenameColumnWidth int
numCPU := runtime.NumCPU()
fxc, fxcAndDxc, verbose, generateExpected, generateSkip := false, false, false, false, false
flag.StringVar(&formatList, "format", "all", "comma separated list of formats to emit. Possible values are: all, wgsl, spvasm, msl, hlsl, glsl")
flag.StringVar(&filter, "filter", "**.wgsl, **.spvasm, **.spv", "comma separated list of glob patterns for test files")
flag.StringVar(&dxcPath, "dxc", "", "path to DXC executable for validating HLSL output")
flag.StringVar(&xcrunPath, "xcrun", "", "path to xcrun executable for validating MSL output")
flag.BoolVar(&fxc, "fxc", false, "validate with FXC instead of DXC")
flag.BoolVar(&fxcAndDxc, "fxc-and-dxc", false, "validate with both FXC and DXC")
flag.BoolVar(&verbose, "verbose", false, "print all run tests, including rows that all pass")
flag.BoolVar(&generateExpected, "generate-expected", false, "create or update all expected outputs")
flag.BoolVar(&generateSkip, "generate-skip", false, "create or update all expected outputs that fail with SKIP")
flag.IntVar(&numCPU, "j", numCPU, "maximum number of concurrent threads to run tests")
flag.IntVar(&maxFilenameColumnWidth, "filename-column-width", 0, "maximum width of the filename column")
flag.Usage = showUsage
flag.Parse()
args := flag.Args()
if len(args) == 0 {
showUsage()
}
if fxcAndDxc {
fxc = true
}
// executable path is the first argument
exe, args := args[0], args[1:]
// (optional) target directory is the second argument
dir := "."
if len(args) > 0 {
dir, args = args[0], args[1:]
}
// Check the executable can be found and actually is executable
if !fileutils.IsExe(exe) {
return fmt.Errorf("'%s' not found or is not executable", exe)
}
exe, err := filepath.Abs(exe)
if err != nil {
return err
}
// Allow using '/' in the filter on Windows
filter = strings.ReplaceAll(filter, "/", string(filepath.Separator))
// Split the --filter flag up by ',', trimming any whitespace at the start and end
globIncludes := strings.Split(filter, ",")
for i, s := range globIncludes {
s = filepath.ToSlash(s) // Replace '\' with '/'
globIncludes[i] = `"` + strings.TrimSpace(s) + `"`
}
// Glob the files to test
files, err := glob.Scan(dir, glob.MustParseConfig(`{
"paths": [
{
"include": [ `+strings.Join(globIncludes, ",")+` ]
},
{
"exclude": [
"**.expected.wgsl",
"**.expected.spvasm",
"**.expected.msl",
"**.expected.hlsl",
"**.expected.glsl"
]
}
]
}`))
if err != nil {
return fmt.Errorf("Failed to glob files: %w", err)
}
// Ensure the files are sorted (globbing should do this, but why not)
sort.Strings(files)
// Parse --format into a list of outputFormat
formats := []outputFormat{}
if formatList == "all" {
formats = []outputFormat{wgsl, spvasm, msl, hlsl, glsl}
} else {
for _, f := range strings.Split(formatList, ",") {
switch strings.TrimSpace(f) {
case "wgsl":
formats = append(formats, wgsl)
case "spvasm":
formats = append(formats, spvasm)
case "msl":
formats = append(formats, msl)
case "hlsl":
formats = append(formats, hlsl)
case "glsl":
formats = append(formats, glsl)
default:
return fmt.Errorf("unknown format '%s'", f)
}
}
}
defaultMSLExe := "xcrun"
if runtime.GOOS == "windows" {
defaultMSLExe = "metal.exe"
}
// If explicit verification compilers have been specified, check they exist.
// Otherwise, look on PATH for them, but don't error if they cannot be found.
for _, tool := range []struct {
name string
lang string
path *string
}{
{"dxc", "hlsl", &dxcPath},
{defaultMSLExe, "msl", &xcrunPath},
} {
if *tool.path == "" {
p, err := exec.LookPath(tool.name)
if err == nil && fileutils.IsExe(p) {
*tool.path = p
}
} else if !fileutils.IsExe(*tool.path) {
return fmt.Errorf("%v not found at '%v'", tool.name, *tool.path)
}
color.Set(color.FgCyan)
fmt.Printf("%-4s", tool.lang)
color.Unset()
fmt.Printf(" validation ")
if *tool.path != "" || (fxc && tool.lang == "hlsl") {
color.Set(color.FgGreen)
tool_path := *tool.path
if fxc && tool.lang == "hlsl" {
if fxcAndDxc {
tool_path += " AND Tint will use FXC dll in PATH"
} else {
tool_path = "Tint will use FXC dll in PATH"
}
}
fmt.Printf("ENABLED (" + tool_path + ")")
} else {
color.Set(color.FgRed)
fmt.Printf("DISABLED")
}
color.Unset()
fmt.Println()
}
fmt.Println()
// Build the list of results.
// These hold the chans used to report the job results.
results := make([]map[outputFormat]chan status, len(files))
for i := range files {
fileResults := map[outputFormat]chan status{}
for _, format := range formats {
fileResults[format] = make(chan status, 1)
}
results[i] = fileResults
}
pendingJobs := make(chan job, 256)
// Spawn numCPU job runners...
for cpu := 0; cpu < numCPU; cpu++ {
go func() {
for job := range pendingJobs {
job.run(dir, exe, fxc, fxcAndDxc, dxcPath, xcrunPath, generateExpected, generateSkip)
}
}()
}
// Issue the jobs...
go func() {
for i, file := range files { // For each test file...
file := filepath.Join(dir, file)
flags := parseFlags(file)
for _, format := range formats { // For each output format...
pendingJobs <- job{
file: file,
flags: flags,
format: format,
result: results[i][format],
}
}
}
close(pendingJobs)
}()
type failure struct {
file string
format outputFormat
err error
}
type stats struct {
numTests, numPass, numSkip, numFail int
timeTaken time.Duration
}
// Statistics per output format
statsByFmt := map[outputFormat]*stats{}
for _, format := range formats {
statsByFmt[format] = &stats{}
}
// Print the table of file x format and gather per-format stats
failures := []failure{}
filenameColumnWidth := maxStringLen(files)
if maxFilenameColumnWidth > 0 {
filenameColumnWidth = maxFilenameColumnWidth
}
red := color.New(color.FgRed)
green := color.New(color.FgGreen)
yellow := color.New(color.FgYellow)
cyan := color.New(color.FgCyan)
printFormatsHeader := func() {
fmt.Printf(strings.Repeat(" ", filenameColumnWidth))
fmt.Printf(" ┃ ")
for _, format := range formats {
cyan.Printf(alignCenter(format, formatWidth(format)))
fmt.Printf(" │ ")
}
fmt.Println()
}
printHorizontalLine := func() {
fmt.Printf(strings.Repeat("━", filenameColumnWidth))
fmt.Printf("━╋━")
for _, format := range formats {
fmt.Printf(strings.Repeat("━", formatWidth(format)))
fmt.Printf("━┿━")
}
fmt.Println()
}
fmt.Println()
printFormatsHeader()
printHorizontalLine()
for i, file := range files {
results := results[i]
row := &strings.Builder{}
rowAllPassed := true
filenameLength := utf8.RuneCountInString(file)
shortFile := file
if filenameLength > filenameColumnWidth {
shortFile = "..." + file[filenameLength-filenameColumnWidth+3:]
}
fmt.Fprintf(row, alignRight(shortFile, filenameColumnWidth))
fmt.Fprintf(row, " ┃ ")
for _, format := range formats {
columnWidth := formatWidth(format)
result := <-results[format]
stats := statsByFmt[format]
stats.numTests++
stats.timeTaken += result.timeTaken
if err := result.err; err != nil {
failures = append(failures, failure{
file: file, format: format, err: err,
})
}
switch result.code {
case pass:
green.Fprintf(row, alignCenter("PASS", columnWidth))
stats.numPass++
case fail:
red.Fprintf(row, alignCenter("FAIL", columnWidth))
rowAllPassed = false
stats.numFail++
case skip:
yellow.Fprintf(row, alignCenter("SKIP", columnWidth))
rowAllPassed = false
stats.numSkip++
default:
fmt.Fprintf(row, alignCenter(result.code, columnWidth))
rowAllPassed = false
}
fmt.Fprintf(row, " │ ")
}
if verbose || !rowAllPassed {
fmt.Fprintln(color.Output, row)
}
}
printHorizontalLine()
printFormatsHeader()
printHorizontalLine()
printStat := func(col *color.Color, name string, num func(*stats) int) {
row := &strings.Builder{}
anyNonZero := false
for _, format := range formats {
columnWidth := formatWidth(format)
count := num(statsByFmt[format])
if count > 0 {
col.Fprintf(row, alignLeft(count, columnWidth))
anyNonZero = true
} else {
fmt.Fprintf(row, alignLeft(count, columnWidth))
}
fmt.Fprintf(row, " │ ")
}
if !anyNonZero {
return
}
col.Printf(alignRight(name, filenameColumnWidth))
fmt.Printf(" ┃ ")
fmt.Fprintln(color.Output, row)
col.Printf(strings.Repeat(" ", filenameColumnWidth))
fmt.Printf(" ┃ ")
for _, format := range formats {
columnWidth := formatWidth(format)
stats := statsByFmt[format]
count := num(stats)
percent := percentage(count, stats.numTests)
if count > 0 {
col.Print(alignRight(percent, columnWidth))
} else {
fmt.Print(alignRight(percent, columnWidth))
}
fmt.Printf(" │ ")
}
fmt.Println()
}
printStat(green, "PASS", func(s *stats) int { return s.numPass })
printStat(yellow, "SKIP", func(s *stats) int { return s.numSkip })
printStat(red, "FAIL", func(s *stats) int { return s.numFail })
cyan.Printf(alignRight("TIME", filenameColumnWidth))
fmt.Printf(" ┃ ")
for _, format := range formats {
timeTaken := printDuration(statsByFmt[format].timeTaken)
cyan.Printf(alignLeft(timeTaken, formatWidth(format)))
fmt.Printf(" │ ")
}
fmt.Println()
for _, f := range failures {
color.Set(color.FgBlue)
fmt.Printf("%s ", f.file)
color.Set(color.FgCyan)
fmt.Printf("%s ", f.format)
color.Set(color.FgRed)
fmt.Println("FAIL")
color.Unset()
fmt.Println(indent(f.err.Error(), 4))
}
if len(failures) > 0 {
fmt.Println()
}
allStats := stats{}
for _, format := range formats {
stats := statsByFmt[format]
allStats.numTests += stats.numTests
allStats.numPass += stats.numPass
allStats.numSkip += stats.numSkip
allStats.numFail += stats.numFail
}
fmt.Printf("%d tests run", allStats.numTests)
if allStats.numPass > 0 {
fmt.Printf(", ")
color.Set(color.FgGreen)
fmt.Printf("%d tests pass", allStats.numPass)
color.Unset()
} else {
fmt.Printf(", %d tests pass", allStats.numPass)
}
if allStats.numSkip > 0 {
fmt.Printf(", ")
color.Set(color.FgYellow)
fmt.Printf("%d tests skipped", allStats.numSkip)
color.Unset()
} else {
fmt.Printf(", %d tests skipped", allStats.numSkip)
}
if allStats.numFail > 0 {
fmt.Printf(", ")
color.Set(color.FgRed)
fmt.Printf("%d tests failed", allStats.numFail)
color.Unset()
} else {
fmt.Printf(", %d tests failed", allStats.numFail)
}
fmt.Println()
fmt.Println()
if allStats.numFail > 0 {
os.Exit(1)
}
return nil
}
// Structures to hold the results of the tests
type statusCode string
const (
fail statusCode = "FAIL"
pass statusCode = "PASS"
skip statusCode = "SKIP"
)
type status struct {
code statusCode
err error
timeTaken time.Duration
}
type job struct {
file string
flags []string
format outputFormat
result chan status
}
func (j job) run(wd, exe string, fxc, fxcAndDxc bool, dxcPath, xcrunPath string, generateExpected, generateSkip bool) {
j.result <- func() status {
// expectedFilePath is the path to the expected output file for the given test
expectedFilePath := j.file + ".expected." + string(j.format)
// Is there an expected output file? If so, load it.
expected, expectedFileExists := "", false
if content, err := ioutil.ReadFile(expectedFilePath); err == nil {
expected = string(content)
expectedFileExists = true
}
skipped := false
if strings.HasPrefix(expected, "SKIP") { // Special SKIP token
skipped = true
}
expected = strings.ReplaceAll(expected, "\r\n", "\n")
file, err := filepath.Rel(wd, j.file)
if err != nil {
file = j.file
}
// Make relative paths use forward slash separators (on Windows) so that paths in tint
// output match expected output that contain errors
file = strings.ReplaceAll(file, `\`, `/`)
args := []string{
file,
"--format", string(j.format),
}
// Can we validate?
validate := false
switch j.format {
case wgsl:
validate = true
case spvasm, glsl:
args = append(args, "--validate") // spirv-val and glslang are statically linked, always available
validate = true
case hlsl:
// Handled below
case msl:
if xcrunPath != "" {
args = append(args, "--xcrun", xcrunPath)
validate = true
}
}
// Invoke the compiler...
ok := false
var out string
start := time.Now()
if j.format == hlsl {
// If fxcAndDxc is set, run FXC first as it's more likely to fail, then DXC iff FXC succeeded.
if fxc || fxcAndDxc {
validate = true
args_fxc := append(args, "--fxc")
args_fxc = append(args_fxc, j.flags...)
ok, out = invoke(wd, exe, args_fxc...)
}
if dxcPath != "" && (!fxc || (fxcAndDxc && ok)) {
validate = true
args_dxc := append(args, "--dxc", dxcPath)
args_dxc = append(args_dxc, j.flags...)
ok, out = invoke(wd, exe, args_dxc...)
}
// If we didn't run either fxc or dxc validation, run as usual
if !validate {
args = append(args, j.flags...)
ok, out = invoke(wd, exe, args...)
}
} else {
args = append(args, j.flags...)
ok, out = invoke(wd, exe, args...)
}
timeTaken := time.Since(start)
out = strings.ReplaceAll(out, "\r\n", "\n")
matched := expected == "" || expected == out
canEmitPassExpectationFile := true
for _, noPass := range dirsWithNoPassExpectations {
if strings.Contains(j.file, filepath.FromSlash(noPass)) {
canEmitPassExpectationFile = false
break
}
}
saveExpectedFile := func(path string, content string) error {
return ioutil.WriteFile(path, []byte(content), 0666)
}
if ok && generateExpected && (validate || !skipped) {
// User requested to update PASS expectations, and test passed.
if canEmitPassExpectationFile {
saveExpectedFile(expectedFilePath, out)
} else if expectedFileExists {
// Test lives in a directory where we do not want to save PASS
// files, and there already exists an expectation file. Test has
// likely started passing. Delete the old expectation.
os.Remove(expectedFilePath)
}
matched = true // test passed and matched expectations
}
switch {
case ok && matched:
// Test passed
return status{code: pass, timeTaken: timeTaken}
// --- Below this point the test has failed ---
case skipped:
if generateSkip {
saveExpectedFile(expectedFilePath, "SKIP: FAILED\n\n"+out)
}
return status{code: skip, timeTaken: timeTaken}
case !ok:
// Compiler returned non-zero exit code
if generateSkip {
saveExpectedFile(expectedFilePath, "SKIP: FAILED\n\n"+out)
}
err := fmt.Errorf("%s", out)
return status{code: fail, err: err, timeTaken: timeTaken}
default:
// Compiler returned zero exit code, or output was not as expected
if generateSkip {
saveExpectedFile(expectedFilePath, "SKIP: FAILED\n\n"+out)
}
// Expected output did not match
dmp := diffmatchpatch.New()
diff := dmp.DiffPrettyText(dmp.DiffMain(expected, out, true))
err := fmt.Errorf(`Output was not as expected
--------------------------------------------------------------------------------
-- Expected: --
--------------------------------------------------------------------------------
%s
--------------------------------------------------------------------------------
-- Got: --
--------------------------------------------------------------------------------
%s
--------------------------------------------------------------------------------
-- Diff: --
--------------------------------------------------------------------------------
%s`,
expected, out, diff)
return status{code: fail, err: err, timeTaken: timeTaken}
}
}()
}
// indent returns the string 's' indented with 'n' whitespace characters
func indent(s string, n int) string {
tab := strings.Repeat(" ", n)
return tab + strings.ReplaceAll(s, "\n", "\n"+tab)
}
// alignLeft returns the string of 'val' padded so that it is aligned left in
// a column of the given width
func alignLeft(val interface{}, width int) string {
s := fmt.Sprint(val)
padding := width - utf8.RuneCountInString(s)
if padding < 0 {
return s
}
return s + strings.Repeat(" ", padding)
}
// alignCenter returns the string of 'val' padded so that it is centered in a
// column of the given width.
func alignCenter(val interface{}, width int) string {
s := fmt.Sprint(val)
padding := width - utf8.RuneCountInString(s)
if padding < 0 {
return s
}
return strings.Repeat(" ", padding/2) + s + strings.Repeat(" ", (padding+1)/2)
}
// alignRight returns the string of 'val' padded so that it is aligned right in
// a column of the given width
func alignRight(val interface{}, width int) string {
s := fmt.Sprint(val)
padding := width - utf8.RuneCountInString(s)
if padding < 0 {
return s
}
return strings.Repeat(" ", padding) + s
}
// maxStringLen returns the maximum number of runes found in all the strings in
// 'l'
func maxStringLen(l []string) int {
max := 0
for _, s := range l {
if c := utf8.RuneCountInString(s); c > max {
max = c
}
}
return max
}
// formatWidth returns the width in runes for the outputFormat column 'b'
func formatWidth(b outputFormat) int {
const min = 6
c := utf8.RuneCountInString(string(b))
if c < min {
return min
}
return c
}
// percentage returns the percentage of n out of total as a string
func percentage(n, total int) string {
if total == 0 {
return "-"
}
f := float64(n) / float64(total)
return fmt.Sprintf("%.1f%c", f*100.0, '%')
}
// invoke runs the executable 'exe' with the provided arguments.
func invoke(wd, exe string, args ...string) (ok bool, output string) {
ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
defer cancel()
cmd := exec.CommandContext(ctx, exe, args...)
cmd.Dir = wd
out, err := cmd.CombinedOutput()
str := string(out)
if err != nil {
if ctx.Err() == context.DeadlineExceeded {
return false, fmt.Sprintf("test timed out after %v", testTimeout)
}
if str != "" {
return false, str
}
return false, err.Error()
}
return true, str
}
var reFlags = regexp.MustCompile(` *\/\/ *flags:(.*)\n`)
// parseFlags looks for a `// flags:` header at the start of the file with the
// given path, returning each of the space delimited tokens that follow for the
// line
func parseFlags(path string) []string {
content, err := ioutil.ReadFile(path)
if err != nil {
return nil
}
header := strings.SplitN(string(content), "\n", 1)[0]
m := reFlags.FindStringSubmatch(header)
if len(m) != 2 {
return nil
}
return strings.Split(m[1], " ")
}
func printDuration(d time.Duration) string {
sec := int(d.Seconds())
min := int(sec) / 60
hour := min / 60
min -= hour * 60
sec -= min * 60
sb := &strings.Builder{}
if hour > 0 {
fmt.Fprintf(sb, "%dh", hour)
}
if min > 0 {
fmt.Fprintf(sb, "%dm", min)
}
if sec > 0 {
fmt.Fprintf(sb, "%ds", sec)
}
return sb.String()
}

6
tools/src/cmd/trim-includes/config.cfg Normal file
View File

@@ -0,0 +1,6 @@
{
"paths": [
{ "include": [ "src/**.h", "src/**.cc" ] },
{ "exclude": [ "src/**_windows.*", "src/**_other.*" ] }
]
}

268
tools/src/cmd/trim-includes/main.go Normal file
View File

@@ -0,0 +1,268 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// trim-includes is a tool to try removing unnecessary include statements from
// all .cc and .h files in the tint project.
//
// trim-includes removes each #include from each file, then runs the provided
// build script to determine whether the line was necessary. If the include is
// required, it is restored, otherwise it is left deleted.
// After all the #include statements have been tested, the file is
// clang-formatted and git staged.
package main
import (
"flag"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"
"sync"
"time"
"dawn.googlesource.com/tint/tools/src/fileutils"
"dawn.googlesource.com/tint/tools/src/glob"
)
var (
// Path to the build script to run after each attempting to remove each
// #include
buildScript = ""
)
func main() {
if err := run(); err != nil {
fmt.Println(err)
os.Exit(1)
}
}
func showUsage() {
fmt.Println(`
trim-includes is a tool to try removing unnecessary include statements from all
.cc and .h files in the tint project.
trim-includes removes each #include from each file, then runs the provided build
script to determine whether the line was necessary. If the include is required,
it is restored, otherwise it is left deleted.
After all the #include statements have been tested, the file is clang-formatted
and git staged.
Usage:
trim-includes <path-to-build-script>`)
os.Exit(1)
}
func run() error {
flag.Parse()
args := flag.Args()
if len(args) < 1 {
showUsage()
}
var err error
buildScript, err = exec.LookPath(args[0])
if err != nil {
return err
}
buildScript, err = filepath.Abs(buildScript)
if err != nil {
return err
}
cfg, err := glob.LoadConfig("config.cfg")
if err != nil {
return err
}
fmt.Println("Checking the project builds with no changes...")
ok, err := tryBuild()
if err != nil {
return err
}
if !ok {
return fmt.Errorf("Project does not build without edits")
}
fmt.Println("Scanning for files...")
paths, err := glob.Scan(fileutils.ProjectRoot(), cfg)
if err != nil {
return err
}
fmt.Printf("Loading %v source files...\n", len(paths))
files, err := loadFiles(paths)
if err != nil {
return err
}
for fileIdx, file := range files {
fmt.Printf("[%d/%d]: %v\n", fileIdx+1, len(files), file.path)
includeLines := file.includesLineNumbers()
enabled := make(map[int]bool, len(file.lines))
for i := range file.lines {
enabled[i] = true
}
for includeIdx, line := range includeLines {
fmt.Printf(" [%d/%d]: %v", includeIdx+1, len(includeLines), file.lines[line])
enabled[line] = false
if err := file.save(enabled); err != nil {
return err
}
ok, err := tryBuild()
if err != nil {
return err
}
if ok {
fmt.Printf(" removed\n")
// Wait a bit so file timestamps get an opportunity to change.
// Attempting to save too soon after a successful build may
// result in a false-positive build.
time.Sleep(time.Second)
} else {
fmt.Printf(" required\n")
enabled[line] = true
}
}
if err := file.save(enabled); err != nil {
return err
}
if err := file.format(); err != nil {
return err
}
if err := file.stage(); err != nil {
return err
}
}
fmt.Println("Done")
return nil
}
// Attempt to build the project. Returns true on successful build, false if
// there was a build failure.
func tryBuild() (bool, error) {
cmd := exec.Command("sh", "-c", buildScript)
out, err := cmd.CombinedOutput()
switch err := err.(type) {
case nil:
return cmd.ProcessState.Success(), nil
case *exec.ExitError:
return false, nil
default:
return false, fmt.Errorf("Test failed with error: %v\n%v", err, string(out))
}
}
type file struct {
path string
lines []string
}
var includeRE = regexp.MustCompile(`^\s*#include (?:\"([^"]*)\"|:?\<([^"]*)\>)`)
// Returns the file path with the extension stripped
func stripExtension(path string) string {
if dot := strings.IndexRune(path, '.'); dot > 0 {
return path[:dot]
}
return path
}
// Returns the zero-based line numbers of all #include statements in the file
func (f *file) includesLineNumbers() []int {
out := []int{}
for i, l := range f.lines {
matches := includeRE.FindStringSubmatch(l)
if len(matches) == 0 {
continue
}
include := matches[1]
if include == "" {
include = matches[2]
}
if strings.HasSuffix(stripExtension(f.path), stripExtension(include)) {
// Don't remove #include for header of cc
continue
}
out = append(out, i)
}
return out
}
// Saves the file, omitting the lines with the zero-based line number that are
// either not in `lines` or have a `false` value.
func (f *file) save(lines map[int]bool) error {
content := []string{}
for i, l := range f.lines {
if lines[i] {
content = append(content, l)
}
}
data := []byte(strings.Join(content, "\n"))
return ioutil.WriteFile(f.path, data, 0666)
}
// Runs clang-format on the file
func (f *file) format() error {
err := exec.Command("clang-format", "-i", f.path).Run()
if err != nil {
return fmt.Errorf("Couldn't format file '%v': %w", f.path, err)
}
return nil
}
// Runs git add on the file
func (f *file) stage() error {
err := exec.Command("git", "-C", fileutils.ProjectRoot(), "add", f.path).Run()
if err != nil {
return fmt.Errorf("Couldn't stage file '%v': %w", f.path, err)
}
return nil
}
// Loads all the files with the given file paths, splitting their content into
// into lines.
func loadFiles(paths []string) ([]file, error) {
wg := sync.WaitGroup{}
wg.Add(len(paths))
files := make([]file, len(paths))
errs := make([]error, len(paths))
for i, path := range paths {
i, path := i, filepath.Join(fileutils.ProjectRoot(), path)
go func() {
defer wg.Done()
body, err := ioutil.ReadFile(path)
if err != nil {
errs[i] = fmt.Errorf("Failed to open %v: %w", path, err)
} else {
content := string(body)
lines := strings.Split(content, "\n")
files[i] = file{path: path, lines: lines}
}
}()
}
wg.Wait()
for _, err := range errs {
if err != nil {
return nil, err
}
}
return files, nil
}

45
tools/src/fileutils/fileutils_common.go Normal file
View File

@@ -0,0 +1,45 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package fileutils contains utility functions for files
package fileutils
import (
"path/filepath"
"runtime"
)
// GoSourcePath returns the absolute path to the .go file that calls the
// function
func GoSourcePath() string {
_, filename, _, ok := runtime.Caller(1)
if !ok {
panic("No caller information")
}
path, err := filepath.Abs(filename)
if err != nil {
panic(err)
}
return path
}
// ProjectRoot returns the path to the tint project root
func ProjectRoot() string {
toolRoot := filepath.Dir(GoSourcePath())
root, err := filepath.Abs(filepath.Join(toolRoot, "../../.."))
if err != nil {
panic(err)
}
return root
}

31
tools/src/fileutils/fileutils_other.go Normal file
View File

@@ -0,0 +1,31 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build !windows
// Package fileutils contains utility functions for files
package fileutils
import (
"os"
)
// IsExe returns true if the file at path is an executable
func IsExe(path string) bool {
s, err := os.Stat(path)
if err != nil {
return false
}
return s.Mode()&0100 != 0
}

38
tools/src/fileutils/fileutils_test.go Normal file
View File

@@ -0,0 +1,38 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fileutils_test
import (
"os"
"path/filepath"
"strings"
"testing"
"dawn.googlesource.com/tint/tools/src/fileutils"
)
func TestGoSourcePath(t *testing.T) {
p := fileutils.GoSourcePath()
if !strings.HasSuffix(p, "fileutils/fileutils_test.go") {
t.Errorf("GoSourcePath() returned %v", p)
}
}
func TestProjectRoot(t *testing.T) {
p := filepath.Join(fileutils.ProjectRoot(), "tint_overrides_with_defaults.gni")
if _, err := os.Stat(p); os.IsNotExist(err) {
t.Errorf("ProjectRoot() returned %v", p)
}
}

21
tools/src/fileutils/fileutils_windows.go Normal file
View File

@@ -0,0 +1,21 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package fileutils contains utility functions for files
package fileutils
// IsExe returns true if the file at path is an executable
func IsExe(path string) bool {
return true
}

95
tools/src/gerrit/gerrit.go Normal file
View File

@@ -0,0 +1,95 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// gerrit provides helpers for obtaining information from Tint's gerrit instance
package gerrit
import (
"fmt"
"io/ioutil"
"os"
"regexp"
"strings"
"github.com/andygrunwald/go-gerrit"
)
const URL = "https://dawn-review.googlesource.com/"
// G is the interface to gerrit
type G struct {
client *gerrit.Client
authenticated bool
}
type Config struct {
Username string
Password string
}
func LoadCredentials() (user, pass string) {
cookiesFile := os.Getenv("HOME") + "/.gitcookies"
if cookies, err := ioutil.ReadFile(cookiesFile); err == nil {
re := regexp.MustCompile(`dawn-review.googlesource.com\s+(?:FALSE|TRUE)[\s/]+(?:FALSE|TRUE)\s+[0-9]+\s+.\s+(.*)=(.*)`)
match := re.FindStringSubmatch(string(cookies))
if len(match) == 3 {
return match[1], match[2]
}
}
return "", ""
}
func New(cfg Config) (*G, error) {
client, err := gerrit.NewClient(URL, nil)
if err != nil {
return nil, fmt.Errorf("couldn't create gerrit client: %w", err)
}
user, pass := cfg.Username, cfg.Password
if user == "" {
user, pass = LoadCredentials()
}
if user != "" {
client.Authentication.SetBasicAuth(user, pass)
}
return &G{client, user != ""}, nil
}
func (g *G) QueryChanges(queryParts ...string) (changes []gerrit.ChangeInfo, query string, err error) {
changes = []gerrit.ChangeInfo{}
query = strings.Join(queryParts, "+")
for {
batch, _, err := g.client.Changes.QueryChanges(&gerrit.QueryChangeOptions{
QueryOptions: gerrit.QueryOptions{Query: []string{query}},
Skip: len(changes),
})
if err != nil {
if !g.authenticated {
err = fmt.Errorf(`query failed, possibly because of authentication.
See https://dawn-review.googlesource.com/new-password for obtaining a username
and password which can be provided with --gerrit-user and --gerrit-pass.
%w`, err)
}
return nil, "", err
}
changes = append(changes, *batch...)
if len(*batch) == 0 || !(*batch)[len(*batch)-1].MoreChanges {
break
}
}
return changes, query, nil
}

381
tools/src/git/git.go Normal file
View File

@@ -0,0 +1,381 @@
// Copyright 2022 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package git
import (
"context"
"encoding/hex"
"errors"
"fmt"
"net/url"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
)
// Hash is a 20 byte, git object hash.
type Hash [20]byte
func (h Hash) String() string { return hex.EncodeToString(h[:]) }
// IsZero returns true if the hash h is all zeros
func (h Hash) IsZero() bool {
zero := Hash{}
return h == zero
}
// ParseHash returns a Hash from a hexadecimal string.
func ParseHash(s string) (Hash, error) {
b, err := hex.DecodeString(s)
if err != nil {
return Hash{}, fmt.Errorf("failed to parse hash '%v':\n %w", s, err)
}
h := Hash{}
copy(h[:], b)
return h, nil
}
// The timeout for git operations if no other timeout is specified
var DefaultTimeout = time.Minute
// Git wraps the 'git' executable
type Git struct {
// Path to the git executable
exe string
// Debug flag to print all command to the `git` executable
LogAllActions bool
}
// New returns a new Git instance
func New(exe string) (*Git, error) {
if _, err := os.Stat(exe); err != nil {
return nil, err
}
return &Git{exe: exe}, nil
}
// Auth holds git authentication credentials
type Auth struct {
Username string
Password string
}
// Empty return true if there's no username or password for authentication
func (a Auth) Empty() bool {
return a.Username == "" && a.Password == ""
}
// ErrRepositoryDoesNotExist indicates that a repository does not exist
var ErrRepositoryDoesNotExist = errors.New("repository does not exist")
// Open opens an existing git repo at path. If the repository does not exist at
// path then ErrRepositoryDoesNotExist is returned.
func (g Git) Open(path string) (*Repository, error) {
info, err := os.Stat(filepath.Join(path, ".git"))
if err != nil || !info.IsDir() {
return nil, ErrRepositoryDoesNotExist
}
return &Repository{g, path}, nil
}
// Optional settings for Git.Clone
type CloneOptions struct {
// If specified then the given branch will be cloned instead of the default
Branch string
// Timeout for the operation
Timeout time.Duration
// Authentication for the clone
Auth Auth
}
// Clone performs a clone of the repository at url to path.
func (g Git) Clone(path, url string, opt *CloneOptions) (*Repository, error) {
if err := os.MkdirAll(path, 0777); err != nil {
return nil, err
}
if opt == nil {
opt = &CloneOptions{}
}
url, err := opt.Auth.addToURL(url)
if err != nil {
return nil, err
}
r := &Repository{g, path}
args := []string{"clone", url, "."}
if opt.Branch != "" {
args = append(args, "--branch", opt.Branch)
}
if _, err := r.run(opt.Timeout, args...); err != nil {
return nil, err
}
return r, nil
}
// Repository points to a git repository
type Repository struct {
// Path to the 'git' executable
Git Git
// Repo directory
Path string
}
// Optional settings for Repository.Fetch
type FetchOptions struct {
// The remote name. Defaults to 'origin'
Remote string
// Timeout for the operation
Timeout time.Duration
// Git authentication for the remote
Auth Auth
}
// Fetch performs a fetch of a reference from the remote, returning the Hash of
// the fetched reference.
func (r Repository) Fetch(ref string, opt *FetchOptions) (Hash, error) {
if opt == nil {
opt = &FetchOptions{}
}
if opt.Remote == "" {
opt.Remote = "origin"
}
if _, err := r.run(opt.Timeout, "fetch", opt.Remote, ref); err != nil {
return Hash{}, err
}
out, err := r.run(0, "rev-parse", "FETCH_HEAD")
if err != nil {
return Hash{}, err
}
return ParseHash(out)
}
// Optional settings for Repository.Push
type PushOptions struct {
// The remote name. Defaults to 'origin'
Remote string
// Timeout for the operation
Timeout time.Duration
// Git authentication for the remote
Auth Auth
}
// Push performs a push of the local reference to the remote reference.
func (r Repository) Push(localRef, remoteRef string, opt *PushOptions) error {
if opt == nil {
opt = &PushOptions{}
}
if opt.Remote == "" {
opt.Remote = "origin"
}
url, err := r.run(opt.Timeout, "remote", "get-url", opt.Remote)
if err != nil {
return err
}
url, err = opt.Auth.addToURL(url)
if err != nil {
return err
}
if _, err := r.run(opt.Timeout, "push", url, localRef+":"+remoteRef); err != nil {
return err
}
return nil
}
// Optional settings for Repository.Add
type AddOptions struct {
// Timeout for the operation
Timeout time.Duration
// Git authentication for the remote
Auth Auth
}
// Add stages the listed files
func (r Repository) Add(path string, opt *AddOptions) error {
if opt == nil {
opt = &AddOptions{}
}
if _, err := r.run(opt.Timeout, "add", path); err != nil {
return err
}
return nil
}
// Optional settings for Repository.Commit
type CommitOptions struct {
// Timeout for the operation
Timeout time.Duration
// Author name
AuthorName string
// Author email address
AuthorEmail string
}
// Commit commits the staged files with the given message, returning the hash of
// commit
func (r Repository) Commit(msg string, opt *CommitOptions) (Hash, error) {
if opt == nil {
opt = &CommitOptions{}
}
args := []string{"commit", "-m", msg}
if opt.AuthorName != "" || opt.AuthorEmail != "" {
args = append(args, "--author", fmt.Sprintf("%v <%v>", opt.AuthorName, opt.AuthorEmail))
}
if _, err := r.run(opt.Timeout, args...); err != nil {
return Hash{}, err
}
out, err := r.run(0, "rev-parse", "HEAD")
if err != nil {
return Hash{}, err
}
return ParseHash(out)
}
// Optional settings for Repository.Checkout
type CheckoutOptions struct {
// Timeout for the operation
Timeout time.Duration
}
// Checkout performs a checkout of a reference.
func (r Repository) Checkout(ref string, opt *CheckoutOptions) error {
if opt == nil {
opt = &CheckoutOptions{}
}
if _, err := r.run(opt.Timeout, "checkout", ref); err != nil {
return err
}
return nil
}
// Optional settings for Repository.Log
type LogOptions struct {
// The git reference to the oldest commit in the range to query.
From string
// The git reference to the newest commit in the range to query.
To string
// The maximum number of entries to return.
Count int
// Timeout for the operation
Timeout time.Duration
}
// CommitInfo describes a single git commit
type CommitInfo struct {
Hash Hash
Date time.Time
Author string
Subject string
Description string
}
// Log returns the list of commits between two references (inclusive).
// The first returned commit is the most recent.
func (r Repository) Log(opt *LogOptions) ([]CommitInfo, error) {
if opt == nil {
opt = &LogOptions{}
}
args := []string{"log"}
rng := "HEAD"
if opt.To != "" {
rng = opt.To
}
if opt.From != "" {
rng = opt.From + "^.." + rng
}
args = append(args, rng, "--pretty=format:ǁ%Hǀ%cIǀ%an <%ae>ǀ%sǀ%b")
if opt.Count != 0 {
args = append(args, fmt.Sprintf("-%d", opt.Count))
}
out, err := r.run(opt.Timeout, args...)
if err != nil {
return nil, err
}
return parseLog(out)
}
func (r Repository) run(timeout time.Duration, args ...string) (string, error) {
return r.Git.run(r.Path, timeout, args...)
}
func (r Repository) runAll(timeout time.Duration, args ...[]string) error {
for _, a := range args {
if _, err := r.run(timeout, a...); err != nil {
return err
}
}
return nil
}
func (g Git) run(dir string, timeout time.Duration, args ...string) (string, error) {
if timeout == 0 {
timeout = DefaultTimeout
}
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
cmd := exec.CommandContext(ctx, g.exe, args...)
cmd.Dir = dir
if g.LogAllActions {
fmt.Printf("%v> %v %v\n", dir, g.exe, strings.Join(args, " "))
}
out, err := cmd.CombinedOutput()
if g.LogAllActions {
fmt.Println(string(out))
}
if err != nil {
return string(out), fmt.Errorf("%v> %v %v failed:\n %w\n%v",
dir, g.exe, strings.Join(args, " "), err, string(out))
}
return strings.TrimSpace(string(out)), nil
}
func (a Auth) addToURL(u string) (string, error) {
if !a.Empty() {
modified, err := url.Parse(u)
if err != nil {
return "", fmt.Errorf("failed to parse url '%v': %v", u, err)
}
modified.User = url.UserPassword(a.Username, a.Password)
u = modified.String()
}
return u, nil
}
func parseLog(str string) ([]CommitInfo, error) {
msgs := strings.Split(str, "ǁ")
cls := make([]CommitInfo, 0, len(msgs))
for _, s := range msgs {
if parts := strings.Split(s, "ǀ"); len(parts) == 5 {
hash, err := ParseHash(parts[0])
if err != nil {
return nil, err
}
date, err := time.Parse(time.RFC3339, parts[1])
if err != nil {
return nil, err
}
cl := CommitInfo{
Hash: hash,
Date: date,
Author: strings.TrimSpace(parts[2]),
Subject: strings.TrimSpace(parts[3]),
Description: strings.TrimSpace(parts[4]),
}
cls = append(cls, cl)
}
}
return cls, nil
}

203
tools/src/glob/glob.go Normal file
View File

@@ -0,0 +1,203 @@
// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package glob provides file globbing utilities
package glob
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"dawn.googlesource.com/tint/tools/src/match"
)
// Scan walks all files and subdirectories from root, returning those
// that Config.shouldExamine() returns true for.
func Scan(root string, cfg Config) ([]string, error) {
files := []string{}
err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
rel, err := filepath.Rel(root, path)
if err != nil {
rel = path
}
if rel == ".git" {
return filepath.SkipDir
}
if !cfg.shouldExamine(root, path) {
return nil
}
if !info.IsDir() {
files = append(files, rel)
}
return nil
})
if err != nil {
return nil, err
}
return files, nil
}
// Configs is a slice of Config.
type Configs []Config
// Config is used to parse the JSON configuration file.
type Config struct {
// Paths holds a number of JSON objects that contain either a "includes" or
// "excludes" key to an array of path patterns.
// Each path pattern is considered in turn to either include or exclude the
// file path for license scanning. Pattern use forward-slashes '/' for
// directory separators, and may use the following wildcards:
// ? - matches any single non-separator character
// * - matches any sequence of non-separator characters
// ** - matches any sequence of characters including separators
//
// Rules are processed in the order in which they are declared, with later
// rules taking precedence over earlier rules.
//
// All files are excluded before the first rule is evaluated.
//
// Example:
//
// {
// "paths": [
// { "exclude": [ "out/*", "build/*" ] },
// { "include": [ "out/foo.txt" ] }
// ],
// }
Paths searchRules
}
// LoadConfig loads a config file at path.
func LoadConfig(path string) (Config, error) {
cfgBody, err := ioutil.ReadFile(path)
if err != nil {
return Config{}, err
}
return ParseConfig(string(cfgBody))
}
// ParseConfig parses the config from a JSON string.
func ParseConfig(config string) (Config, error) {
d := json.NewDecoder(strings.NewReader(config))
cfg := Config{}
if err := d.Decode(&cfg); err != nil {
return Config{}, err
}
return cfg, nil
}
// MustParseConfig parses the config from a JSON string, panicing if the config
// does not parse
func MustParseConfig(config string) Config {
d := json.NewDecoder(strings.NewReader(config))
cfg := Config{}
if err := d.Decode(&cfg); err != nil {
panic(fmt.Errorf("Failed to parse config: %w\nConfig:\n%v", err, config))
}
return cfg
}
// rule is a search path predicate.
// root is the project relative path.
// cond is the value to return if the rule doesn't either include or exclude.
type rule func(path string, cond bool) bool
// searchRules is a ordered list of search rules.
// searchRules is its own type as it has to perform custom JSON unmarshalling.
type searchRules []rule
// UnmarshalJSON unmarshals the array of rules in the form:
// { "include": [ ... ] } or { "exclude": [ ... ] }
func (l *searchRules) UnmarshalJSON(body []byte) error {
type parsed struct {
Include []string
Exclude []string
}
p := []parsed{}
if err := json.NewDecoder(bytes.NewReader(body)).Decode(&p); err != nil {
return err
}
*l = searchRules{}
for _, rule := range p {
rule := rule
switch {
case len(rule.Include) > 0 && len(rule.Exclude) > 0:
return fmt.Errorf("Rule cannot contain both include and exclude")
case len(rule.Include) > 0:
tests := make([]match.Test, len(rule.Include))
for i, pattern := range rule.Include {
test, err := match.New(pattern)
if err != nil {
return err
}
tests[i] = test
}
*l = append(*l, func(path string, cond bool) bool {
for _, test := range tests {
if test(path) {
return true
}
}
return cond
})
case len(rule.Exclude) > 0:
tests := make([]match.Test, len(rule.Exclude))
for i, pattern := range rule.Exclude {
test, err := match.New(pattern)
if err != nil {
return err
}
tests[i] = test
}
*l = append(*l, func(path string, cond bool) bool {
for _, test := range tests {
if test(path) {
return false
}
}
return cond
})
}
}
return nil
}
// shouldExamine returns true if the file at absPath should be scanned.
func (c Config) shouldExamine(root, absPath string) bool {
root = filepath.ToSlash(root) // Canonicalize
absPath = filepath.ToSlash(absPath) // Canonicalize
relPath, err := filepath.Rel(root, absPath)
if err != nil {
return false
}
relPath = filepath.ToSlash(relPath) // Canonicalize
res := false
for _, rule := range c.Paths {
res = rule(relPath, res)
}
return res
}

137
tools/src/list/list.go Normal file
View File

@@ -0,0 +1,137 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package list provides utilities for handling lists of dynamically-typed elements
package list
import (
"fmt"
"reflect"
)
// List is an interface to a list of dynamically-typed elements
type List interface {
// Count returns the number if items in the list
Count() int
// Get returns the element at the index i
Get(i int) interface{}
// Set assigns the element at the index i with v
Set(i int, v interface{})
// Append adds a single item, list, or slice of items to this List
Append(v interface{})
// Copy copies the elements at [dst..dst+count) to [src..src+count)
Copy(dst, src, count int)
// CopyFrom copies the elements [src..src+count) from the list l to the
// elements [dst..dst+count) in this list
CopyFrom(l List, dst, src, count int)
// Reduces the size of the list to count elements
Resize(count int)
// ElementType returns the type of the elements of the list
ElementType() reflect.Type
}
// Wrap returns a List that wraps a slice pointer
func Wrap(s interface{}) List {
ptr := reflect.ValueOf(s)
if ptr.Kind() != reflect.Ptr || ptr.Elem().Kind() != reflect.Slice {
panic(fmt.Errorf("Wrap() must be called with a pointer to slice. Got: %T", s))
}
return list{ptr.Elem()}
}
// New returns a new list of element type elem for n items
func New(elem reflect.Type, count int) List {
slice := reflect.SliceOf(elem)
return list{reflect.MakeSlice(slice, count, count)}
}
// Copy makes a shallow copy of the list
func Copy(l List) List {
out := New(l.ElementType(), l.Count())
out.CopyFrom(l, 0, 0, l.Count())
return out
}
type list struct{ v reflect.Value }
func (l list) Count() int {
return l.v.Len()
}
func (l list) Get(i int) interface{} {
return l.v.Index(i).Interface()
}
func (l list) Set(i int, v interface{}) {
l.v.Index(i).Set(reflect.ValueOf(v))
}
func (l list) Append(v interface{}) {
switch v := v.(type) {
case list:
l.v.Set(reflect.AppendSlice(l.v, reflect.Value(v.v)))
case List:
// v implements `List`, but isn't a `list`. Need to do a piece-wise copy
items := make([]reflect.Value, v.Count())
for i := range items {
items[i] = reflect.ValueOf(v.Get(i))
}
l.v.Set(reflect.Append(l.v, items...))
default:
r := reflect.ValueOf(v)
if r.Type() == l.v.Type() {
l.v.Set(reflect.AppendSlice(l.v, r))
return
}
l.v.Set(reflect.Append(l.v, reflect.ValueOf(v)))
}
}
func (l list) Copy(dst, src, count int) {
reflect.Copy(
l.v.Slice(dst, dst+count),
l.v.Slice(src, src+count),
)
}
func (l list) CopyFrom(o List, dst, src, count int) {
if o, ok := o.(list); ok {
reflect.Copy(
l.v.Slice(dst, dst+count),
o.v.Slice(src, src+count),
)
}
// v implements `List`, but isn't a `list`. Need to do a piece-wise copy
items := make([]reflect.Value, count)
for i := range items {
l.Set(dst+i, o.Get(src+i))
}
}
func (l list) Resize(count int) {
new := reflect.MakeSlice(l.v.Type(), count, count)
reflect.Copy(new, l.v)
l.v.Set(new)
}
func (l list) ElementType() reflect.Type {
return l.v.Type().Elem()
}

227
tools/src/list/list_test.go Normal file
View File

@@ -0,0 +1,227 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either exprel or implied.
// See the License for the specific language governing permilions and
// limitations under the License.
package list_test
import (
"reflect"
"testing"
"dawn.googlesource.com/tint/tools/src/list"
)
// A simple implementation of list.List. Many methods are just stubs
type customList struct{}
func (customList) Count() int { return 3 }
func (customList) Get(i int) interface{} { return 10 + i*10 }
func (customList) Set(i int, v interface{}) {}
func (customList) Append(v interface{}) {}
func (customList) Copy(dst, src, count int) {}
func (customList) CopyFrom(l list.List, dst, src, count int) {}
func (customList) Resize(count int) {}
func (customList) ElementType() reflect.Type { return nil }
var _ list.List = customList{} // Interface compliance check
func TestNew(t *testing.T) {
l := list.New(reflect.TypeOf(0), 3)
if n := l.Count(); n != 3 {
t.Errorf("Count(0): %v", n)
}
if n := l.Get(0); n != 0 {
t.Errorf("Get(0): %v", n)
}
if n := l.Get(1); n != 0 {
t.Errorf("Get(1): %v", n)
}
if n := l.Get(2); n != 0 {
t.Errorf("Get(2): %v", n)
}
}
func TestCopy(t *testing.T) {
slice := []int{1, 2, 3}
l := list.Wrap(&slice)
c := list.Copy(l)
if n := c.Count(); n != 3 {
t.Errorf("Count(0): %v", n)
}
if n := c.Get(0); n != 1 {
t.Errorf("Get(0): %v", n)
}
if n := c.Get(1); n != 2 {
t.Errorf("Get(1): %v", n)
}
if n := c.Get(2); n != 3 {
t.Errorf("Get(2): %v", n)
}
}
func TestListCount(t *testing.T) {
slice := make([]int, 5)
l := list.Wrap(&slice)
if c := l.Count(); c != 5 {
t.Errorf("Count() is %v", c)
}
}
func TestListGrow(t *testing.T) {
slice := []int{}
l := list.Wrap(&slice)
l.Resize(10)
if len(slice) != 10 {
t.Errorf("len(slice) after Resize(10) is %v", len(slice))
}
}
func TestListShrink(t *testing.T) {
slice := make([]int, 10)
l := list.Wrap(&slice)
l.Resize(5)
if len(slice) != 5 {
t.Errorf("len(slice) after Resize(5) is %v", len(slice))
}
}
func TestListCopy(t *testing.T) {
slice := []int{0, 10, 20, 0, 0, 0}
l := list.Wrap(&slice)
l.Copy(3, 1, 2)
if !reflect.DeepEqual(slice, []int{0, 10, 20, 10, 20, 0}) {
t.Errorf("after Copy(), slice: %v", slice)
}
}
func TestListCopyFromList(t *testing.T) {
sliceA := []int{10, 20, 30, 40, 50, 60}
lA := list.Wrap(&sliceA)
sliceB := []int{1, 2, 3, 4, 5, 6}
lB := list.Wrap(&sliceB)
lA.CopyFrom(lB, 1, 2, 3)
if !reflect.DeepEqual(sliceA, []int{10, 3, 4, 5, 50, 60}) {
t.Errorf("after CopyFrom(), slice: %v", sliceA)
}
}
func TestListCopyFromCustomList(t *testing.T) {
sliceA := []int{10, 20, 30, 40, 50, 60}
lA := list.Wrap(&sliceA)
lA.CopyFrom(customList{}, 1, 2, 3)
if !reflect.DeepEqual(sliceA, []int{10, 30, 40, 50, 50, 60}) {
t.Errorf("after CopyFrom(), slice: %v", sliceA)
}
}
func TestListGet(t *testing.T) {
slice := []int{0, 10, 20, 10, 20}
l := list.Wrap(&slice)
if n := l.Get(0); n != 0 {
t.Errorf("Get(0): %v", n)
}
if n := l.Get(1); n != 10 {
t.Errorf("Get(1): %v", n)
}
if n := l.Get(2); n != 20 {
t.Errorf("Get(2): %v", n)
}
if n := l.Get(3); n != 10 {
t.Errorf("Get(3): %v", n)
}
if n := l.Get(4); n != 20 {
t.Errorf("Get(4): %v", n)
}
}
func TestListSet(t *testing.T) {
slice := []int{0, 10, 20, 10, 20}
l := list.Wrap(&slice)
l.Set(0, 50)
l.Set(2, 90)
l.Set(4, 60)
if !reflect.DeepEqual(slice, []int{50, 10, 90, 10, 60}) {
t.Errorf("after Set(), slice: %v", slice)
}
}
func TestListAppendItem(t *testing.T) {
slice := []int{1, 2, 3}
l := list.Wrap(&slice)
l.Append(9)
if c := len(slice); c != 4 {
t.Errorf("len(slice): %v", 4)
}
if n := slice[3]; n != 9 {
t.Errorf("slice[3]: %v", n)
}
}
func TestListAppendItems(t *testing.T) {
slice := []int{1, 2, 3}
l := list.Wrap(&slice)
l.Append([]int{9, 8, 7})
if !reflect.DeepEqual(slice, []int{1, 2, 3, 9, 8, 7}) {
t.Errorf("after Append(), slice: %v", slice)
}
}
func TestListAppendList(t *testing.T) {
sliceA := []int{1, 2, 3}
lA := list.Wrap(&sliceA)
sliceB := []int{9, 8, 7}
lB := list.Wrap(&sliceB)
lA.Append(lB)
if !reflect.DeepEqual(sliceA, []int{1, 2, 3, 9, 8, 7}) {
t.Errorf("after Append(), sliceA: %v", sliceA)
}
if !reflect.DeepEqual(sliceB, []int{9, 8, 7}) {
t.Errorf("after Append(), sliceB: %v", sliceB)
}
}
func TestListAppendCustomList(t *testing.T) {
sliceA := []int{1, 2, 3}
lA := list.Wrap(&sliceA)
lA.Append(customList{})
if !reflect.DeepEqual(sliceA, []int{1, 2, 3, 10, 20, 30}) {
t.Errorf("after Append(), sliceA: %v", sliceA)
}
}

245
tools/src/lut/lut.go Normal file
View File

@@ -0,0 +1,245 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package lut provides a look up table, which compresses indexed data
package lut
import (
"sort"
"dawn.googlesource.com/tint/tools/src/list"
)
// LUT is a look up table.
// The table holds a number of items that are stored in a linear list.
type LUT interface {
// Add adds a sequence of items to the table.
// items can be a single element, a slice of element, or a List of element.
// Returns a pointer to the offset of the first item in the table's list.
// The sequence of items stored at [offset, offset+N), where N is the
// number of items added will remain equal, even after calling Compact().
Add(items interface{}) *int
// Compact reorders the table items so that the table storage is compacted
// by shuffling data around and de-duplicating sequences of common data.
// Each originally added sequence is preserved in the resulting table, with
// the same contiguous ordering, but with a potentially different offset.
// Heuristics are used to shorten the table length, by exploiting common
// subsequences, and removing duplicate sequences.
// Note that shortest common superstring is NP-hard, so heuristics are used.
// Compact updates pointers returned by Add().
Compact()
}
// New returns a new look up table
func New(storage list.List) LUT {
return &lut{storage: storage}
}
// A sequence represents a span of entries in the table
type sequence struct {
offset *int // Pointer to the start index of the sequence
count int // Length of the sequence
}
// lut implements LUT
type lut struct {
storage list.List // The List that backs this LUT
sequences []sequence // The entries in the LUT
}
func (t *lut) Add(items interface{}) *int {
offset := t.storage.Count()
t.storage.Append(items)
count := t.storage.Count() - offset
offsetPtr := &offset
t.sequences = append(t.sequences, sequence{offsetPtr, count})
return offsetPtr
}
func (t lut) Compact() {
// Generate int32 identifiers for each unique item in the table.
// We use these to compare items instead of comparing the real data as this
// function is comparison-heavy, and integer compares are cheap.
srcIDs := t.itemIDs()
dstIDs := make([]int32, len(srcIDs))
// Make a copy the data held in the table, use the copy as the source, and
// t.storage as the destination.
srcData := list.Copy(t.storage)
dstData := t.storage
// Sort all the sequences by length, with the largest first.
// This helps 'seed' the compacted form with the largest items first.
// This can improve the compaction as small sequences can pack into larger,
// placed items.
sort.Slice(t.sequences, func(i, j int) bool {
return t.sequences[i].count > t.sequences[j].count
})
// unplaced is the list of sequences that have not yet been placed.
// All sequences are initially unplaced.
unplaced := make([]sequence, len(t.sequences))
copy(unplaced, t.sequences)
// placed is the list of sequences that have been placed.
// Nothing is initially placed.
placed := make([]sequence, 0, len(t.sequences))
// remove removes the sequence in unplaced with the index i.
remove := func(i int) {
placed = append(placed, unplaced[i])
if i > 0 {
if i < len(unplaced)-1 {
copy(unplaced[i:], unplaced[i+1:])
}
unplaced = unplaced[:len(unplaced)-1]
} else {
unplaced = unplaced[1:]
}
}
// cp copies data from [srcOffset:srcOffset+count] to [dstOffset:dstOffset+count].
cp := func(dstOffset, srcOffset, count int) {
dstData.CopyFrom(srcData, dstOffset, srcOffset, count)
copy(
dstIDs[dstOffset:dstOffset+count],
srcIDs[srcOffset:srcOffset+count],
)
}
// match describes a sequence that can be placed.
type match struct {
dst int // destination offset
src sequence // source sequence
len int // number of items that matched
idx int // sequence index
}
// number of items that have been placed.
newSize := 0
// While there's sequences to place...
for len(unplaced) > 0 {
// Place the next largest, unplaced sequence at the end of the new list
cp(newSize, *unplaced[0].offset, unplaced[0].count)
*unplaced[0].offset = newSize
newSize += unplaced[0].count
remove(0)
for {
// Look for the sequence with the longest match against the
// currently placed data. Any mismatches with currently placed data
// will nullify the match. The head or tail of this sequence may
// extend the currently placed data.
best := match{}
// For each unplaced sequence...
for i := 0; i < len(unplaced); i++ {
seq := unplaced[i]
if best.len >= seq.count {
// The best match is already at least as long as this
// sequence and sequences are sorted by size, so best cannot
// be beaten. Stop searching.
break
}
// Perform a full sweep from left to right, scoring the match...
for shift := -seq.count + 1; shift < newSize; shift++ {
dstS := max(shift, 0)
dstE := min(shift+seq.count, newSize)
count := dstE - dstS
srcS := *seq.offset - min(shift, 0)
srcE := srcS + count
if best.len < count {
if equal(srcIDs[srcS:srcE], dstIDs[dstS:dstE]) {
best = match{shift, seq, count, i}
}
}
}
}
if best.src.offset == nil {
// Nothing matched. Not even one element.
// Resort to placing the next largest sequence at the end.
break
}
if best.dst < 0 {
// Best match wants to place the sequence to the left of the
// current output. We have to shuffle everything...
n := -best.dst
dstData.Copy(n, 0, newSize)
copy(dstIDs[n:n+newSize], dstIDs)
newSize += n
best.dst = 0
for _, p := range placed {
*p.offset += n
}
}
// Place the best matching sequence.
cp(best.dst, *best.src.offset, best.src.count)
newSize = max(newSize, best.dst+best.src.count)
*best.src.offset = best.dst
remove(best.idx)
}
}
// Shrink the output buffer to the new size.
dstData.Resize(newSize)
// All done.
}
// Generate a set of identifiers for all the unique items in storage
func (t lut) itemIDs() []int32 {
storageSize := t.storage.Count()
keys := make([]int32, storageSize)
dataToKey := map[interface{}]int32{}
for i := 0; i < storageSize; i++ {
data := t.storage.Get(i)
key, found := dataToKey[data]
if !found {
key = int32(len(dataToKey))
dataToKey[data] = key
}
keys[i] = key
}
return keys
}
func max(a, b int) int {
if a < b {
return b
}
return a
}
func min(a, b int) int {
if a > b {
return b
}
return a
}
func equal(a, b []int32) bool {
for i, v := range a {
if b[i] != v {
return false
}
}
return true
}

66
tools/src/lut/lut_test.go Normal file
View File

@@ -0,0 +1,66 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package lut_test
import (
"testing"
"dawn.googlesource.com/tint/tools/src/list"
"dawn.googlesource.com/tint/tools/src/lut"
)
func TestCompactWithFragments(t *testing.T) {
runes := []rune{}
lut := lut.New(list.Wrap(&runes))
indices := []*int{
lut.Add([]rune("the life in your")),
lut.Add([]rune("in your life that count")),
lut.Add([]rune("In the end,")),
lut.Add([]rune("the life in")),
lut.Add([]rune("count. It's the")),
lut.Add([]rune("years")),
lut.Add([]rune("in your years.")),
lut.Add([]rune("it's not the years in")),
lut.Add([]rune("not the years")),
lut.Add([]rune("not the years in your")),
lut.Add([]rune("end, it's")),
}
lut.Compact()
expect := "In the end, it's not the years in your life that count. It's the life in your years."
got := string(runes)
if got != expect {
t.Errorf("Compact result was not as expected\nExpected: '%v'\nGot: '%v'", expect, got)
}
expectedIndices := []int{
61, // the life in your
31, // in your life that count
0, // In the end,
61, // the life in
49, // count. It's the
25, // years
70, // in your years.
12, // it's not the years in
17, // not the years
17, // not the years in your
7, // end, it's
}
for i, p := range indices {
if expected, got := expectedIndices[i], *p; expected != got {
t.Errorf("Index %v was not expected. Expected %v, got %v", i, expected, got)
}
}
}

76
tools/src/match/match.go Normal file
View File

@@ -0,0 +1,76 @@
// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package match provides functions for performing filepath [?,*,**] wildcard
// matching.
package match
import (
"fmt"
"regexp"
"strings"
)
// Test is the match predicate returned by New.
type Test func(path string) bool
// New returns a Test function that returns true iff the path matches the
// provided pattern.
//
// pattern uses forward-slashes for directory separators '/', and may use the
// following wildcards:
// ? - matches any single non-separator character
// * - matches any sequence of non-separator characters
// ** - matches any sequence of characters including separators
func New(pattern string) (Test, error) {
// Transform pattern into a regex by replacing the uses of `?`, `*`, `**`
// with corresponding regex patterns.
// As the pattern may contain other regex sequences, the string has to be
// escaped. So:
// a) Replace the patterns of `?`, `*`, `**` with unique placeholder tokens.
// b) Escape the expression so that other sequences don't confuse the regex
// parser.
// c) Replace the placeholder tokens with the corresponding regex tokens.
// Temporary placeholder tokens
const (
starstar = "••"
star = "•"
questionmark = "¿"
)
// Check pattern doesn't contain any of our placeholder tokens
for _, r := range []rune{'•', '¿'} {
if strings.ContainsRune(pattern, r) {
return nil, fmt.Errorf("Pattern must not contain '%c'", r)
}
}
// Replace **, * and ? with placeholder tokens
subbed := pattern
subbed = strings.ReplaceAll(subbed, "**", starstar)
subbed = strings.ReplaceAll(subbed, "*", star)
subbed = strings.ReplaceAll(subbed, "?", questionmark)
// Escape any remaining regex characters
escaped := regexp.QuoteMeta(subbed)
// Insert regex matchers for the substituted tokens
regex := "^" + escaped + "$"
regex = strings.ReplaceAll(regex, starstar, ".*")
regex = strings.ReplaceAll(regex, star, "[^/]*")
regex = strings.ReplaceAll(regex, questionmark, "[^/]")
re, err := regexp.Compile(regex)
if err != nil {
return nil, fmt.Errorf(`Failed to compile regex "%v" for pattern "%v": %w`, regex, pattern, err)
}
return re.MatchString, nil
}

106
tools/src/match/match_test.go Normal file
View File

@@ -0,0 +1,106 @@
// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package match_test
import (
"strings"
"testing"
"dawn.googlesource.com/tint/tools/src/match"
)
func TestMatch(t *testing.T) {
for _, test := range []struct {
pattern string
path string
expect bool
}{
{"a", "a", true},
{"b", "a", false},
{"?", "a", true},
{"a/?/c", "a/x/c", true},
{"a/??/c", "a/x/c", false},
{"a/??/c", "a/xx/c", true},
{"a/???/c", "a/x z/c", true},
{"a/?/c", "a/xx/c", false},
{"a/?/?/c", "a/x/y/c", true},
{"a/?/?/?/c", "a/x/y/z/c", true},
{"a/???/c", "a/x/y/c", false},
{"a/?????", "a/x/y/c", false},
{"*", "a", true},
{"*", "abc", true},
{"*", "abc 123", true},
{"*", "xxx/yyy", false},
{"*/*", "xxx/yyy", true},
{"*/*", "xxx/yyy/zzz", false},
{"*/*/c", "xxx/yyy/c", true},
{"a/*/*", "a/xxx/yyy", true},
{"a/*/c", "a/xxx/c", true},
{"a/*/c", "a/xxx/c", true},
{"a/*/*/c", "a/b/c", false},
{"**", "a", true},
{"**", "abc", true},
{"**", "abc 123", true},
{"**", "xxx/yyy", true},
{"**", "xxx/yyy/zzz", true},
{"**/**", "xxx", false},
{"**/**", "xxx/yyy", true},
{"**/**", "xxx/yyy/zzz", true},
{"**/**/**", "xxx/yyy/zzz", true},
{"**/**/c", "xxx/yyy/c", true},
{"**/**/c", "xxx/yyy/c/d", false},
{"a/**/**", "a/xxx/yyy", true},
{"a/**/c", "a/xxx/c", true},
{"a/**/c", "a/xxx/yyy/c", true},
{"a/**/c", "a/xxx/y y/zzz/c", true},
{"a/**/c", "a/c", false},
{"a/**c", "a/c", true},
{"xxx/**.foo", "xxx/aaa.foo", true},
{"xxx/**.foo", "xxx/yyy/zzz/.foo", true},
{"xxx/**.foo", "xxx/yyy/zzz/bar.foo", true},
} {
f, err := match.New(test.pattern)
if err != nil {
t.Errorf(`match.New("%v")`, test.pattern)
continue
}
matched := f(test.path)
switch {
case matched && !test.expect:
t.Errorf(`Path "%v" matched against pattern "%v"`, test.path, test.pattern)
case !matched && test.expect:
t.Errorf(`Path "%v" did not match against pattern "%v"`, test.path, test.pattern)
}
}
}
func TestErrOnPlaceholder(t *testing.T) {
for _, pattern := range []string{"a/b••c", "a/b•c", "a/b/¿c"} {
_, err := match.New(pattern)
if err == nil {
t.Errorf(`match.New("%v") did not return an expected error`, pattern)
continue
}
if !strings.Contains(err.Error(), "Pattern must not contain") {
t.Errorf(`match.New("%v") returned unrecognised error: %v`, pattern, err)
continue
}
}
}

52
tools/src/substr/substr.go Normal file
View File

@@ -0,0 +1,52 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package substr
import (
diff "github.com/sergi/go-diff/diffmatchpatch"
)
// Fix attempts to reconstruct substr by comparing it to body.
// substr is a fuzzy substring of body.
// Fix returns a new exact substring of body, by calculating a diff of the text.
// If no match could be made, Fix() returns an empty string.
func Fix(body, substr string) string {
dmp := diff.New()
diffs := dmp.DiffMain(body, substr, false)
if len(diffs) == 0 {
return ""
}
front := func() diff.Diff { return diffs[0] }
back := func() diff.Diff { return diffs[len(diffs)-1] }
start, end := 0, len(body)
// Trim edits that remove text from body start
for len(diffs) > 0 && front().Type == diff.DiffDelete {
start += len(front().Text)
diffs = diffs[1:]
}
// Trim edits that remove text from body end
for len(diffs) > 0 && back().Type == diff.DiffDelete {
end -= len(back().Text)
diffs = diffs[:len(diffs)-1]
}
// New substring is the span for the remainder of the edits
return body[start:end]
}

275
tools/src/substr/substr_test.go Normal file
View File

@@ -0,0 +1,275 @@
// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package substr
import (
"strings"
"testing"
)
func TestFixSubstr(t *testing.T) {
type test struct {
body string
substr string
expect string
}
for _, test := range []test{
{
body: "abc_def_ghi_jkl_mno",
substr: "def_XXX_jkl",
expect: "def_ghi_jkl",
},
{
body: "abc\ndef\nghi\njkl\nmno",
substr: "def\nXXX\njkl",
expect: "def\nghi\njkl",
},
{
body: "aaaaa12345ccccc",
substr: "1x345",
expect: "12345",
},
{
body: "aaaaa12345ccccc",
substr: "12x45",
expect: "12345",
},
{
body: "aaaaa12345ccccc",
substr: "123x5",
expect: "12345",
},
{
body: "aaaaaaaaaaaaa",
substr: "bbbbbbbbbbbbb",
expect: "", // cannot produce a sensible diff
}, { ///////////////////////////////////////////////////////////////////
body: `Return{
{
ScalarConstructor[not set]{42u}
}
}
`,
substr: `Return{
{
ScalarConstructor[not set]{42}
}
}`,
expect: `Return{
{
ScalarConstructor[not set]{42u}
}
}`,
}, { ///////////////////////////////////////////////////////////////////
body: `VariableDeclStatement{
Variable{
x_1
function
__u32
}
}
Assignment{
Identifier[not set]{x_1}
ScalarConstructor[not set]{42u}
}
Assignment{
Identifier[not set]{x_1}
ScalarConstructor[not set]{0u}
}
Return{}
`,
substr: `Assignment{
Identifier[not set]{x_1}
ScalarConstructor[not set]{42}
}
Assignment{
Identifier[not set]{x_1}
ScalarConstructor[not set]{0}
}`,
expect: `Assignment{
Identifier[not set]{x_1}
ScalarConstructor[not set]{42u}
}
Assignment{
Identifier[not set]{x_1}
ScalarConstructor[not set]{0u}
}`,
}, { ///////////////////////////////////////////////////////////////////
body: `VariableDeclStatement{
Variable{
a
function
__bool
{
ScalarConstructor[not set]{true}
}
}
}
VariableDeclStatement{
Variable{
b
function
__bool
{
ScalarConstructor[not set]{false}
}
}
}
VariableDeclStatement{
Variable{
c
function
__i32
{
ScalarConstructor[not set]{-1}
}
}
}
VariableDeclStatement{
Variable{
d
function
__u32
{
ScalarConstructor[not set]{1u}
}
}
}
VariableDeclStatement{
Variable{
e
function
__f32
{
ScalarConstructor[not set]{1.500000}
}
}
}
`,
substr: `VariableDeclStatement{
Variable{
a
function
__bool
{
ScalarConstructor[not set]{true}
}
}
}
VariableDeclStatement{
Variable{
b
function
__bool
{
ScalarConstructor[not set]{false}
}
}
}
VariableDeclStatement{
Variable{
c
function
__i32
{
ScalarConstructor[not set]{-1}
}
}
}
VariableDeclStatement{
Variable{
d
function
__u32
{
ScalarConstructor[not set]{1}
}
}
}
VariableDeclStatement{
Variable{
e
function
__f32
{
ScalarConstructor[not set]{1.500000}
}
}
}
`,
expect: `VariableDeclStatement{
Variable{
a
function
__bool
{
ScalarConstructor[not set]{true}
}
}
}
VariableDeclStatement{
Variable{
b
function
__bool
{
ScalarConstructor[not set]{false}
}
}
}
VariableDeclStatement{
Variable{
c
function
__i32
{
ScalarConstructor[not set]{-1}
}
}
}
VariableDeclStatement{
Variable{
d
function
__u32
{
ScalarConstructor[not set]{1u}
}
}
}
VariableDeclStatement{
Variable{
e
function
__f32
{
ScalarConstructor[not set]{1.500000}
}
}
}
`,
},
} {
body := strings.ReplaceAll(test.body, "\n", "␤")
substr := strings.ReplaceAll(test.substr, "\n", "␤")
expect := strings.ReplaceAll(test.expect, "\n", `␤`)
got := strings.ReplaceAll(Fix(test.body, test.substr), "\n", "␤")
if got != expect {
t.Errorf("Test failure:\nbody: '%v'\nsubstr: '%v'\nexpect: '%v'\ngot: '%v'\n\n", body, substr, expect, got)
}
}
}

33
tools/test-runner Executable file
View File

@@ -0,0 +1,33 @@
#!/usr/bin/env bash
# Copyright 2021 The Tint Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e # Fail on any error.
if [ ! -x "$(which go)" ] ; then
echo "error: go needs to be on \$PATH to use $0"
exit 1
fi
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )"
ROOT_DIR="$( cd "${SCRIPT_DIR}/.." >/dev/null 2>&1 && pwd )"
BINARY="${SCRIPT_DIR}/bin/test-runner"
# Rebuild the binary.
# Note, go caches build artifacts, so this is quick for repeat calls
pushd "${SCRIPT_DIR}/src/cmd/test-runner" > /dev/null
go build -o "${BINARY}" main.go
popd > /dev/null
"${BINARY}" "$@"

79
tools/tint-generate-coverage Executable file
View File

@@ -0,0 +1,79 @@
#!/usr/bin/env bash
# Copyright 2020 The Tint Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# See https://clang.llvm.org/docs/SourceBasedCodeCoverage.html
if [ ! -x "$(which llvm-profdata)" ] ; then
echo "error: llvm-profdata needs to be on \$PATH to use $0"
exit 1
fi
if [ ! -x "$(which llvm-cov)" ] ; then
echo "error: llvm-cov needs to be on \$PATH to use $0"
exit 1
fi
TARGET_EXE=$1
if [ ! -x "$TARGET_EXE" ] ; then
echo "Usage: $0 <executable-path> [optional-args]"
echo ""
echo "Generates a lcov.info file at the project root, which can be used by"
echo "tools such as VSCode's Coverage Gutters extension to visualize code"
echo "coverage in the editor".
exit 1
fi
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )"
ROOT_DIR="$( cd "${SCRIPT_DIR}/.." >/dev/null 2>&1 && pwd )"
PROFRAW_FILE="${ROOT_DIR}/tint.profraw"
PROFDATA_FILE="${ROOT_DIR}/tint.profdata"
LCOV_FILE="${ROOT_DIR}/lcov.info"
SUMMARY_FILE="${ROOT_DIR}/coverage.summary"
# Remove any existing coverage data and intermediate files
if [ -f "$PROFRAW_FILE" ]; then rm ${PROFRAW_FILE}; fi
if [ -f "$PROFDATA_FILE" ]; then rm ${PROFDATA_FILE}; fi
if [ -f "$LCOV_FILE" ]; then rm ${LCOV_FILE}; fi
if [ -f "$SUMMARY_FILE" ]; then rm ${SUMMARY_FILE}; fi
# Run the executable to generate the raw coverage data
# https://clang.llvm.org/docs/SourceBasedCodeCoverage.html#running-the-instrumented-program
LLVM_PROFILE_FILE="${PROFRAW_FILE}" "$@"
# Check that coverage information was generated
if [ ! -f "$PROFRAW_FILE" ]; then
echo "lcov.info was not generated. Is coverage generation enabled?"
echo "To enable, run cmake with -DTINT_EMIT_COVERAGE=1".
exit 1
fi
# Fail on any error after running the target executable
set -e
# Index the coverage data
# https://clang.llvm.org/docs/SourceBasedCodeCoverage.html#creating-coverage-reports
llvm-profdata merge -sparse "${PROFRAW_FILE}" -o "${PROFDATA_FILE}"
# Export as lcov
# https://clang.llvm.org/docs/SourceBasedCodeCoverage.html#exporting-coverage-data
llvm-cov export --instr-profile="${PROFDATA_FILE}" --format=lcov --object=${TARGET_EXE} > "${LCOV_FILE}"
# Generate summary report
llvm-cov report --ignore-filename-regex="(.*_test\.cc|third_party/.*)" --instr-profile="${PROFDATA_FILE}" --object=${TARGET_EXE} > "${SUMMARY_FILE}"
# Clean up intermediate files
rm ${PROFRAW_FILE} ${PROFDATA_FILE}

33
tools/trim-includes Executable file
View File

@@ -0,0 +1,33 @@
#!/usr/bin/env bash
# Copyright 2021 The Tint Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e # Fail on any error.
if [ ! -x "$(which go)" ] ; then
echo "error: go needs to be on \$PATH to use $0"
exit 1
fi
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )"
ROOT_DIR="$( cd "${SCRIPT_DIR}/.." >/dev/null 2>&1 && pwd )"
BINARY="${SCRIPT_DIR}/bin/trim-includes"
# Rebuild the binary.
# Note, go caches build artifacts, so this is quick for repeat calls
pushd "${SCRIPT_DIR}/src/cmd/trim-includes" > /dev/null
go build -o "${BINARY}" main.go
popd > /dev/null
"${BINARY}" "$@"