261 lines
9.3 KiB
Python
261 lines
9.3 KiB
Python
|
#!/usr/bin/env python
|
||
|
|
# Copyright 2020 The Tint Authors.
|
||
|
|
#
|
||
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||
|
|
# you may not use this file except in compliance with the License.
|
||
|
|
# You may obtain a copy of the License at
|
||
|
|
#
|
||
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||
|
|
#
|
||
|
|
# Unless required by applicable law or agreed to in writing, software
|
||
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
|
|
# See the License for the specific language governing permissions and
|
||
|
|
# limitations under the License.
|
||
|
|
|
||
|
|
# Test runner for executing a test of tests with Tint. The runner will
|
||
|
|
# find all .wgsl files in the given folder and attempt to convert them
|
||
|
|
# to each of the backend formats. If the file contains a '.fail.' in the
|
||
|
|
# name then the runner will expect the file to fail conversion.
|
||
|
|
|
||
|
|
import base64
|
||
|
|
import copy
|
||
|
|
import difflib
|
||
|
|
import optparse
|
||
|
|
import os
|
||
|
|
import platform
|
||
|
|
import re
|
||
|
|
import subprocess
|
||
|
|
import sys
|
||
|
|
import tempfile
|
||
|
|
|
||
|
|
|
||
|
|
"""
|
||
|
|
A single test case to be executed. Stores the path to the test file
|
||
|
|
and the result of executing the test.
|
||
|
|
"""
|
||
|
|
|
||
|
|
|
||
|
|
class TestCase:
|
||
|
|
def __init__(self, input_path, parse_only):
|
||
|
|
self.input_path = input_path
|
||
|
|
self.parse_only = parse_only
|
||
|
|
self.results = {}
|
||
|
|
|
||
|
|
def IsExpectedFail(self):
|
||
|
|
fail_re = re.compile('^.+[\.]fail[\.]wgsl')
|
||
|
|
return fail_re.match(self.GetInputPath())
|
||
|
|
|
||
|
|
def IsParseOnly(self):
|
||
|
|
return self.parse_only
|
||
|
|
|
||
|
|
def GetInputPath(self):
|
||
|
|
return self.input_path
|
||
|
|
|
||
|
|
def GetResult(self, fmt):
|
||
|
|
return self.results[fmt]
|
||
|
|
|
||
|
|
def GetReason(self):
|
||
|
|
with open(self.GetInputPath()) as test:
|
||
|
|
first_line = test.readline()
|
||
|
|
if (first_line.startswith("# v-")):
|
||
|
|
reason = first_line[2:8]
|
||
|
|
else:
|
||
|
|
reason = ''
|
||
|
|
return reason
|
||
|
|
|
||
|
|
|
||
|
|
"""
|
||
|
|
The test runner, will execute a series of test cases and record the
|
||
|
|
results.
|
||
|
|
"""
|
||
|
|
|
||
|
|
|
||
|
|
class TestRunner:
|
||
|
|
def RunTest(self, tc):
|
||
|
|
"""Runs a single test."""
|
||
|
|
print("Testing {}".format(tc.GetInputPath()))
|
||
|
|
|
||
|
|
cmd = [self.options.test_prog_path]
|
||
|
|
if tc.IsParseOnly():
|
||
|
|
cmd += ['--parse-only']
|
||
|
|
|
||
|
|
languages = ["wgsl", "spvasm", "msl", "hlsl"]
|
||
|
|
try:
|
||
|
|
for lang in languages:
|
||
|
|
lang_cmd = copy.copy(cmd)
|
||
|
|
lang_cmd += ['--format', lang]
|
||
|
|
lang_cmd += [tc.GetInputPath()]
|
||
|
|
err = subprocess.check_output(lang_cmd,
|
||
|
|
stderr=subprocess.STDOUT)
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
failure_reason = "{}".format("".join(map(chr,
|
||
|
|
bytearray(e.output))))
|
||
|
|
if tc.IsExpectedFail():
|
||
|
|
right_reason = tc.GetReason()
|
||
|
|
if (right_reason in failure_reason):
|
||
|
|
return False, ""
|
||
|
|
else:
|
||
|
|
return False, right_reason
|
||
|
|
|
||
|
|
if not tc.IsExpectedFail():
|
||
|
|
print(failure_reason)
|
||
|
|
print(e)
|
||
|
|
return False, ""
|
||
|
|
|
||
|
|
return True, ""
|
||
|
|
|
||
|
|
def RunTests(self):
|
||
|
|
"""Runs a set of test cases"""
|
||
|
|
for tc in self.test_cases:
|
||
|
|
result, reason = self.RunTest(tc)
|
||
|
|
"""evaluate final result based on result, tc.IsExpectedFail() and reason"""
|
||
|
|
if not result:
|
||
|
|
# result == false, expected true, reason:don't care
|
||
|
|
if not tc.IsExpectedFail():
|
||
|
|
print("Expected: " + tc.GetInputPath() +
|
||
|
|
" to pass but failed.")
|
||
|
|
self.failures.append(tc.GetInputPath())
|
||
|
|
# result == false, expected false, reason: wrong
|
||
|
|
else:
|
||
|
|
if reason.startswith("v-"):
|
||
|
|
print("Failed for a wrong reason: " +
|
||
|
|
tc.GetInputPath() +
|
||
|
|
" expected with error code: " + reason)
|
||
|
|
self.failures_wrong_reason.append(tc.GetInputPath())
|
||
|
|
# result == true, expected false, reason:don't care
|
||
|
|
elif tc.IsExpectedFail():
|
||
|
|
print("Expected: " + tc.GetInputPath() +
|
||
|
|
" to fail but passed.")
|
||
|
|
self.failures.append(tc.GetInputPath())
|
||
|
|
|
||
|
|
def GetUnexpectedFailures(self):
|
||
|
|
for failure in self.failures + self.failures_wrong_reason:
|
||
|
|
if failure not in self.known_failures:
|
||
|
|
self.unexpected_failures.append(failure)
|
||
|
|
return
|
||
|
|
|
||
|
|
def SummarizeResults(self):
|
||
|
|
"""Prints a summarization of the test results to STDOUT"""
|
||
|
|
if len(self.unexpected_failures):
|
||
|
|
self.unexpected_failures.sort()
|
||
|
|
print('\nSummary of unexpected failures:')
|
||
|
|
for unexpected_fail in self.unexpected_failures:
|
||
|
|
print(unexpected_fail)
|
||
|
|
|
||
|
|
for f in self.known_failures:
|
||
|
|
if f not in self.failures_wrong_reason + self.failures:
|
||
|
|
self.unexpected_successes.append(f)
|
||
|
|
|
||
|
|
if len(self.unexpected_successes):
|
||
|
|
print('\nSummary of unexpected successes:')
|
||
|
|
for s in self.unexpected_successes:
|
||
|
|
print(s)
|
||
|
|
|
||
|
|
print('')
|
||
|
|
print('Test cases executed: {}'.format(len(self.test_cases)))
|
||
|
|
print(' Successes: {}'.format(
|
||
|
|
(len(self.test_cases) - len(self.failures) -
|
||
|
|
len(self.failures_wrong_reason))))
|
||
|
|
print(' Failures: {}'.format(
|
||
|
|
len(self.failures) + len(self.failures_wrong_reason)))
|
||
|
|
print(' Unexpected Failures: {}'.format(len(
|
||
|
|
self.unexpected_failures)))
|
||
|
|
print(' Unexpected Successes: {}'.format(
|
||
|
|
len(self.unexpected_successes)))
|
||
|
|
print('')
|
||
|
|
|
||
|
|
def Run(self):
|
||
|
|
"""Executes the test runner."""
|
||
|
|
base_path = os.path.abspath(
|
||
|
|
os.path.join(os.path.dirname(__file__), '..'))
|
||
|
|
|
||
|
|
usage = 'usage: %prog [options] (file)'
|
||
|
|
parser = optparse.OptionParser(usage=usage)
|
||
|
|
parser.add_option('--build-dir',
|
||
|
|
default=os.path.join(base_path, 'out', 'Debug'),
|
||
|
|
help='path to build directory')
|
||
|
|
parser.add_option('--test-dir',
|
||
|
|
default=os.path.join(os.path.dirname(__file__), '..',
|
||
|
|
'third_party', 'gpuweb-cts',
|
||
|
|
'src', 'webgpu', 'shader',
|
||
|
|
'validation', 'wgsl'),
|
||
|
|
help='path to directory containing test files')
|
||
|
|
parser.add_option(
|
||
|
|
'--known-failures-file',
|
||
|
|
default=os.path.join(base_path, 'tools', 'known_tint_failures'),
|
||
|
|
help='path to directory containing the known failures file')
|
||
|
|
parser.add_option(
|
||
|
|
'--test-prog-path',
|
||
|
|
default=None,
|
||
|
|
help='path to program to test (default build-dir/tint)')
|
||
|
|
parser.add_option('--parse-only',
|
||
|
|
action="store_true",
|
||
|
|
default=False,
|
||
|
|
help='only parse test cases; do not compile')
|
||
|
|
|
||
|
|
self.options, self.args = parser.parse_args()
|
||
|
|
|
||
|
|
if self.options.test_prog_path == None:
|
||
|
|
test_prog = os.path.abspath(
|
||
|
|
os.path.join(self.options.build_dir, 'tint'))
|
||
|
|
if not os.path.isfile(test_prog):
|
||
|
|
print("Cannot find test program {}".format(test_prog))
|
||
|
|
return 1
|
||
|
|
|
||
|
|
self.options.test_prog_path = test_prog
|
||
|
|
|
||
|
|
if not os.path.isfile(self.options.test_prog_path):
|
||
|
|
print("Cannot find test program '{}'".format(
|
||
|
|
self.options.test_prog_path))
|
||
|
|
return 1
|
||
|
|
|
||
|
|
input_file_re = re.compile('^.+[\.]wgsl')
|
||
|
|
self.test_cases = []
|
||
|
|
|
||
|
|
if self.args:
|
||
|
|
for filename in self.args:
|
||
|
|
input_path = os.path.join(self.options.test_dir, filename)
|
||
|
|
if not os.path.isfile(input_path):
|
||
|
|
print("Cannot find test file '{}'".format(filename))
|
||
|
|
return 1
|
||
|
|
|
||
|
|
self.test_cases.append(
|
||
|
|
TestCase(input_path, self.options.parse_only))
|
||
|
|
|
||
|
|
else:
|
||
|
|
for file_dir, _, filename_list in os.walk(self.options.test_dir):
|
||
|
|
for input_filename in filename_list:
|
||
|
|
if input_file_re.match(input_filename):
|
||
|
|
input_path = os.path.join(file_dir, input_filename)
|
||
|
|
if os.path.isfile(input_path):
|
||
|
|
self.test_cases.append(
|
||
|
|
TestCase(input_path, self.options.parse_only))
|
||
|
|
known_failure_file = self.options.known_failures_file
|
||
|
|
self.known_failures = []
|
||
|
|
with open(known_failure_file, 'r') as f:
|
||
|
|
for failure_filename in f.read().splitlines():
|
||
|
|
self.known_failures.append(
|
||
|
|
os.path.join(self.options.test_dir, failure_filename))
|
||
|
|
|
||
|
|
self.failures = []
|
||
|
|
self.failures_wrong_reason = []
|
||
|
|
self.unexpected_failures = []
|
||
|
|
self.unexpected_successes = []
|
||
|
|
|
||
|
|
self.RunTests()
|
||
|
|
self.GetUnexpectedFailures()
|
||
|
|
self.SummarizeResults()
|
||
|
|
|
||
|
|
return not len(self.unexpected_failures + self.unexpected_successes)
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
runner = TestRunner()
|
||
|
|
return runner.Run()
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
sys.exit(main())
|