tools: Remove old tint test runner
This has been superseded with the end-to-end test runner. Change-Id: I628cf9578313a43d8423abab3dfbacf398a8c6ba Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/86522 Reviewed-by: Dan Sinclair <dsinclair@chromium.org> Commit-Queue: Ben Clayton <bclayton@google.com>
This commit is contained in:
Ben Clayton
Dawn LUCI CQ
parent
c85f8d828e
commit
69d4edd0ad
|
|
@ -1,12 +0,0 @@
|
||||||
break-outside-for-or-switch.fail.wgsl
|
|
||||||
continue-outside-for.fail.wgsl
|
|
||||||
duplicate-stuct-name-v2.fail.wgsl
|
|
||||||
duplicate-stuct-name.fail.wgsl
|
|
||||||
duplicate-var-name-within-func.fail.wgsl
|
|
||||||
global-vars-must-be-unique-v2.fail.wgsl
|
|
||||||
global-vars-must-be-unique-v3.fail.wgsl
|
|
||||||
runtime-array-not-last.fail.wgsl
|
|
||||||
self-recursion-v2.fail.wgsl
|
|
||||||
self-recursion.fail.wgsl
|
|
||||||
struct-member-def-before-use-v3.fail.wgsl
|
|
||||||
struct-use-before-def.fail.wgsl
|
|
||||||
|
|
@ -1,260 +0,0 @@
|
||||||
#!/usr/bin/env python
|
|
||||||
# Copyright 2020 The Tint Authors.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
# Test runner for executing a test of tests with Tint. The runner will
|
|
||||||
# find all .wgsl files in the given folder and attempt to convert them
|
|
||||||
# to each of the backend formats. If the file contains a '.fail.' in the
|
|
||||||
# name then the runner will expect the file to fail conversion.
|
|
||||||
|
|
||||||
import base64
|
|
||||||
import copy
|
|
||||||
import difflib
|
|
||||||
import optparse
|
|
||||||
import os
|
|
||||||
import platform
|
|
||||||
import re
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
import tempfile
|
|
||||||
|
|
||||||
|
|
||||||
"""
|
|
||||||
A single test case to be executed. Stores the path to the test file
|
|
||||||
and the result of executing the test.
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
class TestCase:
|
|
||||||
def __init__(self, input_path, parse_only):
|
|
||||||
self.input_path = input_path
|
|
||||||
self.parse_only = parse_only
|
|
||||||
self.results = {}
|
|
||||||
|
|
||||||
def IsExpectedFail(self):
|
|
||||||
fail_re = re.compile('^.+[\.]fail[\.]wgsl')
|
|
||||||
return fail_re.match(self.GetInputPath())
|
|
||||||
|
|
||||||
def IsParseOnly(self):
|
|
||||||
return self.parse_only
|
|
||||||
|
|
||||||
def GetInputPath(self):
|
|
||||||
return self.input_path
|
|
||||||
|
|
||||||
def GetResult(self, fmt):
|
|
||||||
return self.results[fmt]
|
|
||||||
|
|
||||||
def GetReason(self):
|
|
||||||
with open(self.GetInputPath()) as test:
|
|
||||||
first_line = test.readline()
|
|
||||||
if (first_line.startswith("# v-")):
|
|
||||||
reason = first_line[2:8]
|
|
||||||
else:
|
|
||||||
reason = ''
|
|
||||||
return reason
|
|
||||||
|
|
||||||
|
|
||||||
"""
|
|
||||||
The test runner, will execute a series of test cases and record the
|
|
||||||
results.
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
class TestRunner:
|
|
||||||
def RunTest(self, tc):
|
|
||||||
"""Runs a single test."""
|
|
||||||
print("Testing {}".format(tc.GetInputPath()))
|
|
||||||
|
|
||||||
cmd = [self.options.test_prog_path]
|
|
||||||
if tc.IsParseOnly():
|
|
||||||
cmd += ['--parse-only']
|
|
||||||
|
|
||||||
languages = ["wgsl", "spvasm", "msl", "hlsl"]
|
|
||||||
try:
|
|
||||||
for lang in languages:
|
|
||||||
lang_cmd = copy.copy(cmd)
|
|
||||||
lang_cmd += ['--format', lang]
|
|
||||||
lang_cmd += [tc.GetInputPath()]
|
|
||||||
err = subprocess.check_output(lang_cmd,
|
|
||||||
stderr=subprocess.STDOUT)
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
failure_reason = "{}".format("".join(map(chr,
|
|
||||||
bytearray(e.output))))
|
|
||||||
if tc.IsExpectedFail():
|
|
||||||
right_reason = tc.GetReason()
|
|
||||||
if (right_reason in failure_reason):
|
|
||||||
return False, ""
|
|
||||||
else:
|
|
||||||
return False, right_reason
|
|
||||||
|
|
||||||
if not tc.IsExpectedFail():
|
|
||||||
print(failure_reason)
|
|
||||||
print(e)
|
|
||||||
return False, ""
|
|
||||||
|
|
||||||
return True, ""
|
|
||||||
|
|
||||||
def RunTests(self):
|
|
||||||
"""Runs a set of test cases"""
|
|
||||||
for tc in self.test_cases:
|
|
||||||
result, reason = self.RunTest(tc)
|
|
||||||
"""evaluate final result based on result, tc.IsExpectedFail() and reason"""
|
|
||||||
if not result:
|
|
||||||
# result == false, expected true, reason:don't care
|
|
||||||
if not tc.IsExpectedFail():
|
|
||||||
print("Expected: " + tc.GetInputPath() +
|
|
||||||
" to pass but failed.")
|
|
||||||
self.failures.append(tc.GetInputPath())
|
|
||||||
# result == false, expected false, reason: wrong
|
|
||||||
else:
|
|
||||||
if reason.startswith("v-"):
|
|
||||||
print("Failed for a wrong reason: " +
|
|
||||||
tc.GetInputPath() +
|
|
||||||
" expected with error code: " + reason)
|
|
||||||
self.failures_wrong_reason.append(tc.GetInputPath())
|
|
||||||
# result == true, expected false, reason:don't care
|
|
||||||
elif tc.IsExpectedFail():
|
|
||||||
print("Expected: " + tc.GetInputPath() +
|
|
||||||
" to fail but passed.")
|
|
||||||
self.failures.append(tc.GetInputPath())
|
|
||||||
|
|
||||||
def GetUnexpectedFailures(self):
|
|
||||||
for failure in self.failures + self.failures_wrong_reason:
|
|
||||||
if failure not in self.known_failures:
|
|
||||||
self.unexpected_failures.append(failure)
|
|
||||||
return
|
|
||||||
|
|
||||||
def SummarizeResults(self):
|
|
||||||
"""Prints a summarization of the test results to STDOUT"""
|
|
||||||
if len(self.unexpected_failures):
|
|
||||||
self.unexpected_failures.sort()
|
|
||||||
print('\nSummary of unexpected failures:')
|
|
||||||
for unexpected_fail in self.unexpected_failures:
|
|
||||||
print(unexpected_fail)
|
|
||||||
|
|
||||||
for f in self.known_failures:
|
|
||||||
if f not in self.failures_wrong_reason + self.failures:
|
|
||||||
self.unexpected_successes.append(f)
|
|
||||||
|
|
||||||
if len(self.unexpected_successes):
|
|
||||||
print('\nSummary of unexpected successes:')
|
|
||||||
for s in self.unexpected_successes:
|
|
||||||
print(s)
|
|
||||||
|
|
||||||
print('')
|
|
||||||
print('Test cases executed: {}'.format(len(self.test_cases)))
|
|
||||||
print(' Successes: {}'.format(
|
|
||||||
(len(self.test_cases) - len(self.failures) -
|
|
||||||
len(self.failures_wrong_reason))))
|
|
||||||
print(' Failures: {}'.format(
|
|
||||||
len(self.failures) + len(self.failures_wrong_reason)))
|
|
||||||
print(' Unexpected Failures: {}'.format(len(
|
|
||||||
self.unexpected_failures)))
|
|
||||||
print(' Unexpected Successes: {}'.format(
|
|
||||||
len(self.unexpected_successes)))
|
|
||||||
print('')
|
|
||||||
|
|
||||||
def Run(self):
|
|
||||||
"""Executes the test runner."""
|
|
||||||
base_path = os.path.abspath(
|
|
||||||
os.path.join(os.path.dirname(__file__), '..'))
|
|
||||||
|
|
||||||
usage = 'usage: %prog [options] (file)'
|
|
||||||
parser = optparse.OptionParser(usage=usage)
|
|
||||||
parser.add_option('--build-dir',
|
|
||||||
default=os.path.join(base_path, 'out', 'Debug'),
|
|
||||||
help='path to build directory')
|
|
||||||
parser.add_option('--test-dir',
|
|
||||||
default=os.path.join(os.path.dirname(__file__), '..',
|
|
||||||
'third_party', 'gpuweb-cts',
|
|
||||||
'src', 'webgpu', 'shader',
|
|
||||||
'validation', 'wgsl'),
|
|
||||||
help='path to directory containing test files')
|
|
||||||
parser.add_option(
|
|
||||||
'--known-failures-file',
|
|
||||||
default=os.path.join(base_path, 'tools', 'known_tint_failures'),
|
|
||||||
help='path to directory containing the known failures file')
|
|
||||||
parser.add_option(
|
|
||||||
'--test-prog-path',
|
|
||||||
default=None,
|
|
||||||
help='path to program to test (default build-dir/tint)')
|
|
||||||
parser.add_option('--parse-only',
|
|
||||||
action="store_true",
|
|
||||||
default=False,
|
|
||||||
help='only parse test cases; do not compile')
|
|
||||||
|
|
||||||
self.options, self.args = parser.parse_args()
|
|
||||||
|
|
||||||
if self.options.test_prog_path == None:
|
|
||||||
test_prog = os.path.abspath(
|
|
||||||
os.path.join(self.options.build_dir, 'tint'))
|
|
||||||
if not os.path.isfile(test_prog):
|
|
||||||
print("Cannot find test program {}".format(test_prog))
|
|
||||||
return 1
|
|
||||||
|
|
||||||
self.options.test_prog_path = test_prog
|
|
||||||
|
|
||||||
if not os.path.isfile(self.options.test_prog_path):
|
|
||||||
print("Cannot find test program '{}'".format(
|
|
||||||
self.options.test_prog_path))
|
|
||||||
return 1
|
|
||||||
|
|
||||||
input_file_re = re.compile('^.+[\.]wgsl')
|
|
||||||
self.test_cases = []
|
|
||||||
|
|
||||||
if self.args:
|
|
||||||
for filename in self.args:
|
|
||||||
input_path = os.path.join(self.options.test_dir, filename)
|
|
||||||
if not os.path.isfile(input_path):
|
|
||||||
print("Cannot find test file '{}'".format(filename))
|
|
||||||
return 1
|
|
||||||
|
|
||||||
self.test_cases.append(
|
|
||||||
TestCase(input_path, self.options.parse_only))
|
|
||||||
|
|
||||||
else:
|
|
||||||
for file_dir, _, filename_list in os.walk(self.options.test_dir):
|
|
||||||
for input_filename in filename_list:
|
|
||||||
if input_file_re.match(input_filename):
|
|
||||||
input_path = os.path.join(file_dir, input_filename)
|
|
||||||
if os.path.isfile(input_path):
|
|
||||||
self.test_cases.append(
|
|
||||||
TestCase(input_path, self.options.parse_only))
|
|
||||||
known_failure_file = self.options.known_failures_file
|
|
||||||
self.known_failures = []
|
|
||||||
with open(known_failure_file, 'r') as f:
|
|
||||||
for failure_filename in f.read().splitlines():
|
|
||||||
self.known_failures.append(
|
|
||||||
os.path.join(self.options.test_dir, failure_filename))
|
|
||||||
|
|
||||||
self.failures = []
|
|
||||||
self.failures_wrong_reason = []
|
|
||||||
self.unexpected_failures = []
|
|
||||||
self.unexpected_successes = []
|
|
||||||
|
|
||||||
self.RunTests()
|
|
||||||
self.GetUnexpectedFailures()
|
|
||||||
self.SummarizeResults()
|
|
||||||
|
|
||||||
return not len(self.unexpected_failures + self.unexpected_successes)
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
runner = TestRunner()
|
|
||||||
return runner.Run()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
sys.exit(main())
|
|
||||||
Loading…
Reference in New Issue