encounter
/
dawn-cmake
mirror of https://github.com/encounter/dawn-cmake.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[validation] Improve CTS test wrapper. 

Updated to check if failure happened with the intended error code

Change-Id: If8eee7605b8ab1af481fde067dcb0e64e0df6256
Bug:tint:271
Reviewed-on: https://dawn-review.googlesource.com/c/tint/+/30002
Reviewed-by: Ryan Harrison <rharrison@chromium.org>
Commit-Queue: Ryan Harrison <rharrison@chromium.org>
This commit is contained in:
Sarah Mashayekhi 2020-10-21 19:24:00 +00:00 committed by Commit Bot service account
parent 0ce0704741
commit 8f7c80347d
2 changed files with 99 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
tools/known_tint_failures Normal file
View File

@ -0,0 +1,18 @@
break-outside-for-or-switch.fail.wgsl
continue-outside-for.fail.wgsl
duplicate-entry-point.fail.wgsl
duplicate-stuct-name-v2.fail.wgsl
duplicate-stuct-name.fail.wgsl
duplicate-var-name-within-func.fail.wgsl
global-vars-must-be-unique-v2.fail.wgsl
global-vars-must-be-unique-v3.fail.wgsl
runtime-array-not-last.fail.wgsl
self-recursion-v2.fail.wgsl
self-recursion.fail.wgsl
struct-def-before-use.fail.wgsl
struct-member-def-before-use-v2.fail.wgsl
struct-member-def-before-use-v3.fail.wgsl
struct-member-def-before-use-v4.fail.wgsl
struct-member-def-before-use-v5.fail.wgsl
struct-member-def-before-use.fail.wgsl
struct-use-before-def.fail.wgsl

99
tools/run_tests.py
View File

@ -55,6 +55,15 @@ class TestCase:
def GetResult(self, fmt):
return self.results[fmt]
def GetReason(self):
with open(self.GetInputPath()) as test:
first_line = test.readline()
if (first_line.startswith("# v-")):
reason = first_line[2:8]
else:
reason = ''
return reason
"""
The test runner, will execute a series of test cases and record the
@ -81,40 +90,80 @@ class TestRunner:
stderr=subprocess.STDOUT)
except Exception as e:
if not tc.IsExpectedFail():
print("{}".format("".join(map(chr, bytearray(e.output)))))
print(e)
return False
failure_reason = "{}".format("".join(map(chr,
bytearray(e.output))))
if tc.IsExpectedFail():
right_reason = tc.GetReason()
if (right_reason in failure_reason):
return False, ""
else:
return False, right_reason
return True
if not tc.IsExpectedFail():
print(failure_reason)
print(e)
return False, ""
return True, ""
def RunTests(self):
"""Runs a set of test cases"""
for tc in self.test_cases:
result = self.RunTest(tc)
if not tc.IsExpectedFail() and not result:
self.failures.append(tc.GetInputPath())
elif tc.IsExpectedFail() and result:
result, reason = self.RunTest(tc)
"""evaluate final result based on result, tc.IsExpectedFail() and reason"""
if not result:
# result == false, expected true, reason:don't care
if not tc.IsExpectedFail():
print("Expected: " + tc.GetInputPath() +
" to pass but failed.")
self.failures.append(tc.GetInputPath())
# result == false, expected false, reason: wrong
else:
if reason.startswith("v-"):
print("Failed for a wrong reason: " +
tc.GetInputPath() +
" expected with error code: " + reason)
self.failures_wrong_reason.append(tc.GetInputPath())
# result == true, expected false, reason:don't care
elif tc.IsExpectedFail():
print("Expected: " + tc.GetInputPath() +
" to fail but passed.")
self.failures.append(tc.GetInputPath())
def GetUnexpectedFailures(self):
for failure in self.failures + self.failures_wrong_reason:
if failure not in self.known_failures:
self.unexpected_failures.append(failure)
return
def SummarizeResults(self):
"""Prints a summarization of the test results to STDOUT"""
if len(self.unexpected_failures):
self.unexpected_failures.sort()
print('\nSummary of unexpected failures:')
for unexpected_fail in self.unexpected_failures:
print(unexpected_fail)
if len(self.failures) > 0:
self.failures.sort()
for f in self.known_failures:
if f not in self.failures_wrong_reason + self.failures:
self.unexpected_successes.append(f)
print('\nSummary of Failures:')
for failure in self.failures:
print(failure)
if len(self.unexpected_successes):
print('\nSummary of unexpected successes:')
for s in self.unexpected_successes:
print(s)
print('')
print('Test cases executed: {}'.format(len(self.test_cases)))
print(' Successes: {}'.format(
(len(self.test_cases) - len(self.failures))))
print(' Failures: {}'.format(len(self.failures)))
(len(self.test_cases) - len(self.failures) -
len(self.failures_wrong_reason))))
print(' Failures: {}'.format(
len(self.failures) + len(self.failures_wrong_reason)))
print(' Unexpected Failures: {}'.format(len(
self.unexpected_failures)))
print(' Unexpected Successes: {}'.format(
len(self.unexpected_successes)))
print('')
def Run(self):
@ -133,6 +182,10 @@ class TestRunner:
'src', 'webgpu', 'shader',
'validation', 'wgsl'),
help='path to directory containing test files')
parser.add_option(
'--known-failures-file',
default=os.path.join(base_path, 'tools', 'known_tint_failures'),
help='path to directory containing the known failures file')
parser.add_option(
'--test-prog-path',
default=None,
@ -179,13 +232,23 @@ class TestRunner:
if os.path.isfile(input_path):
self.test_cases.append(
TestCase(input_path, self.options.parse_only))
known_failure_file = self.options.known_failures_file
self.known_failures = []
with open(known_failure_file, 'r') as f:
for failure_filename in f.read().splitlines():
self.known_failures.append(
os.path.join(self.options.test_dir, failure_filename))
self.failures = []
self.failures_wrong_reason = []
self.unexpected_failures = []
self.unexpected_successes = []
self.RunTests()
self.GetUnexpectedFailures()
self.SummarizeResults()
return len(self.failures)
return not len(self.unexpected_failures + self.unexpected_successes)
def main():