diff --git a/tools/known_tint_failures b/tools/known_tint_failures deleted file mode 100644 index baf153bee0..0000000000 --- a/tools/known_tint_failures +++ /dev/null @@ -1,12 +0,0 @@ -break-outside-for-or-switch.fail.wgsl -continue-outside-for.fail.wgsl -duplicate-stuct-name-v2.fail.wgsl -duplicate-stuct-name.fail.wgsl -duplicate-var-name-within-func.fail.wgsl -global-vars-must-be-unique-v2.fail.wgsl -global-vars-must-be-unique-v3.fail.wgsl -runtime-array-not-last.fail.wgsl -self-recursion-v2.fail.wgsl -self-recursion.fail.wgsl -struct-member-def-before-use-v3.fail.wgsl -struct-use-before-def.fail.wgsl diff --git a/tools/run_tests.py b/tools/run_tests.py deleted file mode 100755 index ec1bf86ee4..0000000000 --- a/tools/run_tests.py +++ /dev/null @@ -1,260 +0,0 @@ -#!/usr/bin/env python -# Copyright 2020 The Tint Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Test runner for executing a test of tests with Tint. The runner will -# find all .wgsl files in the given folder and attempt to convert them -# to each of the backend formats. If the file contains a '.fail.' in the -# name then the runner will expect the file to fail conversion. - -import base64 -import copy -import difflib -import optparse -import os -import platform -import re -import subprocess -import sys -import tempfile - - -""" -A single test case to be executed. Stores the path to the test file -and the result of executing the test. -""" - - -class TestCase: - def __init__(self, input_path, parse_only): - self.input_path = input_path - self.parse_only = parse_only - self.results = {} - - def IsExpectedFail(self): - fail_re = re.compile('^.+[\.]fail[\.]wgsl') - return fail_re.match(self.GetInputPath()) - - def IsParseOnly(self): - return self.parse_only - - def GetInputPath(self): - return self.input_path - - def GetResult(self, fmt): - return self.results[fmt] - - def GetReason(self): - with open(self.GetInputPath()) as test: - first_line = test.readline() - if (first_line.startswith("# v-")): - reason = first_line[2:8] - else: - reason = '' - return reason - - -""" -The test runner, will execute a series of test cases and record the -results. -""" - - -class TestRunner: - def RunTest(self, tc): - """Runs a single test.""" - print("Testing {}".format(tc.GetInputPath())) - - cmd = [self.options.test_prog_path] - if tc.IsParseOnly(): - cmd += ['--parse-only'] - - languages = ["wgsl", "spvasm", "msl", "hlsl"] - try: - for lang in languages: - lang_cmd = copy.copy(cmd) - lang_cmd += ['--format', lang] - lang_cmd += [tc.GetInputPath()] - err = subprocess.check_output(lang_cmd, - stderr=subprocess.STDOUT) - - except Exception as e: - failure_reason = "{}".format("".join(map(chr, - bytearray(e.output)))) - if tc.IsExpectedFail(): - right_reason = tc.GetReason() - if (right_reason in failure_reason): - return False, "" - else: - return False, right_reason - - if not tc.IsExpectedFail(): - print(failure_reason) - print(e) - return False, "" - - return True, "" - - def RunTests(self): - """Runs a set of test cases""" - for tc in self.test_cases: - result, reason = self.RunTest(tc) - """evaluate final result based on result, tc.IsExpectedFail() and reason""" - if not result: - # result == false, expected true, reason:don't care - if not tc.IsExpectedFail(): - print("Expected: " + tc.GetInputPath() + - " to pass but failed.") - self.failures.append(tc.GetInputPath()) - # result == false, expected false, reason: wrong - else: - if reason.startswith("v-"): - print("Failed for a wrong reason: " + - tc.GetInputPath() + - " expected with error code: " + reason) - self.failures_wrong_reason.append(tc.GetInputPath()) - # result == true, expected false, reason:don't care - elif tc.IsExpectedFail(): - print("Expected: " + tc.GetInputPath() + - " to fail but passed.") - self.failures.append(tc.GetInputPath()) - - def GetUnexpectedFailures(self): - for failure in self.failures + self.failures_wrong_reason: - if failure not in self.known_failures: - self.unexpected_failures.append(failure) - return - - def SummarizeResults(self): - """Prints a summarization of the test results to STDOUT""" - if len(self.unexpected_failures): - self.unexpected_failures.sort() - print('\nSummary of unexpected failures:') - for unexpected_fail in self.unexpected_failures: - print(unexpected_fail) - - for f in self.known_failures: - if f not in self.failures_wrong_reason + self.failures: - self.unexpected_successes.append(f) - - if len(self.unexpected_successes): - print('\nSummary of unexpected successes:') - for s in self.unexpected_successes: - print(s) - - print('') - print('Test cases executed: {}'.format(len(self.test_cases))) - print(' Successes: {}'.format( - (len(self.test_cases) - len(self.failures) - - len(self.failures_wrong_reason)))) - print(' Failures: {}'.format( - len(self.failures) + len(self.failures_wrong_reason))) - print(' Unexpected Failures: {}'.format(len( - self.unexpected_failures))) - print(' Unexpected Successes: {}'.format( - len(self.unexpected_successes))) - print('') - - def Run(self): - """Executes the test runner.""" - base_path = os.path.abspath( - os.path.join(os.path.dirname(__file__), '..')) - - usage = 'usage: %prog [options] (file)' - parser = optparse.OptionParser(usage=usage) - parser.add_option('--build-dir', - default=os.path.join(base_path, 'out', 'Debug'), - help='path to build directory') - parser.add_option('--test-dir', - default=os.path.join(os.path.dirname(__file__), '..', - 'third_party', 'gpuweb-cts', - 'src', 'webgpu', 'shader', - 'validation', 'wgsl'), - help='path to directory containing test files') - parser.add_option( - '--known-failures-file', - default=os.path.join(base_path, 'tools', 'known_tint_failures'), - help='path to directory containing the known failures file') - parser.add_option( - '--test-prog-path', - default=None, - help='path to program to test (default build-dir/tint)') - parser.add_option('--parse-only', - action="store_true", - default=False, - help='only parse test cases; do not compile') - - self.options, self.args = parser.parse_args() - - if self.options.test_prog_path == None: - test_prog = os.path.abspath( - os.path.join(self.options.build_dir, 'tint')) - if not os.path.isfile(test_prog): - print("Cannot find test program {}".format(test_prog)) - return 1 - - self.options.test_prog_path = test_prog - - if not os.path.isfile(self.options.test_prog_path): - print("Cannot find test program '{}'".format( - self.options.test_prog_path)) - return 1 - - input_file_re = re.compile('^.+[\.]wgsl') - self.test_cases = [] - - if self.args: - for filename in self.args: - input_path = os.path.join(self.options.test_dir, filename) - if not os.path.isfile(input_path): - print("Cannot find test file '{}'".format(filename)) - return 1 - - self.test_cases.append( - TestCase(input_path, self.options.parse_only)) - - else: - for file_dir, _, filename_list in os.walk(self.options.test_dir): - for input_filename in filename_list: - if input_file_re.match(input_filename): - input_path = os.path.join(file_dir, input_filename) - if os.path.isfile(input_path): - self.test_cases.append( - TestCase(input_path, self.options.parse_only)) - known_failure_file = self.options.known_failures_file - self.known_failures = [] - with open(known_failure_file, 'r') as f: - for failure_filename in f.read().splitlines(): - self.known_failures.append( - os.path.join(self.options.test_dir, failure_filename)) - - self.failures = [] - self.failures_wrong_reason = [] - self.unexpected_failures = [] - self.unexpected_successes = [] - - self.RunTests() - self.GetUnexpectedFailures() - self.SummarizeResults() - - return not len(self.unexpected_failures + self.unexpected_successes) - - -def main(): - runner = TestRunner() - return runner.Run() - - -if __name__ == '__main__': - sys.exit(main())