[validation] Improve CTS test wrapper.

Updated to check if failure happened with the intended error code

Change-Id: If8eee7605b8ab1af481fde067dcb0e64e0df6256
Bug:tint:271
Reviewed-on: https://dawn-review.googlesource.com/c/tint/+/30002
Reviewed-by: Ryan Harrison <rharrison@chromium.org>
Commit-Queue: Ryan Harrison <rharrison@chromium.org>

tools/known_tint_failures

@@ -0,0 +1,18 @@
+break-outside-for-or-switch.fail.wgsl
+continue-outside-for.fail.wgsl
+duplicate-entry-point.fail.wgsl
+duplicate-stuct-name-v2.fail.wgsl
+duplicate-stuct-name.fail.wgsl
+duplicate-var-name-within-func.fail.wgsl
+global-vars-must-be-unique-v2.fail.wgsl
+global-vars-must-be-unique-v3.fail.wgsl
+runtime-array-not-last.fail.wgsl
+self-recursion-v2.fail.wgsl
+self-recursion.fail.wgsl
+struct-def-before-use.fail.wgsl
+struct-member-def-before-use-v2.fail.wgsl
+struct-member-def-before-use-v3.fail.wgsl
+struct-member-def-before-use-v4.fail.wgsl
+struct-member-def-before-use-v5.fail.wgsl
+struct-member-def-before-use.fail.wgsl
+struct-use-before-def.fail.wgsl

tools/run_tests.py

@@ -55,6 +55,15 @@ class TestCase:
     def GetResult(self, fmt):
         return self.results[fmt]
 
+    def GetReason(self):
+        with open(self.GetInputPath()) as test:
+            first_line = test.readline()
+        if (first_line.startswith("# v-")):
+            reason = first_line[2:8]
+        else:
+            reason = ''
+        return reason
+
 
 """
 The test runner, will execute a series of test cases and record the
@@ -81,40 +90,80 @@ class TestRunner:
                                               stderr=subprocess.STDOUT)
 
         except Exception as e:
-            if not tc.IsExpectedFail():
+            failure_reason = "{}".format("".join(map(chr,
-                print("{}".format("".join(map(chr, bytearray(e.output)))))
+                                                     bytearray(e.output))))
-                print(e)
+            if tc.IsExpectedFail():
-            return False
+                right_reason = tc.GetReason()
+                if (right_reason in failure_reason):
+                    return False, ""
+                else:
+                    return False, right_reason
 
-        return True
+            if not tc.IsExpectedFail():
+                print(failure_reason)
+                print(e)
+                return False, ""
+
+        return True, ""
 
     def RunTests(self):
         """Runs a set of test cases"""
         for tc in self.test_cases:
-            result = self.RunTest(tc)
+            result, reason = self.RunTest(tc)
-
+            """evaluate final result based on result, tc.IsExpectedFail() and reason"""
-            if not tc.IsExpectedFail() and not result:
+            if not result:
-                self.failures.append(tc.GetInputPath())
+                # result == false, expected true, reason:don't care
-            elif tc.IsExpectedFail() and result:
+                if not tc.IsExpectedFail():
+                    print("Expected: " + tc.GetInputPath() +
+                          " to pass but failed.")
+                    self.failures.append(tc.GetInputPath())
+                # result == false, expected false, reason: wrong
+                else:
+                    if reason.startswith("v-"):
+                        print("Failed for a wrong reason: " +
+                              tc.GetInputPath() +
+                              " expected with error code: " + reason)
+                        self.failures_wrong_reason.append(tc.GetInputPath())
+            # result == true, expected false, reason:don't care
+            elif tc.IsExpectedFail():
                 print("Expected: " + tc.GetInputPath() +
                       " to fail but passed.")
                 self.failures.append(tc.GetInputPath())
 
+    def GetUnexpectedFailures(self):
+        for failure in self.failures + self.failures_wrong_reason:
+            if failure not in self.known_failures:
+                self.unexpected_failures.append(failure)
+        return
+
     def SummarizeResults(self):
         """Prints a summarization of the test results to STDOUT"""
+        if len(self.unexpected_failures):
+            self.unexpected_failures.sort()
+            print('\nSummary of unexpected failures:')
+            for unexpected_fail in self.unexpected_failures:
+                print(unexpected_fail)
 
-        if len(self.failures) > 0:
+        for f in self.known_failures:
-            self.failures.sort()
+            if f not in self.failures_wrong_reason + self.failures:
+                self.unexpected_successes.append(f)
 
-            print('\nSummary of Failures:')
+        if len(self.unexpected_successes):
-            for failure in self.failures:
+            print('\nSummary of unexpected successes:')
-                print(failure)
+            for s in self.unexpected_successes:
+                print(s)
 
         print('')
         print('Test cases executed: {}'.format(len(self.test_cases)))
         print('  Successes:  {}'.format(
-            (len(self.test_cases) - len(self.failures))))
+            (len(self.test_cases) - len(self.failures) -
-        print('  Failures:   {}'.format(len(self.failures)))
+             len(self.failures_wrong_reason))))
+        print('  Failures:   {}'.format(
+            len(self.failures) + len(self.failures_wrong_reason)))
+        print('  Unexpected Failures:  {}'.format(len(
+            self.unexpected_failures)))
+        print('  Unexpected Successes:  {}'.format(
+            len(self.unexpected_successes)))
         print('')
 
     def Run(self):
@@ -133,6 +182,10 @@ class TestRunner:
                                                'src', 'webgpu', 'shader',
                                                'validation', 'wgsl'),
                           help='path to directory containing test files')
+        parser.add_option(
+            '--known-failures-file',
+            default=os.path.join(base_path, 'tools', 'known_tint_failures'),
+            help='path to directory containing the known failures file')
         parser.add_option(
             '--test-prog-path',
             default=None,
@@ -179,13 +232,23 @@ class TestRunner:
                         if os.path.isfile(input_path):
                             self.test_cases.append(
                                 TestCase(input_path, self.options.parse_only))
+            known_failure_file = self.options.known_failures_file
+            self.known_failures = []
+            with open(known_failure_file, 'r') as f:
+                for failure_filename in f.read().splitlines():
+                    self.known_failures.append(
+                        os.path.join(self.options.test_dir, failure_filename))
 
         self.failures = []
+        self.failures_wrong_reason = []
+        self.unexpected_failures = []
+        self.unexpected_successes = []
 
         self.RunTests()
+        self.GetUnexpectedFailures()
         self.SummarizeResults()
 
-        return len(self.failures)
+        return not len(self.unexpected_failures + self.unexpected_successes)
 
 
 def main():