Add simple test runner.

This CL adds a script to run tint over the shaders in a given folder and
attempt to generate the WGSL, HLSL, MSL and SPIRV-ASM shaders. The
GPUWeb CTS is added to third_party and the validation folder set as the
default folder to execute.

Change-Id: I63a0af056416e2f99ed8e3f92f9e2ca31c2b3e49
Reviewed-on: https://dawn-review.googlesource.com/c/tint/+/25561
Reviewed-by: Sarah Mashayekhi <sarahmashay@google.com>
Commit-Queue: Sarah Mashayekhi <sarahmashay@google.com>

.gitignore

@@ -11,6 +11,7 @@ testing
 third_party/cpplint
 third_party/binutils
 third_party/googletest
+third_party/gpuweb-cts
 third_party/llvm-build
 third_party/spirv-headers
 third_party/spirv-tools

DEPS

@@ -20,6 +20,7 @@ vars = {
   'clang_revision': '6412135b3979b680c20cf007ab242d968025fc3e',
   'cpplint_revision': '305ac8725a166ed42e3f5dd3f80d6de2cf840ef1',
   'googletest_revision': 'a781fe29bcf73003559a3583167fe3d647518464',
+  'gpuweb_cts_revision': '40e337a38784ad72fa5c7b9afd1b9c358a9e0f1a',
   'spirv_headers_revision': '308bd07424350a6000f35a77b5f85cd4f3da319e',
   'spirv_tools_revision': '717e7877cac15d393fd3bb1bd872679de8b59add',
   'testing_revision': 'cadd4e1eb3a45f562cc7eb5cd646c9b6f91c87dc',
@@ -29,6 +30,9 @@ deps = {
   'third_party/cpplint': Var('chromium_git') + Var('github') +
       '/google/styleguide.git@' + Var('cpplint_revision'),
 
+  'third_party/gpuweb-cts': Var('chromium_git') + Var('github') +
+      '/gpuweb/cts.git@' + Var('gpuweb_cts_revision'),
+
   'third_party/spirv-headers': Var('chromium_git') + Var('github') +
       '/KhronosGroup/SPIRV-Headers.git@' + Var('spirv_headers_revision'),
 

tools/run_tests.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python
+# Copyright 2020 The Tint Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+import difflib
+import optparse
+import os
+import platform
+import re
+import subprocess
+import sys
+import tempfile
+
+
+class TestCase:
+    def __init__(self, input_path, parse_only):
+        self.input_path = input_path
+        self.parse_only = parse_only
+        self.results = {}
+
+    def IsExpectedFail(self):
+        fail_re = re.compile('^.+[\.]fail[\.]wgsl')
+        return fail_re.match(self.GetInputPath())
+
+    def IsParseOnly(self):
+        return self.parse_only
+
+    def GetInputPath(self):
+        return self.input_path
+
+    def GetResult(self, fmt):
+        return self.results[fmt]
+
+
+class TestRunner:
+    def RunTest(self, tc):
+        print("Testing {}".format(tc.GetInputPath()))
+
+        cmd = [self.options.test_prog_path]
+        if tc.IsParseOnly():
+            cmd += ['--parse-only']
+
+        languages = ["wgsl", "spvasm", "msl", "hlsl"]
+        try:
+            for lang in languages:
+                lang_cmd = cmd.copy()
+                lang_cmd += ['--format', lang]
+                lang_cmd += [tc.GetInputPath()]
+                err = subprocess.check_output(lang_cmd,
+                                              stderr=subprocess.STDOUT)
+
+        except Exception as e:
+            if not tc.IsExpectedFail():
+                print("{}".format("".join(map(chr, bytearray(e.output)))))
+                print(e)
+            return False
+
+        return True
+
+    def RunTests(self):
+        for tc in self.test_cases:
+            result = self.RunTest(tc)
+
+            if not tc.IsExpectedFail() and not result:
+                self.failures.append(tc.GetInputPath())
+            elif tc.IsExpectedFail() and result:
+                print("Expected: " + tc.GetInputPath() +
+                      " to fail but passed.")
+                self.failures.append(tc.GetInputPath())
+
+    def SummarizeResults(self):
+        if len(self.failures) > 0:
+            self.failures.sort()
+
+            print('\nSummary of Failures:')
+            for failure in self.failures:
+                print(failure)
+
+        print('')
+        print('Test cases executed: {}'.format(len(self.test_cases)))
+        print('  Successes:  {}'.format(
+            (len(self.test_cases) - len(self.failures))))
+        print('  Failures:   {}'.format(len(self.failures)))
+        print('')
+
+    def Run(self):
+        base_path = os.path.abspath(
+            os.path.join(os.path.dirname(__file__), '..'))
+
+        usage = 'usage: %prog [options] (file)'
+        parser = optparse.OptionParser(usage=usage)
+        parser.add_option('--build-dir',
+                          default=os.path.join(base_path, 'out', 'Debug'),
+                          help='path to build directory')
+        parser.add_option('--test-dir',
+                          default=os.path.join(os.path.dirname(__file__), '..',
+                                               'third_party', 'gpuweb-cts',
+                                               'src', 'webgpu', 'shader',
+                                               'validation', 'wgsl'),
+                          help='path to directory containing test files')
+        parser.add_option('--test-prog-path',
+                          default=None,
+                          help='path to program to test')
+        parser.add_option('--parse-only',
+                          action="store_true",
+                          default=False,
+                          help='only parse test cases; do not compile')
+
+        self.options, self.args = parser.parse_args()
+
+        if self.options.test_prog_path == None:
+            test_prog = os.path.abspath(
+                os.path.join(self.options.build_dir, 'tint'))
+            if not os.path.isfile(test_prog):
+                print("Cannot find test program {}".format(test_prog))
+                return 1
+
+            self.options.test_prog_path = test_prog
+
+        if not os.path.isfile(self.options.test_prog_path):
+            print("--test-prog-path must point to an executable")
+            return 1
+
+        input_file_re = re.compile('^.+[\.]wgsl')
+        self.test_cases = []
+
+        if self.args:
+            for filename in self.args:
+                input_path = os.path.join(self.options.test_dir, filename)
+                if not os.path.isfile(input_path):
+                    print("Cannot find test file '{}'".format(filename))
+                    return 1
+
+                self.test_cases.append(
+                    TestCase(input_path, self.options.parse_only))
+
+        else:
+            for file_dir, _, filename_list in os.walk(self.options.test_dir):
+                for input_filename in filename_list:
+                    if input_file_re.match(input_filename):
+                        input_path = os.path.join(file_dir, input_filename)
+                        if os.path.isfile(input_path):
+                            self.test_cases.append(
+                                TestCase(input_path, self.options.parse_only))
+
+        self.failures = []
+
+        self.RunTests()
+        self.SummarizeResults()
+
+        return len(self.failures) != 0
+
+
+def main():
+    runner = TestRunner()
+    return runner.Run()
+
+
+if __name__ == '__main__':
+    sys.exit(main())