Add CTS telemetry harness support

Migrates harness support files added in
https://chromium-review.googlesource.com/c/chromium/src/+/3541414
into Dawn's repo.

Tested in
https://chromium-review.googlesource.com/c/chromium/src/+/3537743

Bug: chromium:1306640
Change-Id: I3000b1223219a1da293af910bf442570b70b7c92
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/84043
Reviewed-by: Loko Kung <lokokung@google.com>
Commit-Queue: Austin Eng <enga@chromium.org>

webgpu-cts/scripts/compile_src.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python
+#
+# Copyright 2022 The Dawn Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import shutil
+import sys
+
+from dir_paths import webgpu_cts_root_dir, node_dir
+from tsc_ignore_errors import run_tsc_ignore_errors
+
+try:
+    old_sys_path = sys.path
+    sys.path = [node_dir] + sys.path
+
+    from node import RunNode
+finally:
+    sys.path = old_sys_path
+
+
+def compile_src(out_dir):
+    # First, clean the output directory so deleted files are pruned from old builds.
+    shutil.rmtree(out_dir)
+
+    run_tsc_ignore_errors([
+        '--project',
+        os.path.join(webgpu_cts_root_dir, 'tsconfig.json'),
+        '--outDir',
+        out_dir,
+        '--noEmit',
+        'false',
+        '--noEmitOnError',
+        'false',
+        '--declaration',
+        'false',
+        '--sourceMap',
+        'false',
+        '--target',
+        'ES2017',
+    ])
+
+
+def compile_src_for_node(out_dir, additional_args=None, clean=True):
+    additional_args = additional_args or []
+    if clean:
+        # First, clean the output directory so deleted files are pruned from old builds.
+        shutil.rmtree(out_dir)
+
+    args = [
+        '--project',
+        os.path.join(webgpu_cts_root_dir, 'node.tsconfig.json'),
+        '--outDir',
+        out_dir,
+        '--noEmit',
+        'false',
+        '--noEmitOnError',
+        'false',
+        '--declaration',
+        'false',
+        '--sourceMap',
+        'false',
+        '--target',
+        'ES6',
+    ]
+    args.extend(additional_args)
+
+    run_tsc_ignore_errors(args)
+
+
+if __name__ == '__main__':
+    if len(sys.argv) != 2:
+        print('Usage: compile_src.py GEN_DIR')
+        sys.exit(1)
+
+    gen_dir = sys.argv[1]
+
+    # Compile the CTS src.
+    compile_src(os.path.join(gen_dir, 'src'))
+    compile_src_for_node(os.path.join(gen_dir, 'src-node'))
+
+    # Run gen_listings.js to overwrite the dummy src/webgpu/listings.js created
+    # from transpiling src/
+    RunNode([
+        os.path.join(gen_dir, 'src-node', 'common', 'tools',
+                     'gen_listings.js'),
+        '--no-validate',
+        os.path.join(gen_dir, 'src'),
+        os.path.join(gen_dir, 'src-node', 'webgpu'),
+    ])

webgpu-cts/scripts/dir_paths.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+#
+# Copyright 2022 The Dawn Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+webgpu_cts_scripts_dir = os.path.dirname(os.path.abspath(__file__))
+dawn_third_party_dir = os.path.join(
+    os.path.dirname(os.path.dirname(webgpu_cts_scripts_dir)), 'third_party')
+gn_webgpu_cts_dir = os.path.join(dawn_third_party_dir, 'gn', 'webgpu-cts')
+webgpu_cts_root_dir = os.path.join(dawn_third_party_dir, 'webgpu-cts')
+chromium_third_party_dir = None
+node_dir = None
+
+_possible_chromium_third_party_dir = os.path.dirname(
+    os.path.dirname(dawn_third_party_dir))
+_possible_node_dir = os.path.join(_possible_chromium_third_party_dir, 'node')
+if os.path.exists(_possible_node_dir):
+    chromium_third_party_dir = _possible_chromium_third_party_dir
+    node_dir = _possible_node_dir

webgpu-cts/scripts/gen_ts_dep_lists.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+#
+# Copyright 2022 The Dawn Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import sys
+
+from dir_paths import gn_webgpu_cts_dir, webgpu_cts_root_dir
+from tsc_ignore_errors import run_tsc_ignore_errors
+
+src_prefix = webgpu_cts_root_dir.replace('\\', '/') + '/'
+
+
+def get_ts_sources():
+    # This will output all the source files in the form:
+    # "/absolute/path/to/file.ts"
+    # The path is always Unix-style.
+    # It will also output many Typescript errors since the build doesn't download the .d.ts
+    # dependencies.
+    stdout = run_tsc_ignore_errors([
+        '--project',
+        os.path.join(webgpu_cts_root_dir, 'tsconfig.json'), '--listFiles',
+        '--declaration', 'false', '--sourceMap', 'false'
+    ])
+
+    lines = [l.decode() for l in stdout.splitlines()]
+    return [
+        line[len(src_prefix):] for line in lines
+        if line.startswith(src_prefix + 'src/')
+    ]
+
+
+def get_resource_files():
+    files = os.listdir(os.path.join(webgpu_cts_root_dir, 'src', 'resources'))
+    files.sort()
+    return files
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--check',
+                        action='store_true',
+                        help='Check that the output file is up to date.')
+    parser.add_argument('--stamp', help='Stamp file to write after success.')
+    args = parser.parse_args()
+
+    ts_sources = [x + '\n' for x in get_ts_sources()]
+    ts_sources_txt = os.path.join(gn_webgpu_cts_dir, 'ts_sources.txt')
+
+    resource_files = [x + '\n' for x in get_resource_files()]
+    resource_files_txt = os.path.join(gn_webgpu_cts_dir, 'resource_files.txt')
+
+    if args.check:
+        with open(ts_sources_txt, 'r') as f:
+            txt = f.readlines()
+            if (txt != ts_sources):
+                raise RuntimeError(
+                    '%s is out of date. Please re-run //third_party/dawn/third_party/webgpu-cts/scripts/gen_ts_dep_lists.py\n'
+                    % ts_sources_txt)
+        with open(resource_files_txt, 'r') as f:
+            if (f.readlines() != resource_files):
+                raise RuntimeError(
+                    '%s is out of date. Please re-run //third_party/dawn/third_party/webgpu-cts/scripts/gen_ts_dep_lists.py\n'
+                    % resource_files_txt)
+    else:
+        with open(ts_sources_txt, 'w') as f:
+            f.writelines(ts_sources)
+        with open(resource_files_txt, 'w') as f:
+            f.writelines(resource_files)
+
+    if args.stamp:
+        with open(args.stamp, 'w') as f:
+            f.write('')

webgpu-cts/scripts/list.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+#
+# Copyright 2022 The Dawn Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import logging
+import os
+import shutil
+import sys
+import tempfile
+
+from dir_paths import node_dir
+
+from compile_src import compile_src_for_node
+
+
+def list_testcases(query, js_out_dir=None):
+    if js_out_dir is None:
+        js_out_dir = tempfile.mkdtemp()
+        delete_js_out_dir = True
+    else:
+        delete_js_out_dir = False
+
+    try:
+        logging.info('WebGPU CTS: Transpiling tools...')
+        compile_src_for_node(js_out_dir, [
+            '--incremental', '--tsBuildInfoFile',
+            os.path.join(js_out_dir, 'build.tsbuildinfo')
+        ],
+                             clean=False)
+
+        old_sys_path = sys.path
+        try:
+            sys.path = old_sys_path + [node_dir]
+            from node import RunNode
+        finally:
+            sys.path = old_sys_path
+
+        return RunNode([
+            os.path.join(js_out_dir, 'common', 'runtime', 'cmdline.js'), query,
+            '--list'
+        ])
+    finally:
+        if delete_js_out_dir:
+            shutil.rmtree(js_out_dir)
+
+
+# List all testcases matching a test query.
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--query', default='webgpu:*', help='WebGPU CTS Query')
+    parser.add_argument(
+        '--js-out-dir',
+        default=None,
+        help='Output directory for intermediate compiled JS sources')
+    args = parser.parse_args()
+
+    print(list_testcases(args.query, args.js_out_dir))

webgpu-cts/scripts/tsc_ignore_errors.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+#
+# Copyright 2022 The Dawn Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import subprocess
+import sys
+import os
+
+from dir_paths import node_dir
+
+try:
+    old_sys_path = sys.path
+    sys.path = [node_dir] + sys.path
+
+    from node import GetBinaryPath as get_node_binary_path
+finally:
+    sys.path = old_sys_path
+
+tsc = os.path.join(node_dir, 'node_modules', 'typescript', 'lib', 'tsc.js')
+
+
+def run_tsc_ignore_errors(args):
+    cmd = [get_node_binary_path(), tsc] + args
+    process = subprocess.Popen(cmd,
+                               cwd=os.getcwd(),
+                               stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE)
+
+    stdout, stderr = process.communicate()
+
+    # Typecheck errors go in stdout, not stderr. If we see something in stderr, raise an error.
+    if len(stderr):
+        raise RuntimeError('tsc \'%s\' failed\n%s' % (' '.join(cmd), stderr))
+
+    return stdout
+
+
+if __name__ == '__main__':
+    run_tsc_ignore_errors(sys.argv[1:])