Add CTS telemetry harness support

Migrates harness support files added in
https://chromium-review.googlesource.com/c/chromium/src/+/3541414
into Dawn's repo.

Tested in
https://chromium-review.googlesource.com/c/chromium/src/+/3537743

Bug: chromium:1306640
Change-Id: I3000b1223219a1da293af910bf442570b70b7c92
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/84043
Reviewed-by: Loko Kung <lokokung@google.com>
Commit-Queue: Austin Eng <enga@chromium.org>

third_party/gn/webgpu-cts/BUILD.gn

@@ -1,11 +1,23 @@
-# Copyright 2022 The Chromium Authors.  All rights reserved.
+# Copyright 2022 The Dawn Authors
-# Use of this source code is governed by a BSD-style license that can be
+#
-# found in the LICENSE file.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 # Note: This file is intentionally not used by any other BUILD.gn in Dawn.
 # Instead, Chromium depends directly on this file to build the WebGPU CTS.
 # Scripts called from this file assume Dawn is checked out inside Chromium.
 
+import("../../../scripts/dawn_overrides_with_defaults.gni")
+
 group("webgpu-cts") {
   public_deps = [
     ":compile_src",
@@ -38,12 +50,12 @@ foreach(ts_file, list_from_ts_sources_txt) {
 }
 
 action("compile_src") {
-  script = "scripts/compile_src.py"
+  script = "${dawn_root}/webgpu-cts/scripts/compile_src.py"
 
   inputs = [
              "//third_party/node/node_modules/typescript/lib/tsc.js",
              "//third_party/node/node.py",
-             "scripts/tsc_ignore_errors.py",
+             "${dawn_root}/webgpu-cts/scripts/tsc_ignore_errors.py",
            ] + ts_source_inputs
 
   outputs = js_outputs
@@ -74,7 +86,7 @@ copy("copy_resources") {
 }
 
 action("verify_gen_ts_dep_list") {
-  script = "scripts/gen_ts_dep_lists.py"
+  script = "${dawn_root}/webgpu-cts/scripts/gen_ts_dep_lists.py"
   inputs = [
     # TODO(kainino): Make sure this gets retriggered when the CTS dep changes.
     "resource_files.txt",

third_party/gn/webgpu-cts/scripts/dir_paths.py

@@ -1,17 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2022 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import os
-
-gn_webgpu_cts_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-dawn_third_party_dir = os.path.dirname(os.path.dirname(gn_webgpu_cts_dir))
-webgpu_cts_root_dir = os.path.join(dawn_third_party_dir, 'webgpu-cts')
-
-_possible_chromium_third_party_dir = os.path.dirname(
-    os.path.dirname(dawn_third_party_dir))
-_possible_node_dir = os.path.join(_possible_chromium_third_party_dir, 'node')
-if os.path.exists(_possible_node_dir):
-    chromium_third_party_dir = _possible_chromium_third_party_dir
-    node_dir = _possible_node_dir

webgpu-cts/BUILD.gn

@@ -0,0 +1,44 @@
+# Copyright 2022 The Dawn Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Note: This file is intentionally not used by any other BUILD.gn in Dawn.
+# Instead, Chromium depends directly on this file to build the WebGPU CTS.
+# Scripts called from this file assume Dawn is checked out inside Chromium.
+
+# This needs to be copied to the output directory since the CTS tests also
+# serve resources that are copied into it.
+copy("webgpu_cts_resources") {
+  testonly = true
+  sources = [
+    "test_page.html",
+    "test_runner.js",
+  ]
+  outputs = [ "$target_gen_dir/{{source_file_part}}" ]
+}
+
+group("webgpu-cts") {
+  testonly = true
+  data_deps = [
+    ":webgpu_cts_resources",
+    "../third_party/gn/webgpu-cts",
+  ]
+  data = [
+    "//third_party/node/",
+    "./scripts",
+    "./expectations.txt",
+    "../third_party/webgpu-cts/src",
+    "../third_party/webgpu-cts/node.tsconfig.json",
+    "../third_party/webgpu-cts/tsconfig.json",
+  ]
+}

webgpu-cts/README.md

@@ -0,0 +1,20 @@
+# Running the WebGPU CTS Locally with Chrome
+
+Running the WebGPU CTS locally with Chrome requires a Chromium checkout.
+
+Follow [these instructions](https://www.chromium.org/developers/how-tos/get-the-code/) for checking out
+and building Chrome. You'll also need to build the `telemetry_gpu_integration_test` target.
+
+At the root of a Chromium checkout, run:
+`./content/test/gpu/run_gpu_integration_test.py webgpu_cts --browser=exact --browser-executable=path/to/your/chrome-executable`
+
+If you don't want to build Chrome, you can still run the CTS, by passing the path to an existing Chrome executable to the `--browser-executable` argument. You should still build the `telemetry_gpu_integration_test` target to support all harness
+functionality.
+
+Useful command-line arguments:
+ - `-l`: List all tests that would be run.
+ - `--test-filter`: Filter tests. Run `--help` for more information.
+ - `--help`: See more options.
+ - `--passthrough --show-stdout`: Show browser output. See also `--browser-logging-verbosity`.
+ - `--extra-browser-args`: Pass extra args to the browser executable.
+ - `--is-backend-validation`: Enable backend validation. TODO: rename this to `--backend-validation`.

webgpu-cts/expectations.txt

@@ -0,0 +1,69 @@
+# THIS FILE IS AUTOGENERATED. DO NOT MANUALLY EDIT.
+# SEE //content/test/gpu/process_generated_webgpu_expectations.py
+# BEGIN TAG HEADER (autogenerated, see validate_tag_consistency.py)
+# OS
+# tags: [ android android-lollipop android-marshmallow android-nougat
+#             android-pie android-r android-s
+#         chromeos
+#         fuchsia
+#         linux ubuntu
+#         mac bigsur catalina lion highsierra mac-10.12 mojave monterey
+#             mountainlion sierra
+#         win win7 win8 win10 ]
+# Devices
+# tags: [ android-nexus-5 android-nexus-5x android-pixel-2 android-pixel-4
+#             android-pixel-6 android-shield-android-tv
+#         chromeos-board-amd64-generic chromeos-board-kevin chromeos-board-eve
+#         fuchsia-board-astro fuchsia-board-sherlock fuchsia-board-qemu-x64 ]
+# Platform
+# tags: [ desktop
+#         mobile ]
+# Browser
+# tags: [ android-chromium android-webview-instrumentation
+#         debug debug-x64
+#         release release-x64
+#         fuchsia-chrome web-engine-shell ]
+# GPU
+# tags: [ amd amd-0x6613 amd-0x679e amd-0x6821 amd-0x7340
+#         apple apple-apple-m1 apple-angle-metal-renderer:-apple-m1
+#         arm
+#         google google-0xffff
+#         intel intel-0xa2e intel-0xd26 intel-0xa011 intel-0x3e92 intel-0x3e9b
+#               intel-0x5912
+#         nvidia nvidia-0xfe9 nvidia-0x1cb3 nvidia-0x2184
+#         qualcomm ]
+# Decoder
+# tags: [ passthrough no-passthrough ]
+# ANGLE Backend
+# tags: [ angle-disabled
+#         angle-d3d9 angle-d3d11
+#         angle-metal
+#         angle-opengl angle-opengles
+#         angle-swiftshader
+#         angle-vulkan ]
+# Skia Renderer
+# tags: [ skia-renderer-dawn
+#         skia-renderer-disabled
+#         skia-renderer-gl
+#         skia-renderer-vulkan ]
+# SwiftShader
+# tags: [ swiftshader-gl no-swiftshader-gl ]
+# Driver
+# tags: [ mesa_lt_19.1 ]
+# ASan
+# tags: [ asan no-asan ]
+# Display Server
+# tags: [ display-server-wayland display-server-x ]
+# OOP-Canvas
+# tags: [ oop-c no-oop-c ]
+# WebGPU Backend Validation
+# tags: [ dawn-backend-validation dawn-no-backend-validation ]
+# results: [ Failure RetryOnFailure Skip Slow ]
+# END TAG HEADER
+
+# This will be slowly removed over time as more tests are moved from the web
+# test harness to the Telemetry harness.
+* [ Skip ]
+
+# BEGIN AUTOGENERATED EXPECTATIONS
+

webgpu-cts/scripts/compile_src.py

@@ -1,7 +1,18 @@
 #!/usr/bin/env python
-# Copyright 2021 The Chromium Authors. All rights reserved.
+#
-# Use of this source code is governed by a BSD-style license that can be
+# Copyright 2022 The Dawn Authors
-# found in the LICENSE file.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 import os
 import shutil

webgpu-cts/scripts/dir_paths.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+#
+# Copyright 2022 The Dawn Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+webgpu_cts_scripts_dir = os.path.dirname(os.path.abspath(__file__))
+dawn_third_party_dir = os.path.join(
+    os.path.dirname(os.path.dirname(webgpu_cts_scripts_dir)), 'third_party')
+gn_webgpu_cts_dir = os.path.join(dawn_third_party_dir, 'gn', 'webgpu-cts')
+webgpu_cts_root_dir = os.path.join(dawn_third_party_dir, 'webgpu-cts')
+chromium_third_party_dir = None
+node_dir = None
+
+_possible_chromium_third_party_dir = os.path.dirname(
+    os.path.dirname(dawn_third_party_dir))
+_possible_node_dir = os.path.join(_possible_chromium_third_party_dir, 'node')
+if os.path.exists(_possible_node_dir):
+    chromium_third_party_dir = _possible_chromium_third_party_dir
+    node_dir = _possible_node_dir

webgpu-cts/scripts/gen_ts_dep_lists.py

@@ -1,7 +1,18 @@
 #!/usr/bin/env python
-# Copyright 2021 The Chromium Authors.  All rights reserved.
+#
-# Use of this source code is governed by a BSD-style license that can be
+# Copyright 2022 The Dawn Authors
-# found in the LICENSE file.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 import argparse
 import os

webgpu-cts/scripts/list.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+#
+# Copyright 2022 The Dawn Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import logging
+import os
+import shutil
+import sys
+import tempfile
+
+from dir_paths import node_dir
+
+from compile_src import compile_src_for_node
+
+
+def list_testcases(query, js_out_dir=None):
+    if js_out_dir is None:
+        js_out_dir = tempfile.mkdtemp()
+        delete_js_out_dir = True
+    else:
+        delete_js_out_dir = False
+
+    try:
+        logging.info('WebGPU CTS: Transpiling tools...')
+        compile_src_for_node(js_out_dir, [
+            '--incremental', '--tsBuildInfoFile',
+            os.path.join(js_out_dir, 'build.tsbuildinfo')
+        ],
+                             clean=False)
+
+        old_sys_path = sys.path
+        try:
+            sys.path = old_sys_path + [node_dir]
+            from node import RunNode
+        finally:
+            sys.path = old_sys_path
+
+        return RunNode([
+            os.path.join(js_out_dir, 'common', 'runtime', 'cmdline.js'), query,
+            '--list'
+        ])
+    finally:
+        if delete_js_out_dir:
+            shutil.rmtree(js_out_dir)
+
+
+# List all testcases matching a test query.
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--query', default='webgpu:*', help='WebGPU CTS Query')
+    parser.add_argument(
+        '--js-out-dir',
+        default=None,
+        help='Output directory for intermediate compiled JS sources')
+    args = parser.parse_args()
+
+    print(list_testcases(args.query, args.js_out_dir))

webgpu-cts/scripts/tsc_ignore_errors.py

@@ -1,7 +1,18 @@
 #!/usr/bin/env python
-# Copyright 2021 The Chromium Authors. All rights reserved.
+#
-# Use of this source code is governed by a BSD-style license that can be
+# Copyright 2022 The Dawn Authors
-# found in the LICENSE file.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 import subprocess
 import sys

webgpu-cts/test_page.html

@@ -0,0 +1,24 @@
+<!doctype html>
+<!-- Copyright 2022 The Dawn Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. -->
+
+<title>WebGPU CTS</title>
+<meta charset=utf-8>
+<link rel=help href='https://gpuweb.github.io/gpuweb/'>
+
+<script type=module>
+    import { setBaseResourcePath } from '/gen/third_party/dawn/third_party/webgpu-cts/src/common/framework/resources.js';
+    setBaseResourcePath('/gen/third_party/dawn/third_party/webgpu-cts/resources');
+</script>
+<script type=module src=./test_runner.js></script>

webgpu-cts/test_runner.js

@@ -0,0 +1,65 @@
+// Copyright 2022 The Dawn Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { DefaultTestFileLoader } from '../third_party/webgpu-cts/src/common/internal/file_loader.js';
+import { prettyPrintLog } from '../third_party/webgpu-cts/src/common/internal/logging/log_message.js';
+import { Logger } from '../third_party/webgpu-cts/src/common/internal/logging/logger.js';
+import { parseQuery } from '../third_party/webgpu-cts/src/common/internal/query/parseQuery.js';
+
+import { TestWorker } from '../third_party/webgpu-cts/src/common/runtime/helper/test_worker.js';
+
+var socket;
+
+async function setupWebsocket(port) {
+  socket = new WebSocket('ws://127.0.0.1:' + port)
+  socket.addEventListener('message', runCtsTestViaSocket);
+}
+
+async function runCtsTestViaSocket(event) {
+  let input = JSON.parse(event.data);
+  runCtsTest(input['q'], input['w']);
+}
+
+async function runCtsTest(query, use_worker) {
+  const workerEnabled = use_worker;
+  const worker = workerEnabled ? new TestWorker(false) : undefined;
+
+  const loader = new DefaultTestFileLoader();
+  const filterQuery = parseQuery(query);
+  const testcases = await loader.loadCases(filterQuery);
+
+  const expectations = [];
+
+  const log = new Logger();
+
+  for (const testcase of testcases) {
+    const name = testcase.query.toString();
+
+    const wpt_fn = async () => {
+      const [rec, res] = log.record(name);
+      if (worker) {
+        await worker.run(rec, name, expectations);
+      } else {
+        await testcase.run(rec, expectations);
+      }
+
+      socket.send(JSON.stringify({'s': res.status,
+                                  'l': (res.logs ?? []).map(prettyPrintLog)}));
+    };
+    await wpt_fn();
+  }
+}
+
+window.runCtsTest = runCtsTest;
+window.setupWebsocket = setupWebsocket