From 1cdea90f7b955dd3659a2f88909bdc15581fab04 Mon Sep 17 00:00:00 2001 From: Austin Eng Date: Thu, 24 Mar 2022 00:21:55 +0000 Subject: [PATCH] Add CTS telemetry harness support Migrates harness support files added in https://chromium-review.googlesource.com/c/chromium/src/+/3541414 into Dawn's repo. Tested in https://chromium-review.googlesource.com/c/chromium/src/+/3537743 Bug: chromium:1306640 Change-Id: I3000b1223219a1da293af910bf442570b70b7c92 Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/84043 Reviewed-by: Loko Kung Commit-Queue: Austin Eng --- third_party/gn/webgpu-cts/BUILD.gn | 24 +++++-- .../gn/webgpu-cts/scripts/dir_paths.py | 17 ----- webgpu-cts/BUILD.gn | 44 ++++++++++++ webgpu-cts/README.md | 20 ++++++ webgpu-cts/expectations.txt | 69 ++++++++++++++++++ .../scripts/compile_src.py | 17 ++++- webgpu-cts/scripts/dir_paths.py | 32 +++++++++ .../scripts/gen_ts_dep_lists.py | 17 ++++- webgpu-cts/scripts/list.py | 70 +++++++++++++++++++ .../scripts/tsc_ignore_errors.py | 17 ++++- webgpu-cts/test_page.html | 24 +++++++ webgpu-cts/test_runner.js | 65 +++++++++++++++++ 12 files changed, 384 insertions(+), 32 deletions(-) delete mode 100644 third_party/gn/webgpu-cts/scripts/dir_paths.py create mode 100644 webgpu-cts/BUILD.gn create mode 100644 webgpu-cts/README.md create mode 100644 webgpu-cts/expectations.txt rename {third_party/gn/webgpu-cts => webgpu-cts}/scripts/compile_src.py (78%) create mode 100644 webgpu-cts/scripts/dir_paths.py rename {third_party/gn/webgpu-cts => webgpu-cts}/scripts/gen_ts_dep_lists.py (81%) create mode 100755 webgpu-cts/scripts/list.py rename {third_party/gn/webgpu-cts => webgpu-cts}/scripts/tsc_ignore_errors.py (61%) create mode 100644 webgpu-cts/test_page.html create mode 100644 webgpu-cts/test_runner.js diff --git a/third_party/gn/webgpu-cts/BUILD.gn b/third_party/gn/webgpu-cts/BUILD.gn index 4455250aa3..7023941388 100644 --- a/third_party/gn/webgpu-cts/BUILD.gn +++ b/third_party/gn/webgpu-cts/BUILD.gn @@ -1,11 +1,23 @@ -# Copyright 2022 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. +# Copyright 2022 The Dawn Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # Note: This file is intentionally not used by any other BUILD.gn in Dawn. # Instead, Chromium depends directly on this file to build the WebGPU CTS. # Scripts called from this file assume Dawn is checked out inside Chromium. +import("../../../scripts/dawn_overrides_with_defaults.gni") + group("webgpu-cts") { public_deps = [ ":compile_src", @@ -38,12 +50,12 @@ foreach(ts_file, list_from_ts_sources_txt) { } action("compile_src") { - script = "scripts/compile_src.py" + script = "${dawn_root}/webgpu-cts/scripts/compile_src.py" inputs = [ "//third_party/node/node_modules/typescript/lib/tsc.js", "//third_party/node/node.py", - "scripts/tsc_ignore_errors.py", + "${dawn_root}/webgpu-cts/scripts/tsc_ignore_errors.py", ] + ts_source_inputs outputs = js_outputs @@ -74,7 +86,7 @@ copy("copy_resources") { } action("verify_gen_ts_dep_list") { - script = "scripts/gen_ts_dep_lists.py" + script = "${dawn_root}/webgpu-cts/scripts/gen_ts_dep_lists.py" inputs = [ # TODO(kainino): Make sure this gets retriggered when the CTS dep changes. "resource_files.txt", diff --git a/third_party/gn/webgpu-cts/scripts/dir_paths.py b/third_party/gn/webgpu-cts/scripts/dir_paths.py deleted file mode 100644 index 4b9d3107dc..0000000000 --- a/third_party/gn/webgpu-cts/scripts/dir_paths.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env python -# Copyright 2022 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import os - -gn_webgpu_cts_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -dawn_third_party_dir = os.path.dirname(os.path.dirname(gn_webgpu_cts_dir)) -webgpu_cts_root_dir = os.path.join(dawn_third_party_dir, 'webgpu-cts') - -_possible_chromium_third_party_dir = os.path.dirname( - os.path.dirname(dawn_third_party_dir)) -_possible_node_dir = os.path.join(_possible_chromium_third_party_dir, 'node') -if os.path.exists(_possible_node_dir): - chromium_third_party_dir = _possible_chromium_third_party_dir - node_dir = _possible_node_dir diff --git a/webgpu-cts/BUILD.gn b/webgpu-cts/BUILD.gn new file mode 100644 index 0000000000..a0ecb67ef4 --- /dev/null +++ b/webgpu-cts/BUILD.gn @@ -0,0 +1,44 @@ +# Copyright 2022 The Dawn Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Note: This file is intentionally not used by any other BUILD.gn in Dawn. +# Instead, Chromium depends directly on this file to build the WebGPU CTS. +# Scripts called from this file assume Dawn is checked out inside Chromium. + +# This needs to be copied to the output directory since the CTS tests also +# serve resources that are copied into it. +copy("webgpu_cts_resources") { + testonly = true + sources = [ + "test_page.html", + "test_runner.js", + ] + outputs = [ "$target_gen_dir/{{source_file_part}}" ] +} + +group("webgpu-cts") { + testonly = true + data_deps = [ + ":webgpu_cts_resources", + "../third_party/gn/webgpu-cts", + ] + data = [ + "//third_party/node/", + "./scripts", + "./expectations.txt", + "../third_party/webgpu-cts/src", + "../third_party/webgpu-cts/node.tsconfig.json", + "../third_party/webgpu-cts/tsconfig.json", + ] +} diff --git a/webgpu-cts/README.md b/webgpu-cts/README.md new file mode 100644 index 0000000000..cc4d86d33f --- /dev/null +++ b/webgpu-cts/README.md @@ -0,0 +1,20 @@ +# Running the WebGPU CTS Locally with Chrome + +Running the WebGPU CTS locally with Chrome requires a Chromium checkout. + +Follow [these instructions](https://www.chromium.org/developers/how-tos/get-the-code/) for checking out +and building Chrome. You'll also need to build the `telemetry_gpu_integration_test` target. + +At the root of a Chromium checkout, run: +`./content/test/gpu/run_gpu_integration_test.py webgpu_cts --browser=exact --browser-executable=path/to/your/chrome-executable` + +If you don't want to build Chrome, you can still run the CTS, by passing the path to an existing Chrome executable to the `--browser-executable` argument. You should still build the `telemetry_gpu_integration_test` target to support all harness +functionality. + +Useful command-line arguments: + - `-l`: List all tests that would be run. + - `--test-filter`: Filter tests. Run `--help` for more information. + - `--help`: See more options. + - `--passthrough --show-stdout`: Show browser output. See also `--browser-logging-verbosity`. + - `--extra-browser-args`: Pass extra args to the browser executable. + - `--is-backend-validation`: Enable backend validation. TODO: rename this to `--backend-validation`. diff --git a/webgpu-cts/expectations.txt b/webgpu-cts/expectations.txt new file mode 100644 index 0000000000..07d81d1b89 --- /dev/null +++ b/webgpu-cts/expectations.txt @@ -0,0 +1,69 @@ +# THIS FILE IS AUTOGENERATED. DO NOT MANUALLY EDIT. +# SEE //content/test/gpu/process_generated_webgpu_expectations.py +# BEGIN TAG HEADER (autogenerated, see validate_tag_consistency.py) +# OS +# tags: [ android android-lollipop android-marshmallow android-nougat +# android-pie android-r android-s +# chromeos +# fuchsia +# linux ubuntu +# mac bigsur catalina lion highsierra mac-10.12 mojave monterey +# mountainlion sierra +# win win7 win8 win10 ] +# Devices +# tags: [ android-nexus-5 android-nexus-5x android-pixel-2 android-pixel-4 +# android-pixel-6 android-shield-android-tv +# chromeos-board-amd64-generic chromeos-board-kevin chromeos-board-eve +# fuchsia-board-astro fuchsia-board-sherlock fuchsia-board-qemu-x64 ] +# Platform +# tags: [ desktop +# mobile ] +# Browser +# tags: [ android-chromium android-webview-instrumentation +# debug debug-x64 +# release release-x64 +# fuchsia-chrome web-engine-shell ] +# GPU +# tags: [ amd amd-0x6613 amd-0x679e amd-0x6821 amd-0x7340 +# apple apple-apple-m1 apple-angle-metal-renderer:-apple-m1 +# arm +# google google-0xffff +# intel intel-0xa2e intel-0xd26 intel-0xa011 intel-0x3e92 intel-0x3e9b +# intel-0x5912 +# nvidia nvidia-0xfe9 nvidia-0x1cb3 nvidia-0x2184 +# qualcomm ] +# Decoder +# tags: [ passthrough no-passthrough ] +# ANGLE Backend +# tags: [ angle-disabled +# angle-d3d9 angle-d3d11 +# angle-metal +# angle-opengl angle-opengles +# angle-swiftshader +# angle-vulkan ] +# Skia Renderer +# tags: [ skia-renderer-dawn +# skia-renderer-disabled +# skia-renderer-gl +# skia-renderer-vulkan ] +# SwiftShader +# tags: [ swiftshader-gl no-swiftshader-gl ] +# Driver +# tags: [ mesa_lt_19.1 ] +# ASan +# tags: [ asan no-asan ] +# Display Server +# tags: [ display-server-wayland display-server-x ] +# OOP-Canvas +# tags: [ oop-c no-oop-c ] +# WebGPU Backend Validation +# tags: [ dawn-backend-validation dawn-no-backend-validation ] +# results: [ Failure RetryOnFailure Skip Slow ] +# END TAG HEADER + +# This will be slowly removed over time as more tests are moved from the web +# test harness to the Telemetry harness. +* [ Skip ] + +# BEGIN AUTOGENERATED EXPECTATIONS + diff --git a/third_party/gn/webgpu-cts/scripts/compile_src.py b/webgpu-cts/scripts/compile_src.py similarity index 78% rename from third_party/gn/webgpu-cts/scripts/compile_src.py rename to webgpu-cts/scripts/compile_src.py index 46016aae6c..b08c0a74f6 100755 --- a/third_party/gn/webgpu-cts/scripts/compile_src.py +++ b/webgpu-cts/scripts/compile_src.py @@ -1,7 +1,18 @@ #!/usr/bin/env python -# Copyright 2021 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. +# +# Copyright 2022 The Dawn Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import os import shutil diff --git a/webgpu-cts/scripts/dir_paths.py b/webgpu-cts/scripts/dir_paths.py new file mode 100644 index 0000000000..cc45758f37 --- /dev/null +++ b/webgpu-cts/scripts/dir_paths.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# +# Copyright 2022 The Dawn Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +webgpu_cts_scripts_dir = os.path.dirname(os.path.abspath(__file__)) +dawn_third_party_dir = os.path.join( + os.path.dirname(os.path.dirname(webgpu_cts_scripts_dir)), 'third_party') +gn_webgpu_cts_dir = os.path.join(dawn_third_party_dir, 'gn', 'webgpu-cts') +webgpu_cts_root_dir = os.path.join(dawn_third_party_dir, 'webgpu-cts') +chromium_third_party_dir = None +node_dir = None + +_possible_chromium_third_party_dir = os.path.dirname( + os.path.dirname(dawn_third_party_dir)) +_possible_node_dir = os.path.join(_possible_chromium_third_party_dir, 'node') +if os.path.exists(_possible_node_dir): + chromium_third_party_dir = _possible_chromium_third_party_dir + node_dir = _possible_node_dir diff --git a/third_party/gn/webgpu-cts/scripts/gen_ts_dep_lists.py b/webgpu-cts/scripts/gen_ts_dep_lists.py similarity index 81% rename from third_party/gn/webgpu-cts/scripts/gen_ts_dep_lists.py rename to webgpu-cts/scripts/gen_ts_dep_lists.py index 4b9498f272..5862ab28e7 100755 --- a/third_party/gn/webgpu-cts/scripts/gen_ts_dep_lists.py +++ b/webgpu-cts/scripts/gen_ts_dep_lists.py @@ -1,7 +1,18 @@ #!/usr/bin/env python -# Copyright 2021 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. +# +# Copyright 2022 The Dawn Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import argparse import os diff --git a/webgpu-cts/scripts/list.py b/webgpu-cts/scripts/list.py new file mode 100755 index 0000000000..d63beb4655 --- /dev/null +++ b/webgpu-cts/scripts/list.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# +# Copyright 2022 The Dawn Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import logging +import os +import shutil +import sys +import tempfile + +from dir_paths import node_dir + +from compile_src import compile_src_for_node + + +def list_testcases(query, js_out_dir=None): + if js_out_dir is None: + js_out_dir = tempfile.mkdtemp() + delete_js_out_dir = True + else: + delete_js_out_dir = False + + try: + logging.info('WebGPU CTS: Transpiling tools...') + compile_src_for_node(js_out_dir, [ + '--incremental', '--tsBuildInfoFile', + os.path.join(js_out_dir, 'build.tsbuildinfo') + ], + clean=False) + + old_sys_path = sys.path + try: + sys.path = old_sys_path + [node_dir] + from node import RunNode + finally: + sys.path = old_sys_path + + return RunNode([ + os.path.join(js_out_dir, 'common', 'runtime', 'cmdline.js'), query, + '--list' + ]) + finally: + if delete_js_out_dir: + shutil.rmtree(js_out_dir) + + +# List all testcases matching a test query. +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--query', default='webgpu:*', help='WebGPU CTS Query') + parser.add_argument( + '--js-out-dir', + default=None, + help='Output directory for intermediate compiled JS sources') + args = parser.parse_args() + + print(list_testcases(args.query, args.js_out_dir)) diff --git a/third_party/gn/webgpu-cts/scripts/tsc_ignore_errors.py b/webgpu-cts/scripts/tsc_ignore_errors.py similarity index 61% rename from third_party/gn/webgpu-cts/scripts/tsc_ignore_errors.py rename to webgpu-cts/scripts/tsc_ignore_errors.py index 8fcbb5ff7e..14b8455142 100755 --- a/third_party/gn/webgpu-cts/scripts/tsc_ignore_errors.py +++ b/webgpu-cts/scripts/tsc_ignore_errors.py @@ -1,7 +1,18 @@ #!/usr/bin/env python -# Copyright 2021 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. +# +# Copyright 2022 The Dawn Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import subprocess import sys diff --git a/webgpu-cts/test_page.html b/webgpu-cts/test_page.html new file mode 100644 index 0000000000..89ef808669 --- /dev/null +++ b/webgpu-cts/test_page.html @@ -0,0 +1,24 @@ + + + +WebGPU CTS + + + + + diff --git a/webgpu-cts/test_runner.js b/webgpu-cts/test_runner.js new file mode 100644 index 0000000000..3f4eea2c69 --- /dev/null +++ b/webgpu-cts/test_runner.js @@ -0,0 +1,65 @@ +// Copyright 2022 The Dawn Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { DefaultTestFileLoader } from '../third_party/webgpu-cts/src/common/internal/file_loader.js'; +import { prettyPrintLog } from '../third_party/webgpu-cts/src/common/internal/logging/log_message.js'; +import { Logger } from '../third_party/webgpu-cts/src/common/internal/logging/logger.js'; +import { parseQuery } from '../third_party/webgpu-cts/src/common/internal/query/parseQuery.js'; + +import { TestWorker } from '../third_party/webgpu-cts/src/common/runtime/helper/test_worker.js'; + +var socket; + +async function setupWebsocket(port) { + socket = new WebSocket('ws://127.0.0.1:' + port) + socket.addEventListener('message', runCtsTestViaSocket); +} + +async function runCtsTestViaSocket(event) { + let input = JSON.parse(event.data); + runCtsTest(input['q'], input['w']); +} + +async function runCtsTest(query, use_worker) { + const workerEnabled = use_worker; + const worker = workerEnabled ? new TestWorker(false) : undefined; + + const loader = new DefaultTestFileLoader(); + const filterQuery = parseQuery(query); + const testcases = await loader.loadCases(filterQuery); + + const expectations = []; + + const log = new Logger(); + + for (const testcase of testcases) { + const name = testcase.query.toString(); + + const wpt_fn = async () => { + const [rec, res] = log.record(name); + if (worker) { + await worker.run(rec, name, expectations); + } else { + await testcase.run(rec, expectations); + } + + socket.send(JSON.stringify({'s': res.status, + 'l': (res.logs ?? []).map(prettyPrintLog)})); + }; + await wpt_fn(); + } +} + +window.runCtsTest = runCtsTest; +window.setupWebsocket = setupWebsocket