From d2c9864b8e97f8f131617ef7c48274b6d4be278e Mon Sep 17 00:00:00 2001
From: Brian Sheedy <bsheedy@google.com>
Date: Thu, 24 Mar 2022 22:31:45 +0000
Subject: [PATCH] Add CTS expectation adder script

Adds a script to take a CTS queries, expand them to individual test case
names, and append them to the end of expectations.txt as valid
expectations.

Bug: chromium:1306640
Change-Id: I55067c8c49aeb4360d488841da12c06d88346b8a
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/84740
Commit-Queue: Brian Sheedy <bsheedy@google.com>
Auto-Submit: Brian Sheedy <bsheedy@google.com>
Reviewed-by: Austin Eng <enga@chromium.org>
Commit-Queue: Austin Eng <enga@chromium.org>
---
 .gitignore                                    |  3 +
 .../scripts/add_query_to_expectation_file.py  | 99 +++++++++++++++++++
 webgpu-cts/scripts/dir_paths.py               |  4 +-
 3 files changed, 104 insertions(+), 2 deletions(-)
 create mode 100755 webgpu-cts/scripts/add_query_to_expectation_file.py

diff --git a/.gitignore b/.gitignore
index fa211f11b7..b22f66c17d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -99,3 +99,6 @@ $RECYCLE.BIN/
 
 ### Dawn node tools binaries
 src/dawn/node/tools/bin/
+
+### Cached node transpiled tools
+/.node_transpile_work_dir
diff --git a/webgpu-cts/scripts/add_query_to_expectation_file.py b/webgpu-cts/scripts/add_query_to_expectation_file.py
new file mode 100755
index 0000000000..3597369272
--- /dev/null
+++ b/webgpu-cts/scripts/add_query_to_expectation_file.py
@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+#
+# Copyright 2022 The Dawn Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Script for easily adding expectations to expectations.txt
+
+Converts one or more WebGPU CTS queries into one or more individual expectations
+and appends them to the end of the file.
+"""
+
+import argparse
+import logging
+import os
+import subprocess
+import sys
+
+import dir_paths
+
+LIST_SCRIPT_PATH = os.path.join(dir_paths.webgpu_cts_scripts_dir, 'list.py')
+TRANSPILE_DIR = os.path.join(dir_paths.dawn_dir, '.node_transpile_work_dir')
+EXPECTATION_FILE_PATH = os.path.join(dir_paths.dawn_dir, 'webgpu-cts',
+                                     'expectations.txt')
+
+
+def expand_query(query):
+    cmd = [
+        sys.executable,
+        LIST_SCRIPT_PATH,
+        '--js-out-dir',
+        TRANSPILE_DIR,
+        '--query',
+        query,
+    ]
+    p = subprocess.run(cmd, stdout=subprocess.PIPE, check=True)
+    return p.stdout.decode('utf-8').splitlines()
+
+
+def generate_expectations(queries, tags, results, bug):
+    tags = '[ %s ] ' % ' '.join(tags) if tags else ''
+    results = ' [ %s ]' % ' '.join(results)
+    bug = bug + ' ' if bug else ''
+    content = ''
+    for q in queries:
+        test_names = expand_query(q)
+        if not test_names:
+            logging.warning('Did not get any test names for query %s', q)
+        for tn in test_names:
+            content += '{bug}{tags}{test}{results}\n'.format(bug=bug,
+                                                             tags=tags,
+                                                             test=tn,
+                                                             results=results)
+    with open(EXPECTATION_FILE_PATH, 'a') as outfile:
+        outfile.write(content)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description=('Converts one or more WebGPU CTS queries into one or '
+                     'more individual expectations and appends them to the '
+                     'end of expectations.txt'))
+    parser.add_argument('-b',
+                        '--bug',
+                        help='The bug link to associate with the expectations')
+    parser.add_argument('-t',
+                        '--tag',
+                        action='append',
+                        default=[],
+                        dest='tags',
+                        help=('A tag to restrict the expectation to. Can be '
+                              'specified multiple times.'))
+    parser.add_argument('-r',
+                        '--result',
+                        action='append',
+                        default=[],
+                        dest='results',
+                        required=True,
+                        help=('An expected result for the expectation. Can be '
+                              'specified multiple times, although a single '
+                              'result is the most common usage.'))
+    parser.add_argument('-q',
+                        '--query',
+                        action='append',
+                        default=[],
+                        dest='queries',
+                        help=('A CTS query to expand into expectations. Can '
+                              'be specified multiple times.'))
+    args = parser.parse_args()
+    generate_expectations(args.queries, args.tags, args.results, args.bug)
diff --git a/webgpu-cts/scripts/dir_paths.py b/webgpu-cts/scripts/dir_paths.py
index cc45758f37..8e2b37e16f 100644
--- a/webgpu-cts/scripts/dir_paths.py
+++ b/webgpu-cts/scripts/dir_paths.py
@@ -17,8 +17,8 @@
 import os
 
 webgpu_cts_scripts_dir = os.path.dirname(os.path.abspath(__file__))
-dawn_third_party_dir = os.path.join(
-    os.path.dirname(os.path.dirname(webgpu_cts_scripts_dir)), 'third_party')
+dawn_dir = os.path.dirname(os.path.dirname(webgpu_cts_scripts_dir))
+dawn_third_party_dir = os.path.join(dawn_dir, 'third_party')
 gn_webgpu_cts_dir = os.path.join(dawn_third_party_dir, 'gn', 'webgpu-cts')
 webgpu_cts_root_dir = os.path.join(dawn_third_party_dir, 'webgpu-cts')
 chromium_third_party_dir = None