Add a command line parameter to run the tests skipped by DAWN_SUPPRESSED_TEST_IF

This patch introduces two new macros to replace DAWN_SKIP_TEST_IF():
1. DAWN_SUPPRESSED_TEST_IF(): for the tests failing on a specific HW / backend /
   OS combination
2. DAWN_TEST_UNSUPPORTED_IF(): for the tests which require an extension or a toggle
   to be present /not present or some WIP features.

This patch also adds a command line parameter "--run-suppressed-tests" to disable
the macro DAWN_SUPPRESSED_TEST_IF(), so that we can test any tests that are related
to any specific HW / backend / OS combinations without changing the source code and
re-building dawn_end2end_tests.

This patch also replaces DAWN_SKIP_TEST_IF() with DAWN_SUPPRESSED_TEST_IF() or
DAWN_TEST_UNSUPPORTED_IF() in QueryTests.cpp and ShaderFloat16Tests.cpp to test
the functionality of these two new macros. DAWN_SKIP_TEST_IF() will be completely
replaced by DAWN_SUPPRESSED_TEST_IF() or DAWN_TEST_UNSUPPORTED_IF() in the next
patch.

BUG=dawn:779

Change-Id: I05db632c614b6ad348fcac85da84744e45be3ae1
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/51341
Reviewed-by: Corentin Wallez <cwallez@chromium.org>
Reviewed-by: Austin Eng <enga@chromium.org>
Commit-Queue: Jiawei Shao <jiawei.shao@intel.com>

src/tests/DawnTest.cpp

@@ -239,6 +239,11 @@ void DawnTestEnvironment::ParseArgs(int argc, char** argv) {
             continue;
         }
 
+        if (strcmp("--run-suppressed-tests", argv[i]) == 0) {
+            mRunSuppressedTests = true;
+            continue;
+        }
+
         constexpr const char kEnableBackendValidationSwitch[] = "--enable-backend-validation";
         argLen = sizeof(kEnableBackendValidationSwitch) - 1;
         if (strncmp(argv[i], kEnableBackendValidationSwitch, argLen) == 0) {
@@ -366,7 +371,9 @@ void DawnTestEnvironment::ParseArgs(int argc, char** argv) {
                    "null, opengl, opengles, vulkan\n"
                    "  --exclusive-device-type-preference: Comma-delimited list of preferred device "
                    "types. For each backend, tests will run only on adapters that match the first "
-                   "available device type\n";
+                   "available device type\n"
+                   "  --run-suppressed-tests: Run all the tests that will be skipped by the macro "
+                   "DAWN_SUPPRESS_TEST_IF()\n";
             continue;
         }
 
@@ -547,6 +554,9 @@ void DawnTestEnvironment::PrintTestConfigurationAndAdapterInfo(
            "---------------------\n"
            "UseWire: "
         << (mUseWire ? "true" : "false")
+        << "\n"
+           "Run suppressed tests: "
+        << (mRunSuppressedTests ? "true" : "false")
         << "\n"
            "BackendValidation: ";
 
@@ -626,6 +636,10 @@ bool DawnTestEnvironment::UsesWire() const {
     return mUseWire;
 }
 
+bool DawnTestEnvironment::RunSuppressedTests() const {
+    return mRunSuppressedTests;
+}
+
 dawn_native::BackendValidationLevel DawnTestEnvironment::GetBackendValidationLevel() const {
     return mBackendValidationLevel;
 }
@@ -779,6 +793,10 @@ bool DawnTestBase::IsBackendValidationEnabled() const {
     return gTestEnv->GetBackendValidationLevel() != dawn_native::BackendValidationLevel::Disabled;
 }
 
+bool DawnTestBase::RunSuppressedTests() const {
+    return gTestEnv->RunSuppressedTests();
+}
+
 bool DawnTestBase::IsAsan() const {
 #if defined(ADDRESS_SANITIZER)
     return true;

src/tests/DawnTest.h

@@ -214,6 +214,8 @@ class DawnTestEnvironment : public testing::Environment {
     const std::vector<std::string>& GetEnabledToggles() const;
     const std::vector<std::string>& GetDisabledToggles() const;
 
+    bool RunSuppressedTests() const;
+
   protected:
     std::unique_ptr<dawn_native::Instance> mInstance;
 
@@ -232,6 +234,7 @@ class DawnTestEnvironment : public testing::Environment {
     bool mHasBackendTypeFilter = false;
     wgpu::BackendType mBackendTypeFilter;
     std::string mWireTraceDir;
+    bool mRunSuppressedTests = false;
 
     ToggleParser mToggleParser;
 
@@ -276,6 +279,7 @@ class DawnTestBase {
 
     bool UsesWire() const;
     bool IsBackendValidationEnabled() const;
+    bool RunSuppressedTests() const;
 
     bool IsAsan() const;
 
@@ -438,16 +442,30 @@ class DawnTestBase {
     std::unique_ptr<dawn_platform::Platform> mTestPlatform;
 };
 
-// Skip a test when the given condition is satisfied.
+#define DAWN_SKIP_TEST_IF_BASE(condition, type, reason)   \
-#define DAWN_SKIP_TEST_IF(condition)                            \
     do {                                                  \
         if (condition) {                                  \
-            dawn::InfoLog() << "Test skipped: " #condition "."; \
+            dawn::InfoLog() << "Test " type ": " #reason; \
             GTEST_SKIP();                                 \
             return;                                       \
         }                                                 \
     } while (0)
 
+// Skip a test when the given condition is satisfied.
+// TODO(jiawei.shao@intel.com): Replace this macro with DAWN_TEST_UNSUPPORTED_IF or
+// DAWN_SUPPRESS_TEST_IF.
+#define DAWN_SKIP_TEST_IF(condition) DAWN_SKIP_TEST_IF_BASE(condition, "skipped", condition)
+
+// Skip a test which requires an extension or a toggle to be present / not present or some WIP
+// features.
+#define DAWN_TEST_UNSUPPORTED_IF(condition) \
+    DAWN_SKIP_TEST_IF_BASE(condition, "unsupported", condition)
+
+// Skip a test when the test failing on a specific HW / backend / OS combination. We can disable
+// this macro with the command line parameter "--run-suppressed-tests".
+#define DAWN_SUPPRESS_TEST_IF(condition) \
+    DAWN_SKIP_TEST_IF_BASE(!RunSuppressedTests() && condition, "suppressed", condition)
+
 #define EXPECT_DEPRECATION_WARNING(statement)                                    \
     do {                                                                         \
         if (UsesWire()) {                                                        \
@@ -504,7 +522,6 @@ using DawnTest = DawnTestWithParams<>;
 
 namespace detail {
     // Helper functions used for DAWN_INSTANTIATE_TEST
-    bool IsBackendAvailable(wgpu::BackendType type);
     std::vector<AdapterTestParam> GetAvailableAdapterTestParamsForBackends(
         const BackendTestConfig* params,
         size_t numParams);

src/tests/end2end/QueryTests.cpp

@@ -253,7 +253,7 @@ TEST_P(OcclusionQueryTests, QueryWithScissorTest) {
     // the WriteBuffer and ResolveQuerySet are not executed in order or the ResolveQuerySet does not
     // copy the result to the buffer. In order to integrate end2end tests to Intel driver CL without
     // unknown issues, skip it until we find the root cause.
-    DAWN_SKIP_TEST_IF(IsWindows() && IsVulkan() && IsIntel());
+    DAWN_SUPPRESS_TEST_IF(IsWindows() && IsVulkan() && IsIntel());
 
     // Test there are samples passed scissor testing, the expected occlusion result is non-zero.
     TestOcclusionQueryWithScissorTest({2, 1, 2, 1}, OcclusionExpectation::Result::NonZero);
@@ -305,12 +305,12 @@ TEST_P(OcclusionQueryTests, ResolveSparseQueries) {
     // TODO(hao.x.li@intel.com): Fails on Intel Windows Vulkan due to a driver issue that
     // vkCmdFillBuffer and vkCmdCopyQueryPoolResults are not executed in order, skip it util
     // the issue is fixed.
-    DAWN_SKIP_TEST_IF(IsWindows() && IsVulkan() && IsIntel());
+    DAWN_SUPPRESS_TEST_IF(IsWindows() && IsVulkan() && IsIntel());
 
     // TODO(hao.x.li@intel.com): Investigate why it's failed on D3D12 on Nvidia when running with
     // the previous occlusion tests. Expect resolve to 0 for these unwritten queries but the
     // occlusion result of the previous tests is got.
-    DAWN_SKIP_TEST_IF(IsD3D12() & IsNvidia());
+    DAWN_SUPPRESS_TEST_IF(IsD3D12() & IsNvidia());
 
     constexpr uint32_t kQueryCount = 7;
 
@@ -370,7 +370,7 @@ TEST_P(OcclusionQueryTests, ResolveWithoutWritten) {
     // TODO(hao.x.li@intel.com): Investigate why it's failed on D3D12 on Nvidia when running with
     // the previous occlusion tests. Expect resolve to 0 but the occlusion result of the previous
     // tests is got.
-    DAWN_SKIP_TEST_IF(IsD3D12() & IsNvidia());
+    DAWN_SUPPRESS_TEST_IF(IsD3D12() & IsNvidia());
 
     constexpr uint32_t kQueryCount = 1;
 
@@ -450,7 +450,7 @@ class PipelineStatisticsQueryTests : public QueryTests {
         DawnTest::SetUp();
 
         // Skip all tests if pipeline statistics extension is not supported
-        DAWN_SKIP_TEST_IF(!SupportsExtensions({"pipeline_statistics_query"}));
+        DAWN_TEST_UNSUPPORTED_IF(!SupportsExtensions({"pipeline_statistics_query"}));
     }
 
     std::vector<const char*> GetRequiredExtensions() override {
@@ -508,7 +508,7 @@ class TimestampQueryTests : public QueryTests {
         DawnTest::SetUp();
 
         // Skip all tests if timestamp extension is not supported
-        DAWN_SKIP_TEST_IF(!SupportsExtensions({"timestamp_query"}));
+        DAWN_TEST_UNSUPPORTED_IF(!SupportsExtensions({"timestamp_query"}));
     }
 
     std::vector<const char*> GetRequiredExtensions() override {
@@ -536,7 +536,7 @@ TEST_P(TimestampQueryTests, QuerySetCreation) {
 TEST_P(TimestampQueryTests, TimestampOnCommandEncoder) {
     // TODO(hao.x.li@intel.com): Crash occurs if we only call WriteTimestamp in a command encoder
     // without any copy commands on Metal on AMD GPU. See https://crbug.com/dawn/545.
-    DAWN_SKIP_TEST_IF(IsMetal() && IsAMD());
+    DAWN_SUPPRESS_TEST_IF(IsMetal() && IsAMD());
 
     constexpr uint32_t kQueryCount = 2;
 
@@ -707,7 +707,7 @@ TEST_P(TimestampQueryTests, ResolveSparseQueries) {
     // TODO(hao.x.li@intel.com): Fails on Intel Windows Vulkan due to a driver issue that
     // vkCmdFillBuffer and vkCmdCopyQueryPoolResults are not executed in order, skip it util
     // the issue is fixed.
-    DAWN_SKIP_TEST_IF(IsWindows() && IsVulkan() && IsIntel());
+    DAWN_SUPPRESS_TEST_IF(IsWindows() && IsVulkan() && IsIntel());
 
     constexpr uint32_t kQueryCount = 4;
 
@@ -758,11 +758,11 @@ TEST_P(TimestampQueryTests, ResolveToBufferWithOffset) {
     // TODO(hao.x.li@intel.com): Fails on Intel Windows Vulkan due to a driver issue that
     // vkCmdFillBuffer and vkCmdCopyQueryPoolResults are not executed in order, skip it util
     // the issue is fixed.
-    DAWN_SKIP_TEST_IF(IsWindows() && IsVulkan() && IsIntel());
+    DAWN_SUPPRESS_TEST_IF(IsWindows() && IsVulkan() && IsIntel());
 
     // TODO(hao.x.li@intel.com): Crash occurs if we only call WriteTimestamp in a command encoder
     // without any copy commands on Metal on AMD GPU. See https://crbug.com/dawn/545.
-    DAWN_SKIP_TEST_IF(IsMetal() && IsAMD());
+    DAWN_SUPPRESS_TEST_IF(IsMetal() && IsAMD());
 
     constexpr uint32_t kQueryCount = 2;
     constexpr uint64_t kZero = 0;
@@ -803,7 +803,7 @@ TEST_P(TimestampQueryTests, ResolveTwiceToSameBuffer) {
     // TODO(hao.x.li@intel.com): Fails on Intel Windows Vulkan due to a driver issue that
     // vkCmdFillBuffer and vkCmdCopyQueryPoolResults are not executed in order, skip it util
     // the issue is fixed.
-    DAWN_SKIP_TEST_IF(IsWindows() && IsVulkan() && IsIntel());
+    DAWN_SUPPRESS_TEST_IF(IsWindows() && IsVulkan() && IsIntel());
 
     constexpr uint32_t kQueryCount = 3;
 

src/tests/end2end/ShaderFloat16Tests.cpp

@@ -37,10 +37,10 @@ class ShaderFloat16Tests : public DawnTest {
 
 // Test basic 16bit float arithmetic and 16bit storage features.
 TEST_P(ShaderFloat16Tests, Basic16BitFloatFeaturesTest) {
-    DAWN_SKIP_TEST_IF(!IsShaderFloat16Supported());
+    DAWN_TEST_UNSUPPORTED_IF(!IsShaderFloat16Supported());
-    DAWN_SKIP_TEST_IF(IsD3D12() && IsIntel());  // Flaky crashes. crbug.com/dawn/586
+    DAWN_SUPPRESS_TEST_IF(IsD3D12() && IsIntel());  // Flaky crashes. crbug.com/dawn/586
     // TODO(crbug.com/tint/404): Implement float16 in Tint.
-    DAWN_SKIP_TEST_IF(HasToggleEnabled("use_tint_generator"));
+    DAWN_TEST_UNSUPPORTED_IF(HasToggleEnabled("use_tint_generator"));
 
     uint16_t uniformData[] = {Float32ToFloat16(1.23), Float32ToFloat16(0.0)};  // 0.0 is a padding.
     wgpu::Buffer uniformBuffer = utils::CreateBufferFromData(