From 3fd20367551e113858b87f05fc32bd4f132edc42 Mon Sep 17 00:00:00 2001 From: Jiawei Shao Date: Wed, 19 May 2021 08:31:13 +0000 Subject: [PATCH] Add a command line parameter to run the tests skipped by DAWN_SUPPRESSED_TEST_IF This patch introduces two new macros to replace DAWN_SKIP_TEST_IF(): 1. DAWN_SUPPRESSED_TEST_IF(): for the tests failing on a specific HW / backend / OS combination 2. DAWN_TEST_UNSUPPORTED_IF(): for the tests which require an extension or a toggle to be present /not present or some WIP features. This patch also adds a command line parameter "--run-suppressed-tests" to disable the macro DAWN_SUPPRESSED_TEST_IF(), so that we can test any tests that are related to any specific HW / backend / OS combinations without changing the source code and re-building dawn_end2end_tests. This patch also replaces DAWN_SKIP_TEST_IF() with DAWN_SUPPRESSED_TEST_IF() or DAWN_TEST_UNSUPPORTED_IF() in QueryTests.cpp and ShaderFloat16Tests.cpp to test the functionality of these two new macros. DAWN_SKIP_TEST_IF() will be completely replaced by DAWN_SUPPRESSED_TEST_IF() or DAWN_TEST_UNSUPPORTED_IF() in the next patch. BUG=dawn:779 Change-Id: I05db632c614b6ad348fcac85da84744e45be3ae1 Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/51341 Reviewed-by: Corentin Wallez Reviewed-by: Austin Eng Commit-Queue: Jiawei Shao --- src/tests/DawnTest.cpp | 20 +++++++++++++- src/tests/DawnTest.h | 35 ++++++++++++++++++------ src/tests/end2end/QueryTests.cpp | 22 +++++++-------- src/tests/end2end/ShaderFloat16Tests.cpp | 6 ++-- 4 files changed, 59 insertions(+), 24 deletions(-) diff --git a/src/tests/DawnTest.cpp b/src/tests/DawnTest.cpp index 7b76c08c6b..31fc13df03 100644 --- a/src/tests/DawnTest.cpp +++ b/src/tests/DawnTest.cpp @@ -239,6 +239,11 @@ void DawnTestEnvironment::ParseArgs(int argc, char** argv) { continue; } + if (strcmp("--run-suppressed-tests", argv[i]) == 0) { + mRunSuppressedTests = true; + continue; + } + constexpr const char kEnableBackendValidationSwitch[] = "--enable-backend-validation"; argLen = sizeof(kEnableBackendValidationSwitch) - 1; if (strncmp(argv[i], kEnableBackendValidationSwitch, argLen) == 0) { @@ -366,7 +371,9 @@ void DawnTestEnvironment::ParseArgs(int argc, char** argv) { "null, opengl, opengles, vulkan\n" " --exclusive-device-type-preference: Comma-delimited list of preferred device " "types. For each backend, tests will run only on adapters that match the first " - "available device type\n"; + "available device type\n" + " --run-suppressed-tests: Run all the tests that will be skipped by the macro " + "DAWN_SUPPRESS_TEST_IF()\n"; continue; } @@ -547,6 +554,9 @@ void DawnTestEnvironment::PrintTestConfigurationAndAdapterInfo( "---------------------\n" "UseWire: " << (mUseWire ? "true" : "false") + << "\n" + "Run suppressed tests: " + << (mRunSuppressedTests ? "true" : "false") << "\n" "BackendValidation: "; @@ -626,6 +636,10 @@ bool DawnTestEnvironment::UsesWire() const { return mUseWire; } +bool DawnTestEnvironment::RunSuppressedTests() const { + return mRunSuppressedTests; +} + dawn_native::BackendValidationLevel DawnTestEnvironment::GetBackendValidationLevel() const { return mBackendValidationLevel; } @@ -779,6 +793,10 @@ bool DawnTestBase::IsBackendValidationEnabled() const { return gTestEnv->GetBackendValidationLevel() != dawn_native::BackendValidationLevel::Disabled; } +bool DawnTestBase::RunSuppressedTests() const { + return gTestEnv->RunSuppressedTests(); +} + bool DawnTestBase::IsAsan() const { #if defined(ADDRESS_SANITIZER) return true; diff --git a/src/tests/DawnTest.h b/src/tests/DawnTest.h index f10b802e45..0ea5f52d59 100644 --- a/src/tests/DawnTest.h +++ b/src/tests/DawnTest.h @@ -214,6 +214,8 @@ class DawnTestEnvironment : public testing::Environment { const std::vector& GetEnabledToggles() const; const std::vector& GetDisabledToggles() const; + bool RunSuppressedTests() const; + protected: std::unique_ptr mInstance; @@ -232,6 +234,7 @@ class DawnTestEnvironment : public testing::Environment { bool mHasBackendTypeFilter = false; wgpu::BackendType mBackendTypeFilter; std::string mWireTraceDir; + bool mRunSuppressedTests = false; ToggleParser mToggleParser; @@ -276,6 +279,7 @@ class DawnTestBase { bool UsesWire() const; bool IsBackendValidationEnabled() const; + bool RunSuppressedTests() const; bool IsAsan() const; @@ -438,16 +442,30 @@ class DawnTestBase { std::unique_ptr mTestPlatform; }; -// Skip a test when the given condition is satisfied. -#define DAWN_SKIP_TEST_IF(condition) \ - do { \ - if (condition) { \ - dawn::InfoLog() << "Test skipped: " #condition "."; \ - GTEST_SKIP(); \ - return; \ - } \ +#define DAWN_SKIP_TEST_IF_BASE(condition, type, reason) \ + do { \ + if (condition) { \ + dawn::InfoLog() << "Test " type ": " #reason; \ + GTEST_SKIP(); \ + return; \ + } \ } while (0) +// Skip a test when the given condition is satisfied. +// TODO(jiawei.shao@intel.com): Replace this macro with DAWN_TEST_UNSUPPORTED_IF or +// DAWN_SUPPRESS_TEST_IF. +#define DAWN_SKIP_TEST_IF(condition) DAWN_SKIP_TEST_IF_BASE(condition, "skipped", condition) + +// Skip a test which requires an extension or a toggle to be present / not present or some WIP +// features. +#define DAWN_TEST_UNSUPPORTED_IF(condition) \ + DAWN_SKIP_TEST_IF_BASE(condition, "unsupported", condition) + +// Skip a test when the test failing on a specific HW / backend / OS combination. We can disable +// this macro with the command line parameter "--run-suppressed-tests". +#define DAWN_SUPPRESS_TEST_IF(condition) \ + DAWN_SKIP_TEST_IF_BASE(!RunSuppressedTests() && condition, "suppressed", condition) + #define EXPECT_DEPRECATION_WARNING(statement) \ do { \ if (UsesWire()) { \ @@ -504,7 +522,6 @@ using DawnTest = DawnTestWithParams<>; namespace detail { // Helper functions used for DAWN_INSTANTIATE_TEST - bool IsBackendAvailable(wgpu::BackendType type); std::vector GetAvailableAdapterTestParamsForBackends( const BackendTestConfig* params, size_t numParams); diff --git a/src/tests/end2end/QueryTests.cpp b/src/tests/end2end/QueryTests.cpp index 55aab8a73b..bd88483c3b 100644 --- a/src/tests/end2end/QueryTests.cpp +++ b/src/tests/end2end/QueryTests.cpp @@ -253,7 +253,7 @@ TEST_P(OcclusionQueryTests, QueryWithScissorTest) { // the WriteBuffer and ResolveQuerySet are not executed in order or the ResolveQuerySet does not // copy the result to the buffer. In order to integrate end2end tests to Intel driver CL without // unknown issues, skip it until we find the root cause. - DAWN_SKIP_TEST_IF(IsWindows() && IsVulkan() && IsIntel()); + DAWN_SUPPRESS_TEST_IF(IsWindows() && IsVulkan() && IsIntel()); // Test there are samples passed scissor testing, the expected occlusion result is non-zero. TestOcclusionQueryWithScissorTest({2, 1, 2, 1}, OcclusionExpectation::Result::NonZero); @@ -305,12 +305,12 @@ TEST_P(OcclusionQueryTests, ResolveSparseQueries) { // TODO(hao.x.li@intel.com): Fails on Intel Windows Vulkan due to a driver issue that // vkCmdFillBuffer and vkCmdCopyQueryPoolResults are not executed in order, skip it util // the issue is fixed. - DAWN_SKIP_TEST_IF(IsWindows() && IsVulkan() && IsIntel()); + DAWN_SUPPRESS_TEST_IF(IsWindows() && IsVulkan() && IsIntel()); // TODO(hao.x.li@intel.com): Investigate why it's failed on D3D12 on Nvidia when running with // the previous occlusion tests. Expect resolve to 0 for these unwritten queries but the // occlusion result of the previous tests is got. - DAWN_SKIP_TEST_IF(IsD3D12() & IsNvidia()); + DAWN_SUPPRESS_TEST_IF(IsD3D12() & IsNvidia()); constexpr uint32_t kQueryCount = 7; @@ -370,7 +370,7 @@ TEST_P(OcclusionQueryTests, ResolveWithoutWritten) { // TODO(hao.x.li@intel.com): Investigate why it's failed on D3D12 on Nvidia when running with // the previous occlusion tests. Expect resolve to 0 but the occlusion result of the previous // tests is got. - DAWN_SKIP_TEST_IF(IsD3D12() & IsNvidia()); + DAWN_SUPPRESS_TEST_IF(IsD3D12() & IsNvidia()); constexpr uint32_t kQueryCount = 1; @@ -450,7 +450,7 @@ class PipelineStatisticsQueryTests : public QueryTests { DawnTest::SetUp(); // Skip all tests if pipeline statistics extension is not supported - DAWN_SKIP_TEST_IF(!SupportsExtensions({"pipeline_statistics_query"})); + DAWN_TEST_UNSUPPORTED_IF(!SupportsExtensions({"pipeline_statistics_query"})); } std::vector GetRequiredExtensions() override { @@ -508,7 +508,7 @@ class TimestampQueryTests : public QueryTests { DawnTest::SetUp(); // Skip all tests if timestamp extension is not supported - DAWN_SKIP_TEST_IF(!SupportsExtensions({"timestamp_query"})); + DAWN_TEST_UNSUPPORTED_IF(!SupportsExtensions({"timestamp_query"})); } std::vector GetRequiredExtensions() override { @@ -536,7 +536,7 @@ TEST_P(TimestampQueryTests, QuerySetCreation) { TEST_P(TimestampQueryTests, TimestampOnCommandEncoder) { // TODO(hao.x.li@intel.com): Crash occurs if we only call WriteTimestamp in a command encoder // without any copy commands on Metal on AMD GPU. See https://crbug.com/dawn/545. - DAWN_SKIP_TEST_IF(IsMetal() && IsAMD()); + DAWN_SUPPRESS_TEST_IF(IsMetal() && IsAMD()); constexpr uint32_t kQueryCount = 2; @@ -707,7 +707,7 @@ TEST_P(TimestampQueryTests, ResolveSparseQueries) { // TODO(hao.x.li@intel.com): Fails on Intel Windows Vulkan due to a driver issue that // vkCmdFillBuffer and vkCmdCopyQueryPoolResults are not executed in order, skip it util // the issue is fixed. - DAWN_SKIP_TEST_IF(IsWindows() && IsVulkan() && IsIntel()); + DAWN_SUPPRESS_TEST_IF(IsWindows() && IsVulkan() && IsIntel()); constexpr uint32_t kQueryCount = 4; @@ -758,11 +758,11 @@ TEST_P(TimestampQueryTests, ResolveToBufferWithOffset) { // TODO(hao.x.li@intel.com): Fails on Intel Windows Vulkan due to a driver issue that // vkCmdFillBuffer and vkCmdCopyQueryPoolResults are not executed in order, skip it util // the issue is fixed. - DAWN_SKIP_TEST_IF(IsWindows() && IsVulkan() && IsIntel()); + DAWN_SUPPRESS_TEST_IF(IsWindows() && IsVulkan() && IsIntel()); // TODO(hao.x.li@intel.com): Crash occurs if we only call WriteTimestamp in a command encoder // without any copy commands on Metal on AMD GPU. See https://crbug.com/dawn/545. - DAWN_SKIP_TEST_IF(IsMetal() && IsAMD()); + DAWN_SUPPRESS_TEST_IF(IsMetal() && IsAMD()); constexpr uint32_t kQueryCount = 2; constexpr uint64_t kZero = 0; @@ -803,7 +803,7 @@ TEST_P(TimestampQueryTests, ResolveTwiceToSameBuffer) { // TODO(hao.x.li@intel.com): Fails on Intel Windows Vulkan due to a driver issue that // vkCmdFillBuffer and vkCmdCopyQueryPoolResults are not executed in order, skip it util // the issue is fixed. - DAWN_SKIP_TEST_IF(IsWindows() && IsVulkan() && IsIntel()); + DAWN_SUPPRESS_TEST_IF(IsWindows() && IsVulkan() && IsIntel()); constexpr uint32_t kQueryCount = 3; diff --git a/src/tests/end2end/ShaderFloat16Tests.cpp b/src/tests/end2end/ShaderFloat16Tests.cpp index a09839c76b..d728334571 100644 --- a/src/tests/end2end/ShaderFloat16Tests.cpp +++ b/src/tests/end2end/ShaderFloat16Tests.cpp @@ -37,10 +37,10 @@ class ShaderFloat16Tests : public DawnTest { // Test basic 16bit float arithmetic and 16bit storage features. TEST_P(ShaderFloat16Tests, Basic16BitFloatFeaturesTest) { - DAWN_SKIP_TEST_IF(!IsShaderFloat16Supported()); - DAWN_SKIP_TEST_IF(IsD3D12() && IsIntel()); // Flaky crashes. crbug.com/dawn/586 + DAWN_TEST_UNSUPPORTED_IF(!IsShaderFloat16Supported()); + DAWN_SUPPRESS_TEST_IF(IsD3D12() && IsIntel()); // Flaky crashes. crbug.com/dawn/586 // TODO(crbug.com/tint/404): Implement float16 in Tint. - DAWN_SKIP_TEST_IF(HasToggleEnabled("use_tint_generator")); + DAWN_TEST_UNSUPPORTED_IF(HasToggleEnabled("use_tint_generator")); uint16_t uniformData[] = {Float32ToFloat16(1.23), Float32ToFloat16(0.0)}; // 0.0 is a padding. wgpu::Buffer uniformBuffer = utils::CreateBufferFromData(