Add end2end tests flags --enable-toggles and --disable-toggles

Also changes test helpers to use a generic HasToggleEnabled instead
of a helper function per-toggle.

Bug: dawn:571
Change-Id: Ifd2e787a733382dcd5ad08222616c12cb42fb62b
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/32300
Commit-Queue: Austin Eng <enga@chromium.org>
Reviewed-by: Corentin Wallez <cwallez@chromium.org>

src/tests/DawnTest.cpp

@@ -203,7 +203,7 @@ DawnTestEnvironment::DawnTestEnvironment(int argc, char** argv) {
     ASSERT(instance);
 
     SelectPreferredAdapterProperties(instance.get());
-    PrintTestConfigurationAndAdapterInfo();
+    PrintTestConfigurationAndAdapterInfo(instance.get());
 }
 
 DawnTestEnvironment::~DawnTestEnvironment() = default;
@@ -227,7 +227,29 @@ void DawnTestEnvironment::ParseArgs(int argc, char** argv) {
         }
 
         if (strcmp("--skip-validation", argv[i]) == 0) {
-            mSkipDawnValidation = true;
+            mEnabledToggles.push_back("skip_validation");
+            continue;
+        }
+
+        constexpr const char kEnableTogglesSwitch[] = "--enable-toggles=";
+        argLen = sizeof(kEnableTogglesSwitch) - 1;
+        if (strncmp(argv[i], kEnableTogglesSwitch, argLen) == 0) {
+            std::string toggle;
+            std::stringstream toggles(argv[i] + argLen);
+            while (getline(toggles, toggle, ',')) {
+                mEnabledToggles.push_back(toggle);
+            }
+            continue;
+        }
+
+        constexpr const char kDisableTogglesSwitch[] = "--disable-toggles=";
+        argLen = sizeof(kDisableTogglesSwitch) - 1;
+        if (strncmp(argv[i], kDisableTogglesSwitch, argLen) == 0) {
+            std::string toggle;
+            std::stringstream toggles(argv[i] + argLen);
+            while (getline(toggles, toggle, ',')) {
+                mDisabledToggles.push_back(toggle);
+            }
             continue;
         }
 
@@ -283,14 +305,19 @@ void DawnTestEnvironment::ParseArgs(int argc, char** argv) {
         if (strcmp("-h", argv[i]) == 0 || strcmp("--help", argv[i]) == 0) {
             dawn::InfoLog()
                 << "\n\nUsage: " << argv[0]
-                << " [GTEST_FLAGS...] [-w] [-d] [-c] [--adapter-vendor-id=x]"
+                << " [GTEST_FLAGS...] [-w] [-d] [-c] [--skip-validation]\n"
-                   " [--exclusive-device-type-preference=integrated,cpu,discrete]\n"
+                   "    [--enable-toggles=toggles] [--disable-toggles=toggles]\n"
+                   "    [--adapter-vendor-id=x]"
+                   " [--exclusive-device-type-preference=integrated,cpu,discrete]\n\n"
                    "  -w, --use-wire: Run the tests through the wire (defaults to no wire)\n"
                    "  -d, --enable-backend-validation: Enable backend validation (defaults"
                    " to disabled)\n"
                    "  -c, --begin-capture-on-startup: Begin debug capture on startup "
                    "(defaults to no capture)\n"
                    "  --skip-validation: Skip Dawn validation\n"
+                   "  --enable-toggles: Comma-delimited list of Dawn toggles to enable.\n"
+                   "      ex.) skip_validation,use_tint,disable_robustness,turn_off_vsync\n"
+                   "  --disable-toggles: Comma-delimited list of Dawn toggles to disable\n"
                    "  --adapter-vendor-id: Select adapter by vendor id to run end2end tests"
                    "on multi-GPU systems \n"
                    "  --exclusive-device-type-preference: Comma-delimited list of preferred device "
@@ -414,7 +441,8 @@ std::vector<AdapterTestParam> DawnTestEnvironment::GetAvailableAdapterTestParams
     return testParams;
 }
 
-void DawnTestEnvironment::PrintTestConfigurationAndAdapterInfo() const {
+void DawnTestEnvironment::PrintTestConfigurationAndAdapterInfo(
+    dawn_native::Instance* instance) const {
     dawn::LogMessage log = dawn::InfoLog();
     log << "Testing configuration\n"
            "---------------------\n"
@@ -422,11 +450,29 @@ void DawnTestEnvironment::PrintTestConfigurationAndAdapterInfo() const {
         << (mUseWire ? "true" : "false")
         << "\n"
            "EnableBackendValidation: "
-        << (mEnableBackendValidation ? "true" : "false")
+        << (mEnableBackendValidation ? "true" : "false");
-        << "\n"
+
-           "SkipDawnValidation: "
+    if (GetEnabledToggles().size() > 0) {
-        << (mSkipDawnValidation ? "true" : "false")
+        log << "\n"
-        << "\n"
+               "Enabled Toggles\n";
+        for (const std::string& toggle : GetEnabledToggles()) {
+            const dawn_native::ToggleInfo* info = instance->GetToggleInfo(toggle.c_str());
+            ASSERT(info != nullptr);
+            log << " - " << info->name << ": " << info->description << "\n";
+        }
+    }
+
+    if (GetDisabledToggles().size() > 0) {
+        log << "\n"
+               "Disabled Toggles\n";
+        for (const std::string& toggle : GetDisabledToggles()) {
+            const dawn_native::ToggleInfo* info = instance->GetToggleInfo(toggle.c_str());
+            ASSERT(info != nullptr);
+            log << " - " << info->name << ": " << info->description << "\n";
+        }
+    }
+
+    log << "\n"
            "BeginCaptureOnStartup: "
         << (mBeginCaptureOnStartup ? "true" : "false")
         << "\n"
@@ -472,10 +518,6 @@ bool DawnTestEnvironment::IsBackendValidationEnabled() const {
     return mEnableBackendValidation;
 }
 
-bool DawnTestEnvironment::IsDawnValidationSkipped() const {
-    return mSkipDawnValidation;
-}
-
 dawn_native::Instance* DawnTestEnvironment::GetInstance() const {
     return mInstance.get();
 }
@@ -495,6 +537,14 @@ const char* DawnTestEnvironment::GetWireTraceDir() const {
     return mWireTraceDir.c_str();
 }
 
+const std::vector<std::string>& DawnTestEnvironment::GetEnabledToggles() const {
+    return mEnabledToggles;
+}
+
+const std::vector<std::string>& DawnTestEnvironment::GetDisabledToggles() const {
+    return mDisabledToggles;
+}
+
 class WireServerTraceLayer : public dawn_wire::CommandHandler {
   public:
     WireServerTraceLayer(const char* file, dawn_wire::CommandHandler* handler)
@@ -617,10 +667,6 @@ bool DawnTestBase::IsBackendValidationEnabled() const {
     return gTestEnv->IsBackendValidationEnabled();
 }
 
-bool DawnTestBase::IsDawnValidationSkipped() const {
-    return gTestEnv->IsDawnValidationSkipped();
-}
-
 bool DawnTestBase::HasWGSL() const {
 #ifdef DAWN_ENABLE_WGSL
     return true;
@@ -638,12 +684,10 @@ bool DawnTestBase::IsAsan() const {
 }
 
 bool DawnTestBase::HasToggleEnabled(const char* toggle) const {
-    for (const char* toggleEnabled : mParam.forceEnabledWorkarounds) {
+    auto toggles = dawn_native::GetTogglesUsed(device.Get());
-        if (strcmp(toggle, toggleEnabled) == 0) {
+    return std::find_if(toggles.begin(), toggles.end(), [toggle](const char* name) {
-            return true;
+               return strcmp(toggle, name) == 0;
-        }
+           }) != toggles.end();
-    }
-    return false;
 }
 
 bool DawnTestBase::HasVendorIdFilter() const {
@@ -718,10 +762,18 @@ void DawnTestBase::SetUp() {
     deviceDescriptor.forceDisabledToggles = mParam.forceDisabledWorkarounds;
     deviceDescriptor.requiredExtensions = GetRequiredExtensions();
 
-    static constexpr char kSkipValidationToggle[] = "skip_validation";
+    for (const std::string& toggle : gTestEnv->GetEnabledToggles()) {
-    if (gTestEnv->IsDawnValidationSkipped()) {
+        const dawn_native::ToggleInfo* info =
-        ASSERT(gTestEnv->GetInstance()->GetToggleInfo(kSkipValidationToggle) != nullptr);
+            gTestEnv->GetInstance()->GetToggleInfo(toggle.c_str());
-        deviceDescriptor.forceEnabledToggles.push_back(kSkipValidationToggle);
+        ASSERT(info != nullptr);
+        deviceDescriptor.forceEnabledToggles.push_back(info->name);
+    }
+
+    for (const std::string& toggle : gTestEnv->GetDisabledToggles()) {
+        const dawn_native::ToggleInfo* info =
+            gTestEnv->GetInstance()->GetToggleInfo(toggle.c_str());
+        ASSERT(info != nullptr);
+        deviceDescriptor.forceDisabledToggles.push_back(info->name);
     }
 
     backendDevice = mBackendAdapter.CreateDevice(&deviceDescriptor);

src/tests/DawnTest.h

@@ -190,12 +190,14 @@ class DawnTestEnvironment : public testing::Environment {
 
     bool UsesWire() const;
     bool IsBackendValidationEnabled() const;
-    bool IsDawnValidationSkipped() const;
     dawn_native::Instance* GetInstance() const;
     bool HasVendorIdFilter() const;
     uint32_t GetVendorIdFilter() const;
     const char* GetWireTraceDir() const;
 
+    const std::vector<std::string>& GetEnabledToggles() const;
+    const std::vector<std::string>& GetDisabledToggles() const;
+
   protected:
     std::unique_ptr<dawn_native::Instance> mInstance;
 
@@ -203,15 +205,17 @@ class DawnTestEnvironment : public testing::Environment {
     void ParseArgs(int argc, char** argv);
     std::unique_ptr<dawn_native::Instance> CreateInstanceAndDiscoverAdapters() const;
     void SelectPreferredAdapterProperties(const dawn_native::Instance* instance);
-    void PrintTestConfigurationAndAdapterInfo() const;
+    void PrintTestConfigurationAndAdapterInfo(dawn_native::Instance* instance) const;
 
     bool mUseWire = false;
     bool mEnableBackendValidation = false;
-    bool mSkipDawnValidation = false;
     bool mBeginCaptureOnStartup = false;
     bool mHasVendorIdFilter = false;
     uint32_t mVendorIdFilter = 0;
     std::string mWireTraceDir;
+
+    std::vector<std::string> mEnabledToggles;
+    std::vector<std::string> mDisabledToggles;
     std::vector<dawn_native::DeviceType> mDevicePreferences;
     std::vector<TestAdapterProperties> mAdapterProperties;
 
@@ -249,7 +253,6 @@ class DawnTestBase {
 
     bool UsesWire() const;
     bool IsBackendValidationEnabled() const;
-    bool IsDawnValidationSkipped() const;
     bool HasWGSL() const;
 
     bool IsAsan() const;
@@ -423,7 +426,7 @@ class DawnTestBase {
             size_t warningsAfter =                                               \
                 dawn_native::GetDeprecationWarningCountForTesting(device.Get()); \
             EXPECT_EQ(mLastWarningCount, warningsBefore);                        \
-            if (!IsDawnValidationSkipped()) {                                    \
+            if (!HasToggleEnabled("skip_validation")) {                          \
                 EXPECT_EQ(warningsAfter, warningsBefore + 1);                    \
             }                                                                    \
             mLastWarningCount = warningsAfter;                                   \

src/tests/end2end/BufferTests.cpp

@@ -648,7 +648,7 @@ TEST_P(BufferMappedAtCreationTests, ZeroSizedMappableBuffer) {
 
 // Test that creating a zero-sized error buffer mapped. (it is a different code path)
 TEST_P(BufferMappedAtCreationTests, ZeroSizedErrorBuffer) {
-    DAWN_SKIP_TEST_IF(IsDawnValidationSkipped());
+    DAWN_SKIP_TEST_IF(HasToggleEnabled("skip_validation"));
 
     wgpu::BufferDescriptor descriptor;
     descriptor.size = 0;

src/tests/end2end/CreateReadyPipelineTests.cpp

@@ -98,7 +98,7 @@ TEST_P(CreateReadyPipelineTest, BasicUseOfCreateReadyComputePipeline) {
 // CreateReadyComputePipeline() any error won't be forwarded to the error scope / unhandled error
 // callback.
 TEST_P(CreateReadyPipelineTest, CreateComputePipelineFailed) {
-    DAWN_SKIP_TEST_IF(IsDawnValidationSkipped());
+    DAWN_SKIP_TEST_IF(HasToggleEnabled("skip_validation"));
 
     const char* computeShader = R"(
         #version 450
@@ -210,7 +210,7 @@ TEST_P(CreateReadyPipelineTest, BasicUseOfCreateReadyRenderPipeline) {
 // CreateReadyRenderPipeline() any error won't be forwarded to the error scope / unhandled error
 // callback.
 TEST_P(CreateReadyPipelineTest, CreateRenderPipelineFailed) {
-    DAWN_SKIP_TEST_IF(IsDawnValidationSkipped());
+    DAWN_SKIP_TEST_IF(HasToggleEnabled("skip_validation"));
 
     constexpr wgpu::TextureFormat kRenderAttachmentFormat = wgpu::TextureFormat::Depth32Float;
 

src/tests/end2end/DeprecatedAPITests.cpp

@@ -28,7 +28,7 @@ class DeprecationTests : public DawnTest {
     void SetUp() override {
         DawnTest::SetUp();
         // Skip when validation is off because warnings might be emitted during validation calls
-        DAWN_SKIP_TEST_IF(IsDawnValidationSkipped());
+        DAWN_SKIP_TEST_IF(HasToggleEnabled("skip_validation"));
     }
 };
 

src/tests/end2end/DestroyTests.cpp

@@ -23,7 +23,7 @@ class DestroyTest : public DawnTest {
   protected:
     void SetUp() override {
         DawnTest::SetUp();
-        DAWN_SKIP_TEST_IF(IsDawnValidationSkipped());
+        DAWN_SKIP_TEST_IF(HasToggleEnabled("skip_validation"));
 
         renderPass = utils::CreateBasicRenderPass(device, kRTSize, kRTSize);
 

src/tests/end2end/ObjectCachingTests.cpp

@@ -81,7 +81,7 @@ TEST_P(ObjectCachingTest, BindGroupLayoutViewDimension) {
 
 // Test that an error object doesn't try to uncache itself
 TEST_P(ObjectCachingTest, ErrorObjectDoesntUncache) {
-    DAWN_SKIP_TEST_IF(IsDawnValidationSkipped());
+    DAWN_SKIP_TEST_IF(HasToggleEnabled("skip_validation"));
 
     ASSERT_DEVICE_ERROR(
         wgpu::BindGroupLayout bgl = utils::MakeBindGroupLayout(

src/tests/end2end/SwapChainValidationTests.cpp

@@ -27,7 +27,7 @@ class SwapChainValidationTests : public DawnTest {
     void SetUp() override {
         DawnTest::SetUp();
         DAWN_SKIP_TEST_IF(UsesWire());
-        DAWN_SKIP_TEST_IF(IsDawnValidationSkipped());
+        DAWN_SKIP_TEST_IF(HasToggleEnabled("skip_validation"));
 
         glfwSetErrorCallback([](int code, const char* message) {
             dawn::ErrorLog() << "GLFW error " << code << " " << message;

src/tests/white_box/InternalResourceUsageTests.cpp

@@ -21,7 +21,7 @@ class InternalResourceUsageTests : public DawnTest {};
 // Verify it is an error to create a buffer with a buffer usage that should only be used
 // internally.
 TEST_P(InternalResourceUsageTests, InternalBufferUsage) {
-    DAWN_SKIP_TEST_IF(IsDawnValidationSkipped());
+    DAWN_SKIP_TEST_IF(HasToggleEnabled("skip_validation"));
 
     wgpu::BufferDescriptor descriptor;
     descriptor.size = 4;
@@ -33,7 +33,7 @@ TEST_P(InternalResourceUsageTests, InternalBufferUsage) {
 // Verify it is an error to create a texture with a texture usage that should only be used
 // internally.
 TEST_P(InternalResourceUsageTests, InternalTextureUsage) {
-    DAWN_SKIP_TEST_IF(IsDawnValidationSkipped());
+    DAWN_SKIP_TEST_IF(HasToggleEnabled("skip_validation"));
 
     wgpu::TextureDescriptor descriptor;
     descriptor.format = wgpu::TextureFormat::RGBA8Unorm;