Run dawn_end2end_tests on all available system adapters

By default, the tests will run on all available adapters, so this
adds an --exclusive-device-type-preference flag which takes a list
of comma-delimited device type preferences (discrete,integrated,cpu).
Tests will run only on the first available device type.
This is useful because in Chromium's test infrastructure, the same
test arguments are passed to one machine on which we want to use the
discrete GPU, as well as one machine where we want to use the
integrated GPU.

Bug: dawn:396
Change-Id: Id936fff3356eef3c6d12dfd1407b0e1f0f020dc1
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/21202
Commit-Queue: Austin Eng <enga@chromium.org>
Reviewed-by: Stephen White <senorblanco@chromium.org>

src/tests/DawnTest.cpp

@@ -31,6 +31,7 @@
 #include <algorithm>
 #include <fstream>
 #include <iomanip>
+#include <regex>
 #include <sstream>
 #include <unordered_map>
 
@@ -90,7 +91,7 @@ const RGBA8 RGBA8::kBlue = RGBA8(0, 0, 255, 255);
 const RGBA8 RGBA8::kYellow = RGBA8(255, 255, 0, 255);
 const RGBA8 RGBA8::kWhite = RGBA8(255, 255, 255, 255);
 
-DawnTestParam::DawnTestParam(wgpu::BackendType backendType,
+BackendTestConfig::BackendTestConfig(wgpu::BackendType backendType,
                                      std::initializer_list<const char*> forceEnabledWorkarounds,
                                      std::initializer_list<const char*> forceDisabledWorkarounds)
     : backendType(backendType),
@@ -98,33 +99,33 @@ DawnTestParam::DawnTestParam(wgpu::BackendType backendType,
       forceDisabledWorkarounds(forceDisabledWorkarounds) {
 }
 
-DawnTestParam D3D12Backend(std::initializer_list<const char*> forceEnabledWorkarounds,
+BackendTestConfig D3D12Backend(std::initializer_list<const char*> forceEnabledWorkarounds,
                                std::initializer_list<const char*> forceDisabledWorkarounds) {
-    return DawnTestParam(wgpu::BackendType::D3D12, forceEnabledWorkarounds,
+    return BackendTestConfig(wgpu::BackendType::D3D12, forceEnabledWorkarounds,
                              forceDisabledWorkarounds);
 }
 
-DawnTestParam MetalBackend(std::initializer_list<const char*> forceEnabledWorkarounds,
+BackendTestConfig MetalBackend(std::initializer_list<const char*> forceEnabledWorkarounds,
                                std::initializer_list<const char*> forceDisabledWorkarounds) {
-    return DawnTestParam(wgpu::BackendType::Metal, forceEnabledWorkarounds,
+    return BackendTestConfig(wgpu::BackendType::Metal, forceEnabledWorkarounds,
                              forceDisabledWorkarounds);
 }
 
-DawnTestParam NullBackend(std::initializer_list<const char*> forceEnabledWorkarounds,
+BackendTestConfig NullBackend(std::initializer_list<const char*> forceEnabledWorkarounds,
                               std::initializer_list<const char*> forceDisabledWorkarounds) {
-    return DawnTestParam(wgpu::BackendType::Null, forceEnabledWorkarounds,
+    return BackendTestConfig(wgpu::BackendType::Null, forceEnabledWorkarounds,
                              forceDisabledWorkarounds);
 }
 
-DawnTestParam OpenGLBackend(std::initializer_list<const char*> forceEnabledWorkarounds,
+BackendTestConfig OpenGLBackend(std::initializer_list<const char*> forceEnabledWorkarounds,
                                 std::initializer_list<const char*> forceDisabledWorkarounds) {
-    return DawnTestParam(wgpu::BackendType::OpenGL, forceEnabledWorkarounds,
+    return BackendTestConfig(wgpu::BackendType::OpenGL, forceEnabledWorkarounds,
                              forceDisabledWorkarounds);
 }
 
-DawnTestParam VulkanBackend(std::initializer_list<const char*> forceEnabledWorkarounds,
+BackendTestConfig VulkanBackend(std::initializer_list<const char*> forceEnabledWorkarounds,
                                 std::initializer_list<const char*> forceDisabledWorkarounds) {
-    return DawnTestParam(wgpu::BackendType::Vulkan, forceEnabledWorkarounds,
+    return BackendTestConfig(wgpu::BackendType::Vulkan, forceEnabledWorkarounds,
                              forceDisabledWorkarounds);
 }
 
@@ -133,8 +134,24 @@ TestAdapterProperties::TestAdapterProperties(const wgpu::AdapterProperties& prop
     : wgpu::AdapterProperties(properties), adapterName(properties.name), selected(selected) {
 }
 
-std::ostream& operator<<(std::ostream& os, const DawnTestParam& param) {
+AdapterTestParam::AdapterTestParam(const BackendTestConfig& config,
-    os << ParamName(param.backendType);
+                                   const TestAdapterProperties& adapterProperties)
+    : adapterProperties(adapterProperties),
+      forceEnabledWorkarounds(config.forceEnabledWorkarounds),
+      forceDisabledWorkarounds(config.forceDisabledWorkarounds) {
+}
+
+std::ostream& operator<<(std::ostream& os, const AdapterTestParam& param) {
+    // Sanitize the adapter name for GoogleTest
+    std::string sanitizedName =
+        std::regex_replace(param.adapterProperties.adapterName, std::regex("[^a-zA-Z0-9]+"), "_");
+
+    // Strip trailing underscores, if any.
+    if (sanitizedName.back() == '_') {
+        sanitizedName.back() = '\0';
+    }
+
+    os << ParamName(param.adapterProperties.backendType) << "_" << sanitizedName.c_str();
     for (const char* forceEnabledWorkaround : param.forceEnabledWorkarounds) {
         os << "__e_" << forceEnabledWorkaround;
     }
@@ -159,15 +176,16 @@ void DawnTestEnvironment::SetEnvironment(DawnTestEnvironment* env) {
 DawnTestEnvironment::DawnTestEnvironment(int argc, char** argv) {
     ParseArgs(argc, argv);
 
-    // Create a temporary instance to gather adapter properties. This is done before
+    // Create a temporary instance to select available and preferred adapters. This is done before
-    // test instantiation so FilterBackends can generate test parameterizations only on available
+    // test instantiation so GetAvailableAdapterTestParamsForBackends can generate test
-    // backends. We drop the instance at the end of this function because the Vulkan validation
+    // parameterizations all selected adapters. We drop the instance at the end of this function
-    // layers use static global mutexes which behave badly when Chromium's test launcher forks the
+    // because the Vulkan validation layers use static global mutexes which behave badly when
-    // test process. The instance will be recreated on test environment setup.
+    // Chromium's test launcher forks the test process. The instance will be recreated on test
+    // environment setup.
     std::unique_ptr<dawn_native::Instance> instance = CreateInstanceAndDiscoverAdapters();
     ASSERT(instance);
 
-    GatherAdapterProperties(instance.get());
+    SelectPreferredAdapterProperties(instance.get());
     PrintTestConfigurationAndAdapterInfo();
 }
 
@@ -272,6 +290,29 @@ void DawnTestEnvironment::ParseArgs(int argc, char** argv) {
             continue;
         }
 
+        constexpr const char kExclusiveDeviceTypePreferenceArg[] =
+            "--exclusive-device-type-preference=";
+        argLen = sizeof(kExclusiveDeviceTypePreferenceArg) - 1;
+        if (strncmp(argv[i], kExclusiveDeviceTypePreferenceArg, argLen) == 0) {
+            const char* preference = argv[i] + argLen;
+            if (preference[0] != '\0') {
+                std::istringstream ss(preference);
+                std::string type;
+                while (std::getline(ss, type, ',')) {
+                    if (strcmp(type.c_str(), "discrete") == 0) {
+                        mDevicePreferences.push_back(dawn_native::DeviceType::DiscreteGPU);
+                    } else if (strcmp(type.c_str(), "integrated") == 0) {
+                        mDevicePreferences.push_back(dawn_native::DeviceType::IntegratedGPU);
+                    } else if (strcmp(type.c_str(), "cpu") == 0) {
+                        mDevicePreferences.push_back(dawn_native::DeviceType::CPU);
+                    } else {
+                        dawn::ErrorLog() << "Invalid device type preference: " << type;
+                        UNREACHABLE();
+                    }
+                }
+            }
+        }
+
         constexpr const char kWireTraceDirArg[] = "--wire-trace-dir=";
         argLen = sizeof(kWireTraceDirArg) - 1;
         if (strncmp(argv[i], kWireTraceDirArg, argLen) == 0) {
@@ -289,7 +330,8 @@ void DawnTestEnvironment::ParseArgs(int argc, char** argv) {
         if (strcmp("-h", argv[i]) == 0 || strcmp("--help", argv[i]) == 0) {
             dawn::InfoLog()
                 << "\n\nUsage: " << argv[0]
-                << " [GTEST_FLAGS...] [-w] [-d] [-c] [--adapter-vendor-id=x]\n"
+                << " [GTEST_FLAGS...] [-w] [-d] [-c] [--adapter-vendor-id=x]"
+                   " [--exclusive-device-type-preference=integrated,cpu,discrete]\n"
                    "  -w, --use-wire: Run the tests through the wire (defaults to no wire)\n"
                    "  -d, --enable-backend-validation: Enable backend validation (defaults"
                    " to disabled)\n"
@@ -303,7 +345,10 @@ void DawnTestEnvironment::ParseArgs(int argc, char** argv) {
                    "  --no-use-spvc-parser: Do no use spvc's spir-v parsing insteads of "
                    "spirv-cross's\n"
                    "  --adapter-vendor-id: Select adapter by vendor id to run end2end tests"
-                   "on multi-GPU systems \n";
+                   "on multi-GPU systems \n"
+                   "  --exclusive-device-type-preference: Comma-delimited list of preferred device "
+                   "types. For each backend, tests will run only on adapters that match the first "
+                   "available device type\n";
             continue;
         }
     }
@@ -339,28 +384,77 @@ std::unique_ptr<dawn_native::Instance> DawnTestEnvironment::CreateInstanceAndDis
     return instance;
 }
 
-void DawnTestEnvironment::GatherAdapterProperties(const dawn_native::Instance* instance) {
+void DawnTestEnvironment::SelectPreferredAdapterProperties(const dawn_native::Instance* instance) {
+    // Get the first available preferred device type.
+    dawn_native::DeviceType preferredDeviceType = static_cast<dawn_native::DeviceType>(-1);
+    bool hasDevicePreference = false;
+    for (dawn_native::DeviceType devicePreference : mDevicePreferences) {
         for (const dawn_native::Adapter& adapter : instance->GetAdapters()) {
             wgpu::AdapterProperties properties;
             adapter.GetProperties(&properties);
 
-        mAdapterProperties.emplace_back(
+            if (adapter.GetDeviceType() == devicePreference) {
-            properties, mHasVendorIdFilter && mVendorIdFilter == properties.vendorID);
+                preferredDeviceType = devicePreference;
-    }
+                hasDevicePreference = true;
-}
-
-std::vector<DawnTestParam> DawnTestEnvironment::FilterBackends(const DawnTestParam* params,
-                                                               size_t numParams) const {
-    std::vector<DawnTestParam> backends;
-    for (size_t i = 0; i < numParams; ++i) {
-        for (const auto& adapterProperties : mAdapterProperties) {
-            if (params[i].backendType == adapterProperties.backendType) {
-                backends.push_back(params[i]);
                 break;
             }
         }
+        if (hasDevicePreference) {
+            break;
         }
-    return backends;
+    }
+
+    for (const dawn_native::Adapter& adapter : instance->GetAdapters()) {
+        wgpu::AdapterProperties properties;
+        adapter.GetProperties(&properties);
+
+        // The adapter is selected if:
+        bool selected = false;
+        if (mHasVendorIdFilter) {
+            // It matches the vendor id, if present.
+            selected = mVendorIdFilter == properties.vendorID;
+
+            if (!mDevicePreferences.empty()) {
+                dawn::WarningLog() << "Vendor ID filter provided. Ignoring device type preference.";
+            }
+        } else if (hasDevicePreference) {
+            // There is a device preference and:
+            selected =
+                // The device type matches the first available preferred type for that backend, if
+                // present.
+                (adapter.GetDeviceType() == preferredDeviceType) ||
+                // Always select Unknown OpenGL adapters if we don't want a CPU adapter.
+                // OpenGL will usually be unknown because we can't query the device type.
+                // If we ever have Swiftshader GL (unlikely), we could set the DeviceType properly.
+                (preferredDeviceType != dawn_native::DeviceType::CPU &&
+                 adapter.GetDeviceType() == dawn_native::DeviceType::Unknown &&
+                 properties.backendType == wgpu::BackendType::OpenGL) ||
+                // Always select the Null backend. There are few tests on this backend, and they run
+                // quickly. This is temporary as to not lose coverage. We can group it with
+                // Swiftshader as a CPU adapter when we have Swiftshader tests.
+                (properties.backendType == wgpu::BackendType::Null);
+        } else {
+            // No vendor id or device preference was provided (select all).
+            selected = true;
+        }
+
+        mAdapterProperties.emplace_back(properties, selected);
+    }
+}
+
+std::vector<AdapterTestParam> DawnTestEnvironment::GetAvailableAdapterTestParamsForBackends(
+    const BackendTestConfig* params,
+    size_t numParams) {
+    std::vector<AdapterTestParam> testParams;
+    for (size_t i = 0; i < numParams; ++i) {
+        for (const auto& adapterProperties : mAdapterProperties) {
+            if (params[i].backendType == adapterProperties.backendType &&
+                adapterProperties.selected) {
+                testParams.push_back(AdapterTestParam(params[i], adapterProperties));
+            }
+        }
+    }
+    return testParams;
 }
 
 void DawnTestEnvironment::PrintTestConfigurationAndAdapterInfo() const {
@@ -476,7 +570,7 @@ class WireServerTraceLayer : public dawn_wire::CommandHandler {
 
 // Implementation of DawnTest
 
-DawnTestBase::DawnTestBase(const DawnTestParam& param) : mParam(param) {
+DawnTestBase::DawnTestBase(const AdapterTestParam& param) : mParam(param) {
 }
 
 DawnTestBase::~DawnTestBase() {
@@ -495,51 +589,52 @@ DawnTestBase::~DawnTestBase() {
 }
 
 bool DawnTestBase::IsD3D12() const {
-    return mParam.backendType == wgpu::BackendType::D3D12;
+    return mParam.adapterProperties.backendType == wgpu::BackendType::D3D12;
 }
 
 bool DawnTestBase::IsMetal() const {
-    return mParam.backendType == wgpu::BackendType::Metal;
+    return mParam.adapterProperties.backendType == wgpu::BackendType::Metal;
 }
 
 bool DawnTestBase::IsNull() const {
-    return mParam.backendType == wgpu::BackendType::Null;
+    return mParam.adapterProperties.backendType == wgpu::BackendType::Null;
 }
 
 bool DawnTestBase::IsOpenGL() const {
-    return mParam.backendType == wgpu::BackendType::OpenGL;
+    return mParam.adapterProperties.backendType == wgpu::BackendType::OpenGL;
 }
 
 bool DawnTestBase::IsVulkan() const {
-    return mParam.backendType == wgpu::BackendType::Vulkan;
+    return mParam.adapterProperties.backendType == wgpu::BackendType::Vulkan;
 }
 
 bool DawnTestBase::IsAMD() const {
-    return gpu_info::IsAMD(mAdapterProperties.vendorID);
+    return gpu_info::IsAMD(mParam.adapterProperties.vendorID);
 }
 
 bool DawnTestBase::IsARM() const {
-    return gpu_info::IsARM(mAdapterProperties.vendorID);
+    return gpu_info::IsARM(mParam.adapterProperties.vendorID);
 }
 
 bool DawnTestBase::IsImgTec() const {
-    return gpu_info::IsImgTec(mAdapterProperties.vendorID);
+    return gpu_info::IsImgTec(mParam.adapterProperties.vendorID);
 }
 
 bool DawnTestBase::IsIntel() const {
-    return gpu_info::IsIntel(mAdapterProperties.vendorID);
+    return gpu_info::IsIntel(mParam.adapterProperties.vendorID);
 }
 
 bool DawnTestBase::IsNvidia() const {
-    return gpu_info::IsNvidia(mAdapterProperties.vendorID);
+    return gpu_info::IsNvidia(mParam.adapterProperties.vendorID);
 }
 
 bool DawnTestBase::IsQualcomm() const {
-    return gpu_info::IsQualcomm(mAdapterProperties.vendorID);
+    return gpu_info::IsQualcomm(mParam.adapterProperties.vendorID);
 }
 
 bool DawnTestBase::IsSwiftshader() const {
-    return gpu_info::IsSwiftshader(mAdapterProperties.vendorID, mAdapterProperties.deviceID);
+    return gpu_info::IsSwiftshader(mParam.adapterProperties.vendorID,
+                                   mParam.adapterProperties.deviceID);
 }
 
 bool DawnTestBase::IsWindows() const {
@@ -607,14 +702,11 @@ std::vector<const char*> DawnTestBase::GetRequiredExtensions() {
 }
 
 const wgpu::AdapterProperties& DawnTestBase::GetAdapterProperties() const {
-    return mAdapterProperties;
+    return mParam.adapterProperties;
 }
 
-// This function can only be called after SetUp() because it requires mBackendAdapter to be
-// initialized.
 bool DawnTestBase::SupportsExtensions(const std::vector<const char*>& extensions) {
     ASSERT(mBackendAdapter);
-
     std::set<std::string> supportedExtensionsSet;
     for (const char* supportedExtensionName : mBackendAdapter.GetSupportedExtensions()) {
         supportedExtensionsSet.insert(supportedExtensionName);
@@ -630,69 +722,26 @@ bool DawnTestBase::SupportsExtensions(const std::vector<const char*>& extensions
 }
 
 void DawnTestBase::SetUp() {
-    // Initialize mBackendAdapter, and create the device.
-    const wgpu::BackendType backendType = mParam.backendType;
     {
-        dawn_native::Instance* instance = gTestEnv->GetInstance();
+        // Find the adapter that exactly matches our adapter properties.
-        std::vector<dawn_native::Adapter> adapters = instance->GetAdapters();
+        const auto& adapters = gTestEnv->GetInstance()->GetAdapters();
-
+        const auto& it = std::find_if(
-        static constexpr size_t kInvalidIndex = std::numeric_limits<size_t>::max();
+            adapters.begin(), adapters.end(), [&](const dawn_native::Adapter& adapter) {
-        size_t discreteAdapterIndex = kInvalidIndex;
-        size_t integratedAdapterIndex = kInvalidIndex;
-        size_t cpuAdapterIndex = kInvalidIndex;
-        size_t unknownAdapterIndex = kInvalidIndex;
-
-        for (size_t i = 0; i < adapters.size(); ++i) {
-            const dawn_native::Adapter& adapter = adapters[i];
-
                 wgpu::AdapterProperties properties;
                 adapter.GetProperties(&properties);
 
-            if (properties.backendType == backendType) {
+                return (mParam.adapterProperties.selected &&
-                // If the vendor id doesn't match, skip this adapter.
+                        properties.deviceID == mParam.adapterProperties.deviceID &&
-                if (HasVendorIdFilter() && properties.vendorID != GetVendorIdFilter()) {
+                        properties.vendorID == mParam.adapterProperties.vendorID &&
-                    continue;
+                        properties.adapterType == mParam.adapterProperties.adapterType &&
-                }
+                        properties.backendType == mParam.adapterProperties.backendType &&
-
+                        strcmp(properties.name, mParam.adapterProperties.adapterName.c_str()) == 0);
-                // Find the index of each type of adapter.
+            });
-                switch (adapter.GetDeviceType()) {
+        ASSERT(it != adapters.end());
-                    case dawn_native::DeviceType::DiscreteGPU:
+        mBackendAdapter = *it;
-                        discreteAdapterIndex = i;
-                        break;
-                    case dawn_native::DeviceType::IntegratedGPU:
-                        integratedAdapterIndex = i;
-                        break;
-                    case dawn_native::DeviceType::CPU:
-                        cpuAdapterIndex = i;
-                        break;
-                    case dawn_native::DeviceType::Unknown:
-                        unknownAdapterIndex = i;
-                        break;
-                    default:
-                        UNREACHABLE();
-                        break;
-                }
-            }
-        }
-
-        // Prefer, discrete, then integrated, then CPU, then unknown adapters.
-        if (discreteAdapterIndex != kInvalidIndex) {
-            mBackendAdapter = adapters[discreteAdapterIndex];
-        } else if (integratedAdapterIndex != kInvalidIndex) {
-            mBackendAdapter = adapters[integratedAdapterIndex];
-        } else if (cpuAdapterIndex != kInvalidIndex) {
-            mBackendAdapter = adapters[cpuAdapterIndex];
-        } else if (unknownAdapterIndex != kInvalidIndex) {
-            mBackendAdapter = adapters[unknownAdapterIndex];
-        }
-
-        if (!mBackendAdapter) {
-            return;
-        }
-
-        mBackendAdapter.GetProperties(&mAdapterProperties);
     }
 
+    // Create the device from the adapter
     for (const char* forceEnabledWorkaround : mParam.forceEnabledWorkarounds) {
         ASSERT(gTestEnv->GetInstance()->GetToggleInfo(forceEnabledWorkaround) != nullptr);
     }
@@ -800,10 +849,6 @@ void DawnTestBase::TearDown() {
     }
 }
 
-bool DawnTestBase::HasAdapter() const {
-    return !!mBackendAdapter;
-}
-
 void DawnTestBase::StartExpectDeviceError() {
     mExpectError = true;
     mError = false;
@@ -1027,9 +1072,11 @@ std::ostream& operator<<(std::ostream& stream, const RGBA8& color) {
 }
 
 namespace detail {
-    std::vector<DawnTestParam> FilterBackends(const DawnTestParam* params, size_t numParams) {
+    std::vector<AdapterTestParam> GetAvailableAdapterTestParamsForBackends(
+        const BackendTestConfig* params,
+        size_t numParams) {
         ASSERT(gTestEnv != nullptr);
-        return gTestEnv->FilterBackends(params, numParams);
+        return gTestEnv->GetAvailableAdapterTestParamsForBackends(params, numParams);
     }
 
     // Helper classes to set expectations

src/tests/DawnTest.h

@@ -89,6 +89,17 @@ struct RGBA8 {
 };
 std::ostream& operator<<(std::ostream& stream, const RGBA8& color);
 
+struct BackendTestConfig {
+    BackendTestConfig(wgpu::BackendType backendType,
+                      std::initializer_list<const char*> forceEnabledWorkarounds = {},
+                      std::initializer_list<const char*> forceDisabledWorkarounds = {});
+
+    wgpu::BackendType backendType;
+
+    std::vector<const char*> forceEnabledWorkarounds;
+    std::vector<const char*> forceDisabledWorkarounds;
+};
+
 struct TestAdapterProperties : wgpu::AdapterProperties {
     TestAdapterProperties(const wgpu::AdapterProperties& properties, bool selected);
     std::string adapterName;
@@ -99,32 +110,30 @@ struct TestAdapterProperties : wgpu::AdapterProperties {
     using wgpu::AdapterProperties::name;
 };
 
-struct DawnTestParam {
+struct AdapterTestParam {
-    DawnTestParam(wgpu::BackendType backendType,
+    AdapterTestParam(const BackendTestConfig& config,
-                  std::initializer_list<const char*> forceEnabledWorkarounds = {},
+                     const TestAdapterProperties& adapterProperties);
-                  std::initializer_list<const char*> forceDisabledWorkarounds = {});
-
-    wgpu::BackendType backendType;
 
+    TestAdapterProperties adapterProperties;
     std::vector<const char*> forceEnabledWorkarounds;
     std::vector<const char*> forceDisabledWorkarounds;
 };
 
-std::ostream& operator<<(std::ostream& os, const DawnTestParam& param);
+std::ostream& operator<<(std::ostream& os, const AdapterTestParam& param);
 
-DawnTestParam D3D12Backend(std::initializer_list<const char*> forceEnabledWorkarounds = {},
+BackendTestConfig D3D12Backend(std::initializer_list<const char*> forceEnabledWorkarounds = {},
                                std::initializer_list<const char*> forceDisabledWorkarounds = {});
 
-DawnTestParam MetalBackend(std::initializer_list<const char*> forceEnabledWorkarounds = {},
+BackendTestConfig MetalBackend(std::initializer_list<const char*> forceEnabledWorkarounds = {},
                                std::initializer_list<const char*> forceDisabledWorkarounds = {});
 
-DawnTestParam NullBackend(std::initializer_list<const char*> forceEnabledWorkarounds = {},
+BackendTestConfig NullBackend(std::initializer_list<const char*> forceEnabledWorkarounds = {},
                               std::initializer_list<const char*> forceDisabledWorkarounds = {});
 
-DawnTestParam OpenGLBackend(std::initializer_list<const char*> forceEnabledWorkarounds = {},
+BackendTestConfig OpenGLBackend(std::initializer_list<const char*> forceEnabledWorkarounds = {},
                                 std::initializer_list<const char*> forceDisabledWorkarounds = {});
 
-DawnTestParam VulkanBackend(std::initializer_list<const char*> forceEnabledWorkarounds = {},
+BackendTestConfig VulkanBackend(std::initializer_list<const char*> forceEnabledWorkarounds = {},
                                 std::initializer_list<const char*> forceDisabledWorkarounds = {});
 
 namespace utils {
@@ -150,7 +159,9 @@ class DawnTestEnvironment : public testing::Environment {
 
     static void SetEnvironment(DawnTestEnvironment* env);
 
-    std::vector<DawnTestParam> FilterBackends(const DawnTestParam* params, size_t numParams) const;
+    std::vector<AdapterTestParam> GetAvailableAdapterTestParamsForBackends(
+        const BackendTestConfig* params,
+        size_t numParams);
 
     void SetUp() override;
     void TearDown() override;
@@ -171,7 +182,7 @@ class DawnTestEnvironment : public testing::Environment {
   private:
     void ParseArgs(int argc, char** argv);
     std::unique_ptr<dawn_native::Instance> CreateInstanceAndDiscoverAdapters() const;
-    void GatherAdapterProperties(const dawn_native::Instance* instance);
+    void SelectPreferredAdapterProperties(const dawn_native::Instance* instance);
     void PrintTestConfigurationAndAdapterInfo() const;
 
     bool mUseWire = false;
@@ -185,6 +196,7 @@ class DawnTestEnvironment : public testing::Environment {
     bool mHasVendorIdFilter = false;
     uint32_t mVendorIdFilter = 0;
     std::string mWireTraceDir;
+    std::vector<dawn_native::DeviceType> mDevicePreferences;
     std::vector<TestAdapterProperties> mAdapterProperties;
 };
 
@@ -192,7 +204,7 @@ class DawnTestBase {
     friend class DawnPerfTestBase;
 
   public:
-    DawnTestBase(const DawnTestParam& param);
+    DawnTestBase(const AdapterTestParam& param);
     virtual ~DawnTestBase();
 
     void SetUp();
@@ -257,7 +269,6 @@ class DawnTestBase {
                                               uint32_t pixelSize,
                                               detail::Expectation* expectation);
 
-    bool HasAdapter() const;
     void WaitABit();
     void FlushWire();
 
@@ -272,7 +283,7 @@ class DawnTestBase {
     const wgpu::AdapterProperties& GetAdapterProperties() const;
 
   private:
-    DawnTestParam mParam;
+    AdapterTestParam mParam;
 
     // Things used to set up testing through the Wire.
     std::unique_ptr<dawn_wire::WireServer> mWireServer;
@@ -331,7 +342,6 @@ class DawnTestBase {
     void ResolveExpectations();
 
     dawn_native::Adapter mBackendAdapter;
-    wgpu::AdapterProperties mAdapterProperties;
 };
 
 // Skip a test when the given condition is satisfied.
@@ -344,15 +354,16 @@ class DawnTestBase {
         }                                                       \
     } while (0)
 
-template <typename Params = DawnTestParam>
+template <typename Params = AdapterTestParam>
 class DawnTestWithParams : public DawnTestBase, public ::testing::TestWithParam<Params> {
   private:
     void SetUp() override final {
-        // DawnTestBase::SetUp() gets the adapter, and creates the device and wire.
+        // DawnTestBase::SetUp() creates the device and wire.
-        // It's separate from TestSetUp() so we can skip tests completely if no adapter
+        // It was separate from TestSetUp() so we can skip tests completely if no adapter
         // is available.
+        // TODO(enga): There is now always an available adapter otherwise we fail test environment
+        // SetUp. Consider removing the extra TestSetUp() overload.
         DawnTestBase::SetUp();
-        DAWN_SKIP_TEST_IF(!HasAdapter());
         TestSetUp();
     }
 
@@ -384,14 +395,16 @@ using DawnTest = DawnTestWithParams<>;
     const decltype(DAWN_PP_GET_HEAD(__VA_ARGS__)) testName##params[] = {__VA_ARGS__};   \
     INSTANTIATE_TEST_SUITE_P(                                                           \
         , testName,                                                                     \
-        testing::ValuesIn(::detail::FilterBackends(                                     \
+        testing::ValuesIn(::detail::GetAvailableAdapterTestParamsForBackends(           \
             testName##params, sizeof(testName##params) / sizeof(testName##params[0]))), \
         testing::PrintToStringParamName())
 
 namespace detail {
     // Helper functions used for DAWN_INSTANTIATE_TEST
     bool IsBackendAvailable(wgpu::BackendType type);
-    std::vector<DawnTestParam> FilterBackends(const DawnTestParam* params, size_t numParams);
+    std::vector<AdapterTestParam> GetAvailableAdapterTestParamsForBackends(
+        const BackendTestConfig* params,
+        size_t numParams);
 
     // All classes used to implement the deferred expectations should inherit from this.
     class Expectation {

src/tests/ParamGenerator.h

@@ -19,7 +19,7 @@
 #include <vector>
 
 // ParamStruct is a custom struct which ParamStruct will yield when iterating.
-// The types Params... should by the same as the types passed to the constructor
+// The types Params... should be the same as the types passed to the constructor
 // of ParamStruct.
 template <typename ParamStruct, typename... Params>
 class ParamGenerator {
@@ -28,19 +28,6 @@ class ParamGenerator {
 
     static constexpr auto s_indexSequence = std::make_index_sequence<sizeof...(Params)>{};
 
-    // Default template that returns the same params.
-    template <typename P>
-    static std::vector<P> FilterBackends(std::vector<P> params) {
-        return params;
-    }
-
-    // Template specialization for DawnTestParam that filters the backends by
-    // those supported.
-    template <>
-    static std::vector<DawnTestParam> FilterBackends(std::vector<DawnTestParam> params) {
-        return ::detail::FilterBackends(params.data(), params.size());
-    }
-
     // Using an N-dimensional Index, extract params from ParamTuple and pass
     // them to the constructor of ParamStruct.
     template <size_t... Is>
@@ -59,7 +46,7 @@ class ParamGenerator {
   public:
     using value_type = ParamStruct;
 
-    ParamGenerator(std::vector<Params>... params) : mParams(FilterBackends(params)...) {
+    ParamGenerator(std::vector<Params>... params) : mParams(params...) {
     }
 
     class Iterator : public std::iterator<std::forward_iterator_tag, ParamStruct, size_t> {
@@ -121,8 +108,10 @@ class ParamGenerator {
 };
 
 template <typename Param, typename... Params>
-auto MakeParamGenerator(std::initializer_list<Params>&&... params) {
+auto MakeParamGenerator(std::vector<BackendTestConfig>&& first,
-    return ParamGenerator<Param, Params...>(
+                        std::initializer_list<Params>&&... params) {
+    return ParamGenerator<Param, AdapterTestParam, Params...>(
+        ::detail::GetAvailableAdapterTestParamsForBackends(first.data(), first.size()),
         std::forward<std::initializer_list<Params>&&>(params)...);
 }
 

src/tests/perf_tests/BufferUploadPerf.cpp

@@ -37,11 +37,11 @@ namespace {
         BufferSize_16MB = 16 * 1024 * 1024,
     };
 
-    struct BufferUploadParams : DawnTestParam {
+    struct BufferUploadParams : AdapterTestParam {
-        BufferUploadParams(const DawnTestParam& param,
+        BufferUploadParams(const AdapterTestParam& param,
                            UploadMethod uploadMethod,
                            UploadSize uploadSize)
-            : DawnTestParam(param), uploadMethod(uploadMethod), uploadSize(uploadSize) {
+            : AdapterTestParam(param), uploadMethod(uploadMethod), uploadSize(uploadSize) {
         }
 
         UploadMethod uploadMethod;
@@ -49,7 +49,7 @@ namespace {
     };
 
     std::ostream& operator<<(std::ostream& ostream, const BufferUploadParams& param) {
-        ostream << static_cast<const DawnTestParam&>(param);
+        ostream << static_cast<const AdapterTestParam&>(param);
 
         switch (param.uploadMethod) {
             case UploadMethod::SetSubData:

src/tests/perf_tests/DawnPerfTest.h

@@ -108,7 +108,7 @@ class DawnPerfTestBase {
     std::unique_ptr<utils::Timer> mTimer;
 };
 
-template <typename Params = DawnTestParam>
+template <typename Params = AdapterTestParam>
 class DawnPerfTestWithParams : public DawnTestWithParams<Params>, public DawnPerfTestBase {
   protected:
     DawnPerfTestWithParams(unsigned int iterationsPerStep, unsigned int maxStepsInFlight)

src/tests/perf_tests/DrawCallPerf.cpp

@@ -136,15 +136,15 @@ namespace {
         };
     }
 
-    struct DrawCallParamForTest : DawnTestParam {
+    struct DrawCallParamForTest : AdapterTestParam {
-        DrawCallParamForTest(const DawnTestParam& backendParam, DrawCallParam param)
+        DrawCallParamForTest(const AdapterTestParam& backendParam, DrawCallParam param)
-            : DawnTestParam(backendParam), param(param) {
+            : AdapterTestParam(backendParam), param(param) {
         }
         DrawCallParam param;
     };
 
     std::ostream& operator<<(std::ostream& ostream, const DrawCallParamForTest& testParams) {
-        ostream << static_cast<const DawnTestParam&>(testParams);
+        ostream << static_cast<const AdapterTestParam&>(testParams);
 
         const DrawCallParam& param = testParams.param;