DawnTest: Allow checking for device errors.

The previous assumption is that all validation tests would be unittests
and that the end2end tests would always produce valid sequences of
commands. This not true because we can't test the validation of
backend-specific entrypoints in the unittests.

BUG=dawn:112

Change-Id: I89e2fe017bf3ecf6a83c9e8cdf4324c33f95a721
Reviewed-on: https://dawn-review.googlesource.com/c/5100
Reviewed-by: Kai Ninomiya <kainino@chromium.org>
Reviewed-by: Austin Eng <enga@chromium.org>
Commit-Queue: Corentin Wallez <cwallez@chromium.org>

src/tests/DawnTest.cpp

@@ -52,12 +52,6 @@ namespace {
         }
     }
 
-    // End2end tests should test valid commands produce the expected result so no error
-    // should happen. Failure cases should be tested in the validation tests.
-    void DeviceErrorCauseTestFailure(const char* message, dawnCallbackUserdata) {
-        FAIL() << "Device level failure: " << message;
-    }
-
     struct MapReadUserdata {
         DawnTest* test;
         size_t slot;
@@ -315,8 +309,8 @@ void DawnTest::SetUp() {
         static_cast<dawn::TextureFormat>(mBinding->GetPreferredSwapChainTextureFormat()),
         dawn::TextureUsageBit::OutputAttachment, 400, 400);
 
-    // The end2end tests should never cause validation errors. These should be tested in unittests.
-    device.SetErrorCallback(DeviceErrorCauseTestFailure, 0);
+    device.SetErrorCallback(OnDeviceError,
+                            static_cast<dawnCallbackUserdata>(reinterpret_cast<uintptr_t>(this)));
 }
 
 void DawnTest::TearDown() {
@@ -330,6 +324,24 @@ void DawnTest::TearDown() {
     }
 }
 
+void DawnTest::StartExpectDeviceError() {
+    mExpectError = true;
+    mError = false;
+}
+bool DawnTest::EndExpectDeviceError() {
+    mExpectError = false;
+    return mError;
+}
+
+// static
+void DawnTest::OnDeviceError(const char* message, dawnCallbackUserdata userdata) {
+    DawnTest* self = reinterpret_cast<DawnTest*>(static_cast<uintptr_t>(userdata));
+
+    ASSERT_TRUE(self->mExpectError) << "Got unexpected device error: " << message;
+    ASSERT_FALSE(self->mError) << "Got two errors in expect block";
+    self->mError = true;
+}
+
 std::ostringstream& DawnTest::AddBufferExpectation(const char* file,
                                                    int line,
                                                    const dawn::Buffer& buffer,

src/tests/DawnTest.h

@@ -41,6 +41,13 @@
                           sizeof(RGBA8),                                                  \
                           new detail::ExpectEq<RGBA8>(expected, (width) * (height)))
 
+// Should only be used to test validation of function that can't be tested by regular validation
+// tests;
+#define ASSERT_DEVICE_ERROR(statement) \
+    StartExpectDeviceError();          \
+    statement;                         \
+    ASSERT_TRUE(EndExpectDeviceError());
+
 struct RGBA8 {
     constexpr RGBA8() : RGBA8(0, 0, 0, 0) {
     }
@@ -124,6 +131,9 @@ class DawnTest : public ::testing::TestWithParam<dawn_native::BackendType> {
     bool IsLinux() const;
     bool IsMacOS() const;
 
+    void StartExpectDeviceError();
+    bool EndExpectDeviceError();
+
   protected:
     dawn::Device device;
     dawn::Queue queue;
@@ -160,6 +170,11 @@ class DawnTest : public ::testing::TestWithParam<dawn_native::BackendType> {
     std::unique_ptr<utils::TerribleCommandBuffer> mS2cBuf;
     void FlushWire();
 
+    // Tracking for validation errors
+    static void OnDeviceError(const char* message, dawnCallbackUserdata userdata);
+    bool mExpectError = false;
+    bool mError = false;
+
     // MapRead buffers used to get data for the expectations
     struct ReadbackSlot {
         dawn::Buffer buffer;

src/tests/end2end/BasicTests.cpp

@@ -33,4 +33,16 @@ TEST_P(BasicTests, BufferSetSubData) {
     EXPECT_BUFFER_U32_EQ(value, buffer, 0);
 }
 
+// Test a validation error for buffer setSubData, but really this is the most basic test possible
+// for ASSERT_DEVICE_ERROR
+TEST_P(BasicTests, BufferSetSubDataError) {
+    dawn::BufferDescriptor descriptor;
+    descriptor.size = 4;
+    descriptor.usage = dawn::BufferUsageBit::TransferSrc | dawn::BufferUsageBit::TransferDst;
+    dawn::Buffer buffer = device.CreateBuffer(&descriptor);
+
+    uint8_t value = 187;
+    ASSERT_DEVICE_ERROR(buffer.SetSubData(1000, sizeof(value), &value));
+}
+
 DAWN_INSTANTIATE_TEST(BasicTests, D3D12Backend, MetalBackend, OpenGLBackend, VulkanBackend);

src/tests/unittests/validation/ValidationTest.cpp

@@ -84,6 +84,7 @@ std::string ValidationTest::GetLastDeviceErrorMessage() const {
     return mDeviceErrorMessage;
 }
 
+// static
 void ValidationTest::OnDeviceError(const char* message, dawnCallbackUserdata userdata) {
     auto self = reinterpret_cast<ValidationTest*>(static_cast<uintptr_t>(userdata));
     self->mDeviceErrorMessage = message;
@@ -100,6 +101,7 @@ void ValidationTest::OnDeviceError(const char* message, dawnCallbackUserdata use
     self->mError = true;
 }
 
+// static
 void ValidationTest::OnBuilderErrorStatus(dawnBuilderErrorStatus status, const char* message, dawn::CallbackUserdata userdata1, dawn::CallbackUserdata userdata2) {
     auto* self = reinterpret_cast<ValidationTest*>(static_cast<uintptr_t>(userdata1));
     size_t index = static_cast<size_t>(userdata2);