Return nullptr from MakeErrorMapped if OOM allocating staging data

Bug: chromium:970305
Change-Id: If5ea037170260b8b8dddf3e3428ad2e77d4b67a4
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/7800
Commit-Queue: Austin Eng <enga@chromium.org>
Reviewed-by: Kai Ninomiya <kainino@chromium.org>

src/dawn_native/Buffer.cpp

@@ -37,7 +37,8 @@ namespace dawn_native {
                 ASSERT(mappedPointer != nullptr);
 
                 ErrorBuffer* buffer = new ErrorBuffer(device);
-                buffer->mFakeMappedData = std::unique_ptr<uint8_t[]>(new uint8_t[size]);
+                buffer->mFakeMappedData =
+                    std::unique_ptr<uint8_t[]>(new (std::nothrow) uint8_t[size]);
                 *mappedPointer = buffer->mFakeMappedData.get();
 
                 return buffer;

src/dawn_native/Device.cpp

@@ -263,23 +263,30 @@ namespace dawn_native {
         BufferBase* buffer = nullptr;
         uint8_t* data = nullptr;
 
+        uint64_t size = descriptor->size;
         if (ConsumedError(CreateBufferInternal(&buffer, descriptor)) ||
             ConsumedError(buffer->MapAtCreation(&data))) {
             // Map failed. Replace the buffer with an error buffer.
             if (buffer != nullptr) {
                 delete buffer;
             }
-            buffer = BufferBase::MakeErrorMapped(this, descriptor->size, &data);
+            buffer = BufferBase::MakeErrorMapped(this, size, &data);
         }
 
         ASSERT(buffer != nullptr);
-        ASSERT(data != nullptr);
+        if (data == nullptr) {
+            // |data| may be nullptr if there was an OOM in MakeErrorMapped.
+            // Non-zero dataLength and nullptr data is used to indicate there should be
+            // mapped data but the allocation failed.
+            ASSERT(buffer->IsError());
+        } else {
+            memset(data, 0, size);
+        }
 
         DawnCreateBufferMappedResult result = {};
         result.buffer = reinterpret_cast<DawnBuffer>(buffer);
         result.data = data;
-        result.dataLength = descriptor->size;
+        result.dataLength = size;
-        memset(result.data, 0, result.dataLength);
 
         return result;
     }

src/dawn_wire/server/ObjectStorage.h

@@ -37,10 +37,13 @@ namespace dawn_wire { namespace server {
     template <typename T>
     struct ObjectData : public ObjectDataBase<T> {};
 
+    enum class BufferMapWriteState { Unmapped, Mapped, MapError };
+
     template <>
     struct ObjectData<DawnBuffer> : public ObjectDataBase<DawnBuffer> {
         void* mappedData = nullptr;
         size_t mappedDataSize = 0;
+        BufferMapWriteState mapWriteState = BufferMapWriteState::Unmapped;
     };
 
     // Keeps track of the mapping between client IDs and backend objects.

src/dawn_wire/server/ServerBuffer.cpp

@@ -20,10 +20,11 @@
 namespace dawn_wire { namespace server {
 
     bool Server::PreHandleBufferUnmap(const BufferUnmapCmd& cmd) {
-        auto* selfData = BufferObjects().Get(cmd.selfId);
+        auto* buffer = BufferObjects().Get(cmd.selfId);
-        DAWN_ASSERT(selfData != nullptr);
+        DAWN_ASSERT(buffer != nullptr);
 
-        selfData->mappedData = nullptr;
+        buffer->mappedData = nullptr;
+        buffer->mapWriteState = BufferMapWriteState::Unmapped;
 
         return true;
     }
@@ -70,7 +71,14 @@ namespace dawn_wire { namespace server {
 
         DawnCreateBufferMappedResult result = mProcs.deviceCreateBufferMapped(device, descriptor);
         ASSERT(result.buffer != nullptr);
-        ASSERT(result.data != nullptr);
+        if (result.data == nullptr && result.dataLength != 0) {
+            // Non-zero dataLength but null data is used to indicate an allocation error.
+            resultData->mapWriteState = BufferMapWriteState::MapError;
+        } else {
+            // The buffer is mapped and has a valid mappedData pointer.
+            // The buffer may still be an error with fake staging data.
+            resultData->mapWriteState = BufferMapWriteState::Mapped;
+        }
         resultData->handle = result.buffer;
         resultData->mappedData = result.data;
         resultData->mappedDataSize = result.dataLength;
@@ -103,11 +111,22 @@ namespace dawn_wire { namespace server {
         }
 
         auto* buffer = BufferObjects().Get(bufferId);
-        if (buffer == nullptr || buffer->mappedData == nullptr || buffer->mappedDataSize != count) {
+        if (buffer == nullptr || buffer->mappedDataSize != count) {
             return false;
         }
+        switch (buffer->mapWriteState) {
+            case BufferMapWriteState::Unmapped:
+                return false;
+            case BufferMapWriteState::MapError:
+                // The buffer is mapped but there was an error allocating mapped data.
+                // Do not perform the memcpy.
+                return true;
+            case BufferMapWriteState::Mapped:
+                break;
+        }
 
         ASSERT(data != nullptr);
+        ASSERT(buffer->mappedData != nullptr);
         memcpy(buffer->mappedData, data, count);
 
         return true;
@@ -180,6 +199,7 @@ namespace dawn_wire { namespace server {
         cmd.Serialize(allocatedBuffer);
 
         if (status == DAWN_BUFFER_MAP_ASYNC_STATUS_SUCCESS) {
+            bufferData->mapWriteState = BufferMapWriteState::Mapped;
             bufferData->mappedData = ptr;
             bufferData->mappedDataSize = dataLength;
         }