Use a common helper for std::nothrow

It's come up multiple times that ASAN doesn't support
std::nothrow which leads to OOM bugs filed by the fuzzers.
Use a common helper to avoid this and return nullptr for large
allocations when ASAN is enabled.

Bug: none
Change-Id: I492b4ff4e498cf82d4ca08ba849671d3d16b9cfb
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/36280
Reviewed-by: Jiawei Shao <jiawei.shao@intel.com>
Reviewed-by: Corentin Wallez <cwallez@chromium.org>
Commit-Queue: Austin Eng <enga@chromium.org>

src/common/Alloc.h

@@ -0,0 +1,32 @@
+// Copyright 2020 The Dawn Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef COMMON_ALLOC_H_
+#define COMMON_ALLOC_H_
+
+#include <new>
+
+template <typename T>
+T* AllocNoThrow(size_t count) {
+#if defined(ADDRESS_SANITIZER)
+    if (count * sizeof(T) >= 0x70000000) {
+        // std::nothrow isn't implemented on ASAN and it has a 2GB allocation limit.
+        // Catch large allocations and error out so fuzzers make progress.
+        return nullptr;
+    }
+#endif
+    return new (std::nothrow) T[count];
+}
+
+#endif  // COMMON_ALLOC_H_

src/common/BUILD.gn

@@ -152,6 +152,7 @@ config("dawn_internal") {
 if (is_win || is_linux || is_chromeos || is_mac || is_fuchsia || is_android) {
   static_library("common") {
     sources = [
+      "Alloc.h",
       "Assert.cpp",
       "Assert.h",
       "BitSetIterator.h",

src/common/CMakeLists.txt

@@ -14,6 +14,7 @@
 
 add_library(dawn_common STATIC ${DAWN_DUMMY_FILE})
 target_sources(dawn_common PRIVATE
+    "Alloc.h"
     "Assert.cpp"
     "Assert.h"
     "BitSetIterator.h"

src/dawn_native/Buffer.cpp

@@ -14,6 +14,7 @@
 
 #include "dawn_native/Buffer.h"
 
+#include "common/Alloc.h"
 #include "common/Assert.h"
 #include "dawn_native/Commands.h"
 #include "dawn_native/Device.h"
@@ -56,8 +57,8 @@ namespace dawn_native {
                         descriptor->size < uint64_t(std::numeric_limits<size_t>::max());
 
                     if (isValidSize) {
-                        mFakeMappedData = std::unique_ptr<uint8_t[]>(new (std::nothrow)
-                                                                         uint8_t[descriptor->size]);
+                        mFakeMappedData =
+                            std::unique_ptr<uint8_t[]>(AllocNoThrow<uint8_t>(descriptor->size));
                     }
                 }
             }
@@ -298,7 +299,8 @@ namespace dawn_native {
         if (mSize == 0) {
             return reinterpret_cast<uint8_t*>(intptr_t(0xCAFED00D));
         }
-        return static_cast<uint8_t*>(GetMappedPointerImpl()) + offset;
+        uint8_t* start = static_cast<uint8_t*>(GetMappedPointerImpl());
+        return start == nullptr ? nullptr : start + offset;
     }
 
     void BufferBase::Destroy() {

src/dawn_wire/ChunkedCommandHandler.cpp

@@ -14,6 +14,8 @@
 
 #include "dawn_wire/ChunkedCommandHandler.h"
 
+#include "common/Alloc.h"
+
 #include <algorithm>
 #include <cstring>
 
@@ -60,16 +62,9 @@ namespace dawn_wire {
         size_t initialSize) {
         ASSERT(!mChunkedCommandData);
 
-#if defined(ADDRESS_SANITIZER)
-        if (commandSize >= 0x70000000) {
-            // std::nothrow isn't implemented on ASAN and it has a 2GB allocation limit.
-            // Catch large allocations and error out so fuzzers make progress.
-            return ChunkedCommandsResult::Error;
-        }
-#endif
         // Reserve space for all the command data we're expecting, and copy the initial data
         // to the start of the memory.
-        mChunkedCommandData.reset(new (std::nothrow) char[commandSize]);
+        mChunkedCommandData.reset(AllocNoThrow<char>(commandSize));
         if (!mChunkedCommandData) {
             return ChunkedCommandsResult::Error;
         }

src/dawn_wire/ChunkedCommandSerializer.h

@@ -15,6 +15,7 @@
 #ifndef DAWNWIRE_CHUNKEDCOMMANDSERIALIZER_H_
 #define DAWNWIRE_CHUNKEDCOMMANDSERIALIZER_H_
 
+#include "common/Alloc.h"
 #include "common/Compiler.h"
 #include "dawn_wire/Wire.h"
 #include "dawn_wire/WireCmd_autogen.h"
@@ -82,7 +83,7 @@ namespace dawn_wire {
                 return;
             }
 
-            auto cmdSpace = std::unique_ptr<char[]>(new (std::nothrow) char[requiredSize]);
+            auto cmdSpace = std::unique_ptr<char[]>(AllocNoThrow<char>(requiredSize));
             if (!cmdSpace) {
                 return;
             }

src/dawn_wire/client/ClientInlineMemoryTransferService.cpp

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "common/Alloc.h"
 #include "common/Assert.h"
 #include "dawn_wire/WireClient.h"
 #include "dawn_wire/client/Client.h"
@@ -43,7 +44,7 @@ namespace dawn_wire { namespace client {
                     return false;
                 }
 
-                mStagingData = std::unique_ptr<uint8_t[]>(new (std::nothrow) uint8_t[mSize]);
+                mStagingData = std::unique_ptr<uint8_t[]>(AllocNoThrow<uint8_t>(mSize));
                 if (!mStagingData) {
                     return false;
                 }
@@ -77,7 +78,7 @@ namespace dawn_wire { namespace client {
             }
 
             std::pair<void*, size_t> Open() override {
-                mStagingData = std::unique_ptr<uint8_t[]>(new (std::nothrow) uint8_t[mSize]);
+                mStagingData = std::unique_ptr<uint8_t[]>(AllocNoThrow<uint8_t>(mSize));
                 if (!mStagingData) {
                     return std::make_pair(nullptr, 0);
                 }