Add basic Dawn microbenchmarks using Google benchmark

The initial test tests bind group layout creation of
different sizes, cache / no-cache hit, with and without
multiple threads.

Change-Id: Ic9ed6c6f1c298d35cd1358c7ff492027c83649a7
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/127346
Commit-Queue: Austin Eng <enga@chromium.org>
Reviewed-by: Ben Clayton <bclayton@google.com>
Kokoro: Kokoro <noreply+kokoro@google.com>

CMakeLists.txt

@@ -133,6 +133,7 @@ option_if_not_defined(DAWN_USE_X11 "Enable support for X11 surface" ${USE_X11})
 option_if_not_defined(DAWN_BUILD_SAMPLES "Enables building Dawn's samples" ${BUILD_SAMPLES})
 option_if_not_defined(DAWN_BUILD_NODE_BINDINGS "Enables building Dawn's NodeJS bindings" OFF)
 option_if_not_defined(DAWN_ENABLE_SWIFTSHADER "Enables building Swiftshader as part of the build and Vulkan adapter discovery" OFF)
+option_if_not_defined(DAWN_BUILD_BENCHMARKS "Build Dawn benchmarks" OFF)
 
 option_if_not_defined(DAWN_ENABLE_PIC "Build with Position-Independent-Code enabled" OFF)
 
@@ -164,7 +165,7 @@ option_if_not_defined(TINT_BUILD_FUZZERS "Build fuzzers" OFF)
 option_if_not_defined(TINT_BUILD_SPIRV_TOOLS_FUZZER "Build SPIRV-Tools fuzzer" OFF)
 option_if_not_defined(TINT_BUILD_AST_FUZZER "Build AST fuzzer" OFF)
 option_if_not_defined(TINT_BUILD_REGEX_FUZZER "Build regex fuzzer" OFF)
-option_if_not_defined(TINT_BUILD_BENCHMARKS "Build benchmarks" OFF)
+option_if_not_defined(TINT_BUILD_BENCHMARKS "Build Tint benchmarks" OFF)
 option_if_not_defined(TINT_BUILD_TESTS "Build tests" ON)
 option_if_not_defined(TINT_BUILD_AS_OTHER_OS "Override OS detection to force building of *_other.cc files" OFF)
 option_if_not_defined(TINT_BUILD_REMOTE_COMPILE "Build the remote-compile tool for validating shaders on a remote machine" OFF)
@@ -277,6 +278,7 @@ message(STATUS "Dawn build X11 support: ${DAWN_USE_X11}")
 message(STATUS "Dawn build samples: ${DAWN_BUILD_SAMPLES}")
 message(STATUS "Dawn build Node bindings: ${DAWN_BUILD_NODE_BINDINGS}")
 message(STATUS "Dawn build Swiftshader: ${DAWN_ENABLE_SWIFTSHADER}")
+message(STATUS "Dawn build benchmarks: ${DAWN_BUILD_BENCHMARKS}")
 
 message(STATUS "Dawn build PIC: ${DAWN_ENABLE_PIC}")
 

DEPS

@@ -110,6 +110,10 @@ deps = {
     'url': '{chromium_git}/catapult.git@c1e70d412ce01fb194f73f7abfdac710aae87dae',
     'condition': 'dawn_standalone',
   },
+  'third_party/google_benchmark/src': {
+    'url': '{chromium_git}/external/github.com/google/benchmark.git' + '@' + 'efc89f0b524780b1994d5dddd83a92718e5be492',
+    'condition': 'dawn_standalone',
+  },
 
   # Jinja2 and MarkupSafe for the code generator
   'third_party/jinja2': {
@@ -204,10 +208,6 @@ deps = {
   },
 
   # Misc dependencies inherited from Tint
-  'third_party/benchmark': {
-    'url': '{chromium_git}/external/github.com/google/benchmark.git@e991355c02b93fe17713efe04cbc2e278e00fdbd',
-    'condition': 'dawn_standalone',
-  },
   'third_party/protobuf': {
     'url': '{chromium_git}/external/github.com/protocolbuffers/protobuf.git@fde7cf7358ec7cd69e8db9be4f1fa6a5c431386a',
     'condition': 'dawn_standalone',

src/dawn/CMakeLists.txt

@@ -23,6 +23,7 @@ add_subdirectory(wire)
 # TODO(dawn:269): Remove once the implementation-based swapchains are removed.
 add_subdirectory(utils)
 add_subdirectory(glfw)
+add_subdirectory(tests/benchmarks)
 
 if (DAWN_BUILD_SAMPLES)
     #TODO(dawn:269): Add this once implementation-based swapchains are removed.

src/dawn/tests/BUILD.gn

@@ -24,6 +24,7 @@ group("tests") {
     ":dawn_end2end_tests",
     ":dawn_perf_tests",
     ":dawn_unittests",
+    "${dawn_root}/src/dawn/tests/benchmarks:dawn_benchmarks",
   ]
 }
 

src/dawn/tests/benchmarks/BGLCreation.cpp

@@ -0,0 +1,115 @@
+// Copyright 2023 The Dawn Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <benchmark/benchmark.h>
+#include <dawn/webgpu_cpp.h>
+#include <array>
+#include <vector>
+
+#include "dawn/common/Log.h"
+#include "dawn/tests/benchmarks/NullDeviceSetup.h"
+
+static void RedundantBGLCreation(benchmark::State& state) {
+    static wgpu::Device device = nullptr;
+
+    if (state.thread_index() == 0) {
+        std::vector<wgpu::FeatureName> requiredFeatures;
+        if (state.threads() > 1) {
+            requiredFeatures.push_back(wgpu::FeatureName::ImplicitDeviceSynchronization);
+        }
+
+        wgpu::DeviceDescriptor deviceDesc = {};
+        deviceDesc.requiredFeatures = requiredFeatures.data();
+        deviceDesc.requiredFeaturesCount = requiredFeatures.size();
+        device = CreateNullDevice(deviceDesc);
+    }
+
+    std::vector<wgpu::BindGroupLayoutEntry> entries(state.range(0));
+    for (uint32_t i = 0; i < entries.size(); ++i) {
+        entries[i].binding = i;
+        entries[i].visibility = wgpu::ShaderStage::Vertex | wgpu::ShaderStage::Fragment;
+        entries[i].buffer.type = wgpu::BufferBindingType::Uniform;
+    }
+
+    wgpu::BindGroupLayoutDescriptor bglDesc = {};
+    bglDesc.entryCount = entries.size();
+    bglDesc.entries = entries.data();
+
+    thread_local std::vector<wgpu::BindGroupLayout> bgls;
+    bgls.reserve(100000);
+    for (auto _ : state) {
+        bgls.push_back(device.CreateBindGroupLayout(&bglDesc));
+    }
+    bgls.clear();
+
+    if (state.thread_index() == 0) {
+        device = nullptr;
+    }
+}
+
+static void UniqueBGLCreation(benchmark::State& state) {
+    static wgpu::Device device = nullptr;
+
+    if (state.thread_index() == 0) {
+        std::vector<wgpu::FeatureName> requiredFeatures;
+        if (state.threads() > 1) {
+            requiredFeatures.push_back(wgpu::FeatureName::ImplicitDeviceSynchronization);
+        }
+
+        wgpu::DeviceDescriptor deviceDesc = {};
+        deviceDesc.requiredFeatures = requiredFeatures.data();
+        deviceDesc.requiredFeaturesCount = requiredFeatures.size();
+        device = CreateNullDevice(deviceDesc);
+    }
+
+    std::vector<wgpu::BindGroupLayoutEntry> entries(state.range(0));
+    for (uint32_t i = 0; i < entries.size(); ++i) {
+        entries[i].binding = i;
+        entries[i].visibility = wgpu::ShaderStage::Vertex | wgpu::ShaderStage::Fragment;
+        entries[i].buffer.type = wgpu::BufferBindingType::Uniform;
+    }
+    entries[0].buffer.minBindingSize = 4u;
+
+    wgpu::BindGroupLayoutDescriptor bglDesc = {};
+    bglDesc.entryCount = entries.size();
+    bglDesc.entries = entries.data();
+
+    thread_local std::vector<wgpu::BindGroupLayout> bgls;
+    bgls.reserve(100000);
+    for (auto _ : state) {
+        entries[0].buffer.minBindingSize += 4;
+        bgls.push_back(device.CreateBindGroupLayout(&bglDesc));
+    }
+    bgls.clear();
+
+    if (state.thread_index() == 0) {
+        device = nullptr;
+    }
+}
+
+BENCHMARK(RedundantBGLCreation)
+    ->Setup(SetupNullBackend)
+    ->Arg(1)
+    ->Arg(12)
+    ->Threads(1)
+    ->Threads(4)
+    ->Threads(16);
+
+BENCHMARK(UniqueBGLCreation)
+    ->Setup(SetupNullBackend)
+    ->Arg(1)
+    ->Arg(12)
+    ->Threads(1)
+    ->Threads(4)
+    ->Threads(16);

src/dawn/tests/benchmarks/BUILD.gn

@@ -0,0 +1,35 @@
+# Copyright 2023 The Dawn Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import("../../../../scripts/dawn_overrides_with_defaults.gni")
+
+import("//testing/test.gni")
+
+test("dawn_benchmarks") {
+  deps = [
+    "${dawn_root}/src/dawn:cpp",
+    "${dawn_root}/src/dawn:proc",
+    "${dawn_root}/src/dawn/common",
+    "${dawn_root}/src/dawn/native:sources",
+    "${dawn_root}/src/dawn/native:static",
+    "//third_party/google_benchmark",
+    "//third_party/google_benchmark:benchmark_main",
+  ]
+  sources = [
+    "BGLCreation.cpp",
+    "NullDeviceSetup.cpp",
+    "NullDeviceSetup.h",
+  ]
+  configs += [ "${dawn_root}/include/dawn:public" ]
+}

src/dawn/tests/benchmarks/CMakeLists.txt

@@ -0,0 +1,34 @@
+# Copyright 2023 The Dawn Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if (${DAWN_BUILD_BENCHMARKS})
+  add_executable(dawn_benchmarks
+    "BGLCreation.cpp"
+    "NullDeviceSetup.cpp"
+    "NullDeviceSetup.h"
+  )
+  set_target_properties(dawn_benchmarks PROPERTIES FOLDER "Benchmarks")
+
+  target_include_directories(dawn_benchmarks PUBLIC "${PROJECT_SOURCE_DIR}/include")
+  target_include_directories(dawn_benchmarks PUBLIC "${PROJECT_SOURCE_DIR}/src")
+
+  target_link_libraries(dawn_benchmarks PRIVATE
+    benchmark::benchmark
+    benchmark::benchmark_main
+    dawn_common
+    dawn_native
+    dawncpp_headers
+    dawncpp
+    dawn_proc)
+endif()

src/dawn/tests/benchmarks/NullDeviceSetup.cpp

@@ -0,0 +1,84 @@
+// Copyright 2023 The Dawn Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "dawn/tests/benchmarks/NullDeviceSetup.h"
+
+#include <benchmark/benchmark.h>
+#include <dawn/webgpu_cpp.h>
+#include <memory>
+
+#include "dawn/common/Assert.h"
+#include "dawn/common/Log.h"
+#include "dawn/dawn_proc.h"
+#include "dawn/native/DawnNative.h"
+
+namespace {
+std::unique_ptr<dawn::native::Instance> nativeInstance = nullptr;
+WGPUAdapter nullBackendAdapter = nullptr;
+}  // namespace
+
+void SetupNullBackend(const benchmark::State& state) {
+    if (!nativeInstance) {
+        nativeInstance = std::make_unique<dawn::native::Instance>();
+        nativeInstance->DiscoverDefaultAdapters();
+    }
+
+    if (!nullBackendAdapter) {
+        for (auto& a : nativeInstance->GetAdapters()) {
+            wgpu::AdapterProperties properties;
+            a.GetProperties(&properties);
+            if (properties.backendType == wgpu::BackendType::Null) {
+                nullBackendAdapter = a.Get();
+            }
+        }
+    }
+    ASSERT(nullBackendAdapter != nullptr);
+}
+
+wgpu::Device CreateNullDevice(const wgpu::DeviceDescriptor& desc) {
+    dawnProcSetProcs(&dawn::native::GetProcs());
+
+    wgpu::Device device;
+
+    wgpu::Adapter(nullBackendAdapter)
+        .RequestDevice(
+            &desc,
+            [](WGPURequestDeviceStatus status, WGPUDevice cDevice, char const* message,
+               void* userdata) {
+                ASSERT(status == WGPURequestDeviceStatus_Success);
+                *reinterpret_cast<wgpu::Device*>(userdata) = wgpu::Device::Acquire(cDevice);
+            },
+            &device);
+    while (!device) {
+        wgpuInstanceProcessEvents(nativeInstance->Get());
+    }
+
+    device.SetUncapturedErrorCallback(
+        [](WGPUErrorType, char const* message, void* userdata) {
+            dawn::ErrorLog() << message;
+            UNREACHABLE();
+        },
+        nullptr);
+
+    device.SetDeviceLostCallback(
+        [](WGPUDeviceLostReason reason, char const* message, void* userdata) {
+            if (reason == WGPUDeviceLostReason_Undefined) {
+                dawn::ErrorLog() << message;
+                UNREACHABLE();
+            }
+        },
+        nullptr);
+
+    return device;
+}

src/dawn/tests/benchmarks/NullDeviceSetup.h

@@ -0,0 +1,31 @@
+// Copyright 2023 The Dawn Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef DAWN_TESTS_BENCHMARKS_NULLDEVICESETUP
+#define DAWN_TESTS_BENCHMARKS_NULLDEVICESETUP
+
+#include <dawn/webgpu_cpp.h>
+
+namespace benchmark {
+class State;
+}
+
+namespace wgpu {
+struct DeviceDescriptor;
+}
+
+void SetupNullBackend(const benchmark::State& state);
+wgpu::Device CreateNullDevice(const wgpu::DeviceDescriptor& desc);
+
+#endif  // DAWN_TESTS_BENCHMARKS_NULLDEVICESETUP

third_party/CMakeLists.txt

@@ -106,9 +106,9 @@ if (NOT TARGET vk_swiftshader AND ${DAWN_ENABLE_SWIFTSHADER})
     add_subdirectory(${DAWN_SWIFTSHADER_DIR} "${CMAKE_CURRENT_BINARY_DIR}/swiftshader")
 endif()
 
-if (${TINT_BUILD_BENCHMARKS})
+if (${TINT_BUILD_BENCHMARKS} OR ${DAWN_BUILD_BENCHMARKS})
     set(BENCHMARK_ENABLE_TESTING FALSE CACHE BOOL FALSE FORCE)
-    add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/benchmark EXCLUDE_FROM_ALL)
+    add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/google_benchmark/src EXCLUDE_FROM_ALL)
 endif()
 
 if (NOT TARGET gmock AND ${TINT_BUILD_TESTS})

third_party/google_benchmark/BUILD.gn

@@ -0,0 +1,91 @@
+# Copyright 2019 The Chromium Authors
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("//build_overrides/build.gni")
+
+config("benchmark_config") {
+  include_dirs = [ "src/include" ]
+
+  if (!is_component_build) {
+    defines = [ "BENCHMARK_STATIC_DEFINE" ]
+  }
+}
+
+component("google_benchmark") {
+  testonly = true
+
+  public = [
+    "src/include/benchmark/benchmark.h",
+    "src/include/benchmark/export.h",
+  ]
+
+  sources = [
+    "src/src/arraysize.h",
+    "src/src/benchmark.cc",
+    "src/src/benchmark_api_internal.cc",
+    "src/src/benchmark_api_internal.h",
+    "src/src/benchmark_name.cc",
+    "src/src/benchmark_register.cc",
+    "src/src/benchmark_register.h",
+    "src/src/benchmark_runner.cc",
+    "src/src/benchmark_runner.h",
+    "src/src/check.cc",
+    "src/src/check.h",
+    "src/src/colorprint.cc",
+    "src/src/colorprint.h",
+    "src/src/commandlineflags.cc",
+    "src/src/commandlineflags.h",
+    "src/src/complexity.cc",
+    "src/src/complexity.h",
+    "src/src/console_reporter.cc",
+    "src/src/counter.cc",
+    "src/src/counter.h",
+    "src/src/csv_reporter.cc",
+    "src/src/cycleclock.h",
+    "src/src/internal_macros.h",
+    "src/src/json_reporter.cc",
+    "src/src/log.h",
+    "src/src/mutex.h",
+    "src/src/perf_counters.cc",
+    "src/src/perf_counters.h",
+    "src/src/re.h",
+    "src/src/reporter.cc",
+    "src/src/statistics.cc",
+    "src/src/statistics.h",
+    "src/src/string_util.cc",
+    "src/src/string_util.h",
+    "src/src/sysinfo.cc",
+    "src/src/thread_manager.h",
+    "src/src/thread_timer.h",
+    "src/src/timers.cc",
+    "src/src/timers.h",
+  ]
+
+  all_dependent_configs = [ ":benchmark_config" ]
+
+  if (build_with_chromium) {
+    configs -= [ "//build/config/compiler:chromium_code" ]
+    configs += [ "//build/config/compiler:no_chromium_code" ]
+  }
+
+  if (is_win) {
+    configs -= [ "//build/config/win:nominmax" ]
+  }
+
+  defines = [
+    "benchmark_EXPORTS=1",
+
+    # Tell gtest to always use standard regular expressions.
+    "HAVE_GNU_POSIX_REGEX=0",
+    "HAVE_POSIX_REGEX=0",
+    "HAVE_STD_REGEX=1",
+  ]
+}
+
+component("benchmark_main") {
+  testonly = true
+  sources = [ "src/src/benchmark_main.cc" ]
+  defines = [ "benchmark_EXPORTS=1" ]
+  deps = [ ":google_benchmark" ]
+}

third_party/google_benchmark/README.chromium

@@ -0,0 +1,13 @@
+Name: Google Benchmark
+Short Name: benchmark
+URL: https://github.com/google/benchmark
+Version: efc89f0b524780b1994d5dddd83a92718e5be492
+License: Apache 2.0
+License File: NOT_SHIPPED
+Security Critical: no
+
+Description:
+A microbenchmark support library.
+
+Local Additions:
+* gn file for building in chromium