From 6a7bba54fa597011f6b34ace757385df744db7a0 Mon Sep 17 00:00:00 2001 From: Austin Eng Date: Mon, 17 Apr 2023 18:11:51 +0000 Subject: [PATCH] Add basic Dawn microbenchmarks using Google benchmark The initial test tests bind group layout creation of different sizes, cache / no-cache hit, with and without multiple threads. Change-Id: Ic9ed6c6f1c298d35cd1358c7ff492027c83649a7 Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/127346 Commit-Queue: Austin Eng Reviewed-by: Ben Clayton Kokoro: Kokoro --- CMakeLists.txt | 4 +- DEPS | 8 +- src/dawn/CMakeLists.txt | 1 + src/dawn/tests/BUILD.gn | 1 + src/dawn/tests/benchmarks/BGLCreation.cpp | 115 ++++++++++++++++++ src/dawn/tests/benchmarks/BUILD.gn | 35 ++++++ src/dawn/tests/benchmarks/CMakeLists.txt | 34 ++++++ src/dawn/tests/benchmarks/NullDeviceSetup.cpp | 84 +++++++++++++ src/dawn/tests/benchmarks/NullDeviceSetup.h | 31 +++++ third_party/CMakeLists.txt | 4 +- third_party/google_benchmark/BUILD.gn | 91 ++++++++++++++ third_party/google_benchmark/README.chromium | 13 ++ 12 files changed, 414 insertions(+), 7 deletions(-) create mode 100644 src/dawn/tests/benchmarks/BGLCreation.cpp create mode 100644 src/dawn/tests/benchmarks/BUILD.gn create mode 100644 src/dawn/tests/benchmarks/CMakeLists.txt create mode 100644 src/dawn/tests/benchmarks/NullDeviceSetup.cpp create mode 100644 src/dawn/tests/benchmarks/NullDeviceSetup.h create mode 100644 third_party/google_benchmark/BUILD.gn create mode 100644 third_party/google_benchmark/README.chromium diff --git a/CMakeLists.txt b/CMakeLists.txt index 346bfe2750..3eb8136745 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -133,6 +133,7 @@ option_if_not_defined(DAWN_USE_X11 "Enable support for X11 surface" ${USE_X11}) option_if_not_defined(DAWN_BUILD_SAMPLES "Enables building Dawn's samples" ${BUILD_SAMPLES}) option_if_not_defined(DAWN_BUILD_NODE_BINDINGS "Enables building Dawn's NodeJS bindings" OFF) option_if_not_defined(DAWN_ENABLE_SWIFTSHADER "Enables building Swiftshader as part of the build and Vulkan adapter discovery" OFF) +option_if_not_defined(DAWN_BUILD_BENCHMARKS "Build Dawn benchmarks" OFF) option_if_not_defined(DAWN_ENABLE_PIC "Build with Position-Independent-Code enabled" OFF) @@ -164,7 +165,7 @@ option_if_not_defined(TINT_BUILD_FUZZERS "Build fuzzers" OFF) option_if_not_defined(TINT_BUILD_SPIRV_TOOLS_FUZZER "Build SPIRV-Tools fuzzer" OFF) option_if_not_defined(TINT_BUILD_AST_FUZZER "Build AST fuzzer" OFF) option_if_not_defined(TINT_BUILD_REGEX_FUZZER "Build regex fuzzer" OFF) -option_if_not_defined(TINT_BUILD_BENCHMARKS "Build benchmarks" OFF) +option_if_not_defined(TINT_BUILD_BENCHMARKS "Build Tint benchmarks" OFF) option_if_not_defined(TINT_BUILD_TESTS "Build tests" ON) option_if_not_defined(TINT_BUILD_AS_OTHER_OS "Override OS detection to force building of *_other.cc files" OFF) option_if_not_defined(TINT_BUILD_REMOTE_COMPILE "Build the remote-compile tool for validating shaders on a remote machine" OFF) @@ -277,6 +278,7 @@ message(STATUS "Dawn build X11 support: ${DAWN_USE_X11}") message(STATUS "Dawn build samples: ${DAWN_BUILD_SAMPLES}") message(STATUS "Dawn build Node bindings: ${DAWN_BUILD_NODE_BINDINGS}") message(STATUS "Dawn build Swiftshader: ${DAWN_ENABLE_SWIFTSHADER}") +message(STATUS "Dawn build benchmarks: ${DAWN_BUILD_BENCHMARKS}") message(STATUS "Dawn build PIC: ${DAWN_ENABLE_PIC}") diff --git a/DEPS b/DEPS index 363c82ece3..03b9b4e7fd 100644 --- a/DEPS +++ b/DEPS @@ -110,6 +110,10 @@ deps = { 'url': '{chromium_git}/catapult.git@c1e70d412ce01fb194f73f7abfdac710aae87dae', 'condition': 'dawn_standalone', }, + 'third_party/google_benchmark/src': { + 'url': '{chromium_git}/external/github.com/google/benchmark.git' + '@' + 'efc89f0b524780b1994d5dddd83a92718e5be492', + 'condition': 'dawn_standalone', + }, # Jinja2 and MarkupSafe for the code generator 'third_party/jinja2': { @@ -204,10 +208,6 @@ deps = { }, # Misc dependencies inherited from Tint - 'third_party/benchmark': { - 'url': '{chromium_git}/external/github.com/google/benchmark.git@e991355c02b93fe17713efe04cbc2e278e00fdbd', - 'condition': 'dawn_standalone', - }, 'third_party/protobuf': { 'url': '{chromium_git}/external/github.com/protocolbuffers/protobuf.git@fde7cf7358ec7cd69e8db9be4f1fa6a5c431386a', 'condition': 'dawn_standalone', diff --git a/src/dawn/CMakeLists.txt b/src/dawn/CMakeLists.txt index 8fb3302971..6a32109102 100644 --- a/src/dawn/CMakeLists.txt +++ b/src/dawn/CMakeLists.txt @@ -23,6 +23,7 @@ add_subdirectory(wire) # TODO(dawn:269): Remove once the implementation-based swapchains are removed. add_subdirectory(utils) add_subdirectory(glfw) +add_subdirectory(tests/benchmarks) if (DAWN_BUILD_SAMPLES) #TODO(dawn:269): Add this once implementation-based swapchains are removed. diff --git a/src/dawn/tests/BUILD.gn b/src/dawn/tests/BUILD.gn index f0a29321c8..284e26ea03 100644 --- a/src/dawn/tests/BUILD.gn +++ b/src/dawn/tests/BUILD.gn @@ -24,6 +24,7 @@ group("tests") { ":dawn_end2end_tests", ":dawn_perf_tests", ":dawn_unittests", + "${dawn_root}/src/dawn/tests/benchmarks:dawn_benchmarks", ] } diff --git a/src/dawn/tests/benchmarks/BGLCreation.cpp b/src/dawn/tests/benchmarks/BGLCreation.cpp new file mode 100644 index 0000000000..36ff003614 --- /dev/null +++ b/src/dawn/tests/benchmarks/BGLCreation.cpp @@ -0,0 +1,115 @@ +// Copyright 2023 The Dawn Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +#include "dawn/common/Log.h" +#include "dawn/tests/benchmarks/NullDeviceSetup.h" + +static void RedundantBGLCreation(benchmark::State& state) { + static wgpu::Device device = nullptr; + + if (state.thread_index() == 0) { + std::vector requiredFeatures; + if (state.threads() > 1) { + requiredFeatures.push_back(wgpu::FeatureName::ImplicitDeviceSynchronization); + } + + wgpu::DeviceDescriptor deviceDesc = {}; + deviceDesc.requiredFeatures = requiredFeatures.data(); + deviceDesc.requiredFeaturesCount = requiredFeatures.size(); + device = CreateNullDevice(deviceDesc); + } + + std::vector entries(state.range(0)); + for (uint32_t i = 0; i < entries.size(); ++i) { + entries[i].binding = i; + entries[i].visibility = wgpu::ShaderStage::Vertex | wgpu::ShaderStage::Fragment; + entries[i].buffer.type = wgpu::BufferBindingType::Uniform; + } + + wgpu::BindGroupLayoutDescriptor bglDesc = {}; + bglDesc.entryCount = entries.size(); + bglDesc.entries = entries.data(); + + thread_local std::vector bgls; + bgls.reserve(100000); + for (auto _ : state) { + bgls.push_back(device.CreateBindGroupLayout(&bglDesc)); + } + bgls.clear(); + + if (state.thread_index() == 0) { + device = nullptr; + } +} + +static void UniqueBGLCreation(benchmark::State& state) { + static wgpu::Device device = nullptr; + + if (state.thread_index() == 0) { + std::vector requiredFeatures; + if (state.threads() > 1) { + requiredFeatures.push_back(wgpu::FeatureName::ImplicitDeviceSynchronization); + } + + wgpu::DeviceDescriptor deviceDesc = {}; + deviceDesc.requiredFeatures = requiredFeatures.data(); + deviceDesc.requiredFeaturesCount = requiredFeatures.size(); + device = CreateNullDevice(deviceDesc); + } + + std::vector entries(state.range(0)); + for (uint32_t i = 0; i < entries.size(); ++i) { + entries[i].binding = i; + entries[i].visibility = wgpu::ShaderStage::Vertex | wgpu::ShaderStage::Fragment; + entries[i].buffer.type = wgpu::BufferBindingType::Uniform; + } + entries[0].buffer.minBindingSize = 4u; + + wgpu::BindGroupLayoutDescriptor bglDesc = {}; + bglDesc.entryCount = entries.size(); + bglDesc.entries = entries.data(); + + thread_local std::vector bgls; + bgls.reserve(100000); + for (auto _ : state) { + entries[0].buffer.minBindingSize += 4; + bgls.push_back(device.CreateBindGroupLayout(&bglDesc)); + } + bgls.clear(); + + if (state.thread_index() == 0) { + device = nullptr; + } +} + +BENCHMARK(RedundantBGLCreation) + ->Setup(SetupNullBackend) + ->Arg(1) + ->Arg(12) + ->Threads(1) + ->Threads(4) + ->Threads(16); + +BENCHMARK(UniqueBGLCreation) + ->Setup(SetupNullBackend) + ->Arg(1) + ->Arg(12) + ->Threads(1) + ->Threads(4) + ->Threads(16); diff --git a/src/dawn/tests/benchmarks/BUILD.gn b/src/dawn/tests/benchmarks/BUILD.gn new file mode 100644 index 0000000000..d8ea27bc4d --- /dev/null +++ b/src/dawn/tests/benchmarks/BUILD.gn @@ -0,0 +1,35 @@ +# Copyright 2023 The Dawn Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import("../../../../scripts/dawn_overrides_with_defaults.gni") + +import("//testing/test.gni") + +test("dawn_benchmarks") { + deps = [ + "${dawn_root}/src/dawn:cpp", + "${dawn_root}/src/dawn:proc", + "${dawn_root}/src/dawn/common", + "${dawn_root}/src/dawn/native:sources", + "${dawn_root}/src/dawn/native:static", + "//third_party/google_benchmark", + "//third_party/google_benchmark:benchmark_main", + ] + sources = [ + "BGLCreation.cpp", + "NullDeviceSetup.cpp", + "NullDeviceSetup.h", + ] + configs += [ "${dawn_root}/include/dawn:public" ] +} diff --git a/src/dawn/tests/benchmarks/CMakeLists.txt b/src/dawn/tests/benchmarks/CMakeLists.txt new file mode 100644 index 0000000000..f575866737 --- /dev/null +++ b/src/dawn/tests/benchmarks/CMakeLists.txt @@ -0,0 +1,34 @@ +# Copyright 2023 The Dawn Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if (${DAWN_BUILD_BENCHMARKS}) + add_executable(dawn_benchmarks + "BGLCreation.cpp" + "NullDeviceSetup.cpp" + "NullDeviceSetup.h" + ) + set_target_properties(dawn_benchmarks PROPERTIES FOLDER "Benchmarks") + + target_include_directories(dawn_benchmarks PUBLIC "${PROJECT_SOURCE_DIR}/include") + target_include_directories(dawn_benchmarks PUBLIC "${PROJECT_SOURCE_DIR}/src") + + target_link_libraries(dawn_benchmarks PRIVATE + benchmark::benchmark + benchmark::benchmark_main + dawn_common + dawn_native + dawncpp_headers + dawncpp + dawn_proc) +endif() diff --git a/src/dawn/tests/benchmarks/NullDeviceSetup.cpp b/src/dawn/tests/benchmarks/NullDeviceSetup.cpp new file mode 100644 index 0000000000..a4b76bba09 --- /dev/null +++ b/src/dawn/tests/benchmarks/NullDeviceSetup.cpp @@ -0,0 +1,84 @@ +// Copyright 2023 The Dawn Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "dawn/tests/benchmarks/NullDeviceSetup.h" + +#include +#include +#include + +#include "dawn/common/Assert.h" +#include "dawn/common/Log.h" +#include "dawn/dawn_proc.h" +#include "dawn/native/DawnNative.h" + +namespace { +std::unique_ptr nativeInstance = nullptr; +WGPUAdapter nullBackendAdapter = nullptr; +} // namespace + +void SetupNullBackend(const benchmark::State& state) { + if (!nativeInstance) { + nativeInstance = std::make_unique(); + nativeInstance->DiscoverDefaultAdapters(); + } + + if (!nullBackendAdapter) { + for (auto& a : nativeInstance->GetAdapters()) { + wgpu::AdapterProperties properties; + a.GetProperties(&properties); + if (properties.backendType == wgpu::BackendType::Null) { + nullBackendAdapter = a.Get(); + } + } + } + ASSERT(nullBackendAdapter != nullptr); +} + +wgpu::Device CreateNullDevice(const wgpu::DeviceDescriptor& desc) { + dawnProcSetProcs(&dawn::native::GetProcs()); + + wgpu::Device device; + + wgpu::Adapter(nullBackendAdapter) + .RequestDevice( + &desc, + [](WGPURequestDeviceStatus status, WGPUDevice cDevice, char const* message, + void* userdata) { + ASSERT(status == WGPURequestDeviceStatus_Success); + *reinterpret_cast(userdata) = wgpu::Device::Acquire(cDevice); + }, + &device); + while (!device) { + wgpuInstanceProcessEvents(nativeInstance->Get()); + } + + device.SetUncapturedErrorCallback( + [](WGPUErrorType, char const* message, void* userdata) { + dawn::ErrorLog() << message; + UNREACHABLE(); + }, + nullptr); + + device.SetDeviceLostCallback( + [](WGPUDeviceLostReason reason, char const* message, void* userdata) { + if (reason == WGPUDeviceLostReason_Undefined) { + dawn::ErrorLog() << message; + UNREACHABLE(); + } + }, + nullptr); + + return device; +} diff --git a/src/dawn/tests/benchmarks/NullDeviceSetup.h b/src/dawn/tests/benchmarks/NullDeviceSetup.h new file mode 100644 index 0000000000..d8c7cad4ed --- /dev/null +++ b/src/dawn/tests/benchmarks/NullDeviceSetup.h @@ -0,0 +1,31 @@ +// Copyright 2023 The Dawn Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DAWN_TESTS_BENCHMARKS_NULLDEVICESETUP +#define DAWN_TESTS_BENCHMARKS_NULLDEVICESETUP + +#include + +namespace benchmark { +class State; +} + +namespace wgpu { +struct DeviceDescriptor; +} + +void SetupNullBackend(const benchmark::State& state); +wgpu::Device CreateNullDevice(const wgpu::DeviceDescriptor& desc); + +#endif // DAWN_TESTS_BENCHMARKS_NULLDEVICESETUP diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt index 6ca33d464e..47ad676ad1 100644 --- a/third_party/CMakeLists.txt +++ b/third_party/CMakeLists.txt @@ -106,9 +106,9 @@ if (NOT TARGET vk_swiftshader AND ${DAWN_ENABLE_SWIFTSHADER}) add_subdirectory(${DAWN_SWIFTSHADER_DIR} "${CMAKE_CURRENT_BINARY_DIR}/swiftshader") endif() -if (${TINT_BUILD_BENCHMARKS}) +if (${TINT_BUILD_BENCHMARKS} OR ${DAWN_BUILD_BENCHMARKS}) set(BENCHMARK_ENABLE_TESTING FALSE CACHE BOOL FALSE FORCE) - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/benchmark EXCLUDE_FROM_ALL) + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/google_benchmark/src EXCLUDE_FROM_ALL) endif() if (NOT TARGET gmock AND ${TINT_BUILD_TESTS}) diff --git a/third_party/google_benchmark/BUILD.gn b/third_party/google_benchmark/BUILD.gn new file mode 100644 index 0000000000..779cf2a356 --- /dev/null +++ b/third_party/google_benchmark/BUILD.gn @@ -0,0 +1,91 @@ +# Copyright 2019 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("//build_overrides/build.gni") + +config("benchmark_config") { + include_dirs = [ "src/include" ] + + if (!is_component_build) { + defines = [ "BENCHMARK_STATIC_DEFINE" ] + } +} + +component("google_benchmark") { + testonly = true + + public = [ + "src/include/benchmark/benchmark.h", + "src/include/benchmark/export.h", + ] + + sources = [ + "src/src/arraysize.h", + "src/src/benchmark.cc", + "src/src/benchmark_api_internal.cc", + "src/src/benchmark_api_internal.h", + "src/src/benchmark_name.cc", + "src/src/benchmark_register.cc", + "src/src/benchmark_register.h", + "src/src/benchmark_runner.cc", + "src/src/benchmark_runner.h", + "src/src/check.cc", + "src/src/check.h", + "src/src/colorprint.cc", + "src/src/colorprint.h", + "src/src/commandlineflags.cc", + "src/src/commandlineflags.h", + "src/src/complexity.cc", + "src/src/complexity.h", + "src/src/console_reporter.cc", + "src/src/counter.cc", + "src/src/counter.h", + "src/src/csv_reporter.cc", + "src/src/cycleclock.h", + "src/src/internal_macros.h", + "src/src/json_reporter.cc", + "src/src/log.h", + "src/src/mutex.h", + "src/src/perf_counters.cc", + "src/src/perf_counters.h", + "src/src/re.h", + "src/src/reporter.cc", + "src/src/statistics.cc", + "src/src/statistics.h", + "src/src/string_util.cc", + "src/src/string_util.h", + "src/src/sysinfo.cc", + "src/src/thread_manager.h", + "src/src/thread_timer.h", + "src/src/timers.cc", + "src/src/timers.h", + ] + + all_dependent_configs = [ ":benchmark_config" ] + + if (build_with_chromium) { + configs -= [ "//build/config/compiler:chromium_code" ] + configs += [ "//build/config/compiler:no_chromium_code" ] + } + + if (is_win) { + configs -= [ "//build/config/win:nominmax" ] + } + + defines = [ + "benchmark_EXPORTS=1", + + # Tell gtest to always use standard regular expressions. + "HAVE_GNU_POSIX_REGEX=0", + "HAVE_POSIX_REGEX=0", + "HAVE_STD_REGEX=1", + ] +} + +component("benchmark_main") { + testonly = true + sources = [ "src/src/benchmark_main.cc" ] + defines = [ "benchmark_EXPORTS=1" ] + deps = [ ":google_benchmark" ] +} diff --git a/third_party/google_benchmark/README.chromium b/third_party/google_benchmark/README.chromium new file mode 100644 index 0000000000..34ce0ebed7 --- /dev/null +++ b/third_party/google_benchmark/README.chromium @@ -0,0 +1,13 @@ +Name: Google Benchmark +Short Name: benchmark +URL: https://github.com/google/benchmark +Version: efc89f0b524780b1994d5dddd83a92718e5be492 +License: Apache 2.0 +License File: NOT_SHIPPED +Security Critical: no + +Description: +A microbenchmark support library. + +Local Additions: +* gn file for building in chromium