benchmarks: Add a basic set of benchmarks

Add google benchmark to the DEPs.

Implement a basic set of benchmarks for each of the writers and the WGSL parser.

Add build rules for CMake. GN build rules TODO.

Add a simple go tool (ported from Marl) to diff two benchmarks. Less
noisy than the one provided by google benchmark.

Bug: tint:1378
Change-Id: I73cf92c5d9fd2d3bfac8f264864fd774afbd5d01
Reviewed-on: https://dawn-review.googlesource.com/c/tint/+/76840
Kokoro: Kokoro <noreply+kokoro@google.com>
Reviewed-by: Ryan Harrison <rharrison@chromium.org>
Commit-Queue: Ben Clayton <bclayton@chromium.org>

src/CMakeLists.txt

@@ -598,7 +598,10 @@ if(${TINT_BUILD_SPV_READER} OR ${TINT_BUILD_SPV_WRITER})
   endif()
 endif()
 
-if(${TINT_BUILD_TESTS})
+################################################################################
+# Tests
+################################################################################
+if(TINT_BUILD_TESTS)
   set(TINT_TEST_SRCS
     ast/alias_test.cc
     ast/array_test.cc
@@ -1114,6 +1117,7 @@ if(${TINT_BUILD_TESTS})
   endif()
 
   add_executable(tint_unittests ${TINT_TEST_SRCS})
+  set_target_properties(${target} PROPERTIES FOLDER "Tests")
 
   if(NOT MSVC)
     target_compile_options(tint_unittests PRIVATE
@@ -1133,4 +1137,41 @@ if(${TINT_BUILD_TESTS})
   endif()
 
   add_test(NAME tint_unittests COMMAND tint_unittests)
-endif()
+endif(TINT_BUILD_TESTS)
+
+################################################################################
+# Benchmarks
+################################################################################
+if(TINT_BUILD_BENCHMARKS)
+  if(NOT TINT_BUILD_WGSL_READER)
+    message(FATAL_ERROR "TINT_BUILD_BENCHMARKS requires TINT_BUILD_WGSL_READER")
+  endif()
+
+  set(TINT_BENCHMARK_SRC
+    "benchmark/benchmark.cc"
+    "reader/wgsl/parser_bench.cc"
+  )
+
+  if (${TINT_BUILD_GLSL_WRITER})
+    list(APPEND TINT_BENCHMARK_SRC writer/glsl/generator_bench.cc)
+  endif()
+  if (${TINT_BUILD_HLSL_WRITER})
+    list(APPEND TINT_BENCHMARK_SRC writer/hlsl/generator_bench.cc)
+  endif()
+  if (${TINT_BUILD_MSL_WRITER})
+    list(APPEND TINT_BENCHMARK_SRC writer/msl/generator_bench.cc)
+  endif()
+  if (${TINT_BUILD_SPV_WRITER})
+    list(APPEND TINT_BENCHMARK_SRC writer/spirv/generator_bench.cc)
+  endif()
+  if (${TINT_BUILD_WGSL_WRITER})
+    list(APPEND TINT_BENCHMARK_SRC writer/wgsl/generator_bench.cc)
+  endif()
+
+  add_executable(tint-benchmark ${TINT_BENCHMARK_SRC})
+  set_target_properties(${target} PROPERTIES FOLDER "Benchmarks")
+
+  tint_core_compile_options(tint-benchmark)
+
+  target_link_libraries(tint-benchmark PRIVATE benchmark::benchmark libtint)
+endif(TINT_BUILD_BENCHMARKS)

src/benchmark/benchmark.cc

@@ -0,0 +1,123 @@
+// Copyright 2022 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "src/benchmark/benchmark.h"
+
+#include <filesystem>
+#include <sstream>
+#include <utility>
+#include <vector>
+
+namespace tint::benchmark {
+namespace {
+
+std::filesystem::path kInputFileDir;
+
+/// Copies the content from the file named `input_file` to `buffer`,
+/// assuming each element in the file is of type `T`.  If any error occurs,
+/// writes error messages to the standard error stream and returns false.
+/// Assumes the size of a `T` object is divisible by its required alignment.
+/// @returns true if we successfully read the file.
+template <typename T>
+std::variant<std::vector<T>, Error> ReadFile(const std::string& input_file) {
+  FILE* file = nullptr;
+#if defined(_MSC_VER)
+  fopen_s(&file, input_file.c_str(), "rb");
+#else
+  file = fopen(input_file.c_str(), "rb");
+#endif
+  if (!file) {
+    return Error{"Failed to open " + input_file};
+  }
+
+  fseek(file, 0, SEEK_END);
+  const auto file_size = static_cast<size_t>(ftell(file));
+  if (0 != (file_size % sizeof(T))) {
+    std::stringstream err;
+    err << "File " << input_file
+        << " does not contain an integral number of objects: " << file_size
+        << " bytes in the file, require " << sizeof(T) << " bytes per object";
+    fclose(file);
+    return Error{err.str()};
+  }
+  fseek(file, 0, SEEK_SET);
+
+  std::vector<T> buffer;
+  buffer.resize(file_size / sizeof(T));
+
+  size_t bytes_read = fread(buffer.data(), 1, file_size, file);
+  fclose(file);
+  if (bytes_read != file_size) {
+    return Error{"Failed to read " + input_file};
+  }
+
+  return buffer;
+}
+
+bool FindBenchmarkInputDir() {
+  // Attempt to find the benchmark input files by searching up from the current
+  // working directory.
+  auto path = std::filesystem::current_path();
+  while (std::filesystem::is_directory(path)) {
+    auto test = path / "test" / "benchmark";
+    if (std::filesystem::is_directory(test)) {
+      kInputFileDir = test;
+      return true;
+    }
+    auto parent = path.parent_path();
+    if (path == parent) {
+      break;
+    }
+    path = parent;
+  }
+  return false;
+}
+
+}  // namespace
+
+std::variant<tint::Source::File, Error> LoadInputFile(std::string name) {
+  auto path = (kInputFileDir / name).string();
+  auto data = ReadFile<uint8_t>(path);
+  if (auto* buf = std::get_if<std::vector<uint8_t>>(&data)) {
+    return tint::Source::File(path, std::string(buf->begin(), buf->end()));
+  }
+  return std::get<Error>(data);
+}
+
+std::variant<ProgramAndFile, Error> LoadProgram(std::string name) {
+  auto res = benchmark::LoadInputFile(name);
+  if (auto err = std::get_if<benchmark::Error>(&res)) {
+    return *err;
+  }
+  auto& file = std::get<Source::File>(res);
+  auto program = reader::wgsl::Parse(&file);
+  if (program.Diagnostics().contains_errors()) {
+    return Error{program.Diagnostics().str()};
+  }
+  return ProgramAndFile{std::move(program), std::move(file)};
+}
+
+}  // namespace tint::benchmark
+
+int main(int argc, char** argv) {
+  ::benchmark::Initialize(&argc, argv);
+  if (::benchmark::ReportUnrecognizedArguments(argc, argv)) {
+    return 1;
+  }
+  if (!tint::benchmark::FindBenchmarkInputDir()) {
+    std::cerr << "failed to locate benchmark input files" << std::endl;
+    return 1;
+  }
+  ::benchmark::RunSpecifiedBenchmarks();
+}

src/benchmark/benchmark.h

@@ -0,0 +1,69 @@
+// Copyright 2022 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef SRC_BENCHMARK_BENCHMARK_H_
+#define SRC_BENCHMARK_BENCHMARK_H_
+
+#include <memory>
+#include <string>
+#include <variant>  // NOLINT: Found C system header after C++ system header.
+
+#include "benchmark/benchmark.h"
+#include "src/utils/concat.h"
+#include "tint/tint.h"
+
+namespace tint::benchmark {
+
+/// Error indicates an operation did not complete successfully.
+struct Error {
+  /// The error message.
+  std::string msg;
+};
+
+/// ProgramAndFile holds a Program and a Source::File.
+struct ProgramAndFile {
+  /// The tint program parsed from file.
+  Program program;
+  /// The source file
+  Source::File file;
+};
+
+/// LoadInputFile attempts to load a benchmark input file with the given file
+/// name.
+/// @param name the file name
+/// @returns either the loaded Source::File or an Error
+std::variant<Source::File, Error> LoadInputFile(std::string name);
+
+/// LoadInputFile attempts to load a benchmark input program with the given file
+/// name.
+/// @param name the file name
+/// @returns either the loaded Program or an Error
+std::variant<ProgramAndFile, Error> LoadProgram(std::string name);
+
+/// Declares a benchmark with the given function and WGSL file name
+#define TINT_BENCHMARK_WGSL_PROGRAM(FUNC, WGSL_NAME) \
+  BENCHMARK_CAPTURE(FUNC, WGSL_NAME, WGSL_NAME);
+
+/// Declares a set of benchmarks for the given function using a list of WGSL
+/// files in `<tint>/test/benchmark`.
+#define TINT_BENCHMARK_WGSL_PROGRAMS(FUNC)                   \
+  TINT_BENCHMARK_WGSL_PROGRAM(FUNC, "empty.wgsl");           \
+  TINT_BENCHMARK_WGSL_PROGRAM(FUNC, "particles.wgsl");       \
+  TINT_BENCHMARK_WGSL_PROGRAM(FUNC, "simple_fragment.wgsl"); \
+  TINT_BENCHMARK_WGSL_PROGRAM(FUNC, "simple_vertex.wgsl");   \
+  TINT_BENCHMARK_WGSL_PROGRAM(FUNC, "simple_compute.wgsl");
+
+}  // namespace tint::benchmark
+
+#endif  // SRC_BENCHMARK_BENCHMARK_H_

src/reader/wgsl/parser_bench.cc

@@ -0,0 +1,40 @@
+// Copyright 2022 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <string>
+
+#include "src/benchmark/benchmark.h"
+
+namespace tint::reader::wgsl {
+namespace {
+
+void ParseWGSL(::benchmark::State& state, std::string input_name) {
+  auto res = benchmark::LoadInputFile(input_name);
+  if (auto err = std::get_if<benchmark::Error>(&res)) {
+    state.SkipWithError(err->msg.c_str());
+    return;
+  }
+  auto& file = std::get<Source::File>(res);
+  for (auto _ : state) {
+    auto res = Parse(&file);
+    if (res.Diagnostics().contains_errors()) {
+      state.SkipWithError(res.Diagnostics().str().c_str());
+    }
+  }
+}
+
+TINT_BENCHMARK_WGSL_PROGRAMS(ParseWGSL);
+
+}  // namespace
+}  // namespace tint::reader::wgsl

src/writer/glsl/generator_bench.cc

@@ -0,0 +1,50 @@
+// Copyright 2022 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <string>
+
+#include "src/ast/module.h"
+#include "src/benchmark/benchmark.h"
+
+namespace tint::writer::glsl {
+namespace {
+
+void GenerateGLSL(::benchmark::State& state, std::string input_name) {
+  auto res = benchmark::LoadProgram(input_name);
+  if (auto err = std::get_if<benchmark::Error>(&res)) {
+    state.SkipWithError(err->msg.c_str());
+    return;
+  }
+  auto& program = std::get<benchmark::ProgramAndFile>(res).program;
+  std::vector<std::string> entry_points;
+  for (auto& fn : program.AST().Functions()) {
+    if (fn->IsEntryPoint()) {
+      entry_points.emplace_back(program.Symbols().NameFor(fn->symbol));
+    }
+  }
+
+  for (auto _ : state) {
+    for (auto& ep : entry_points) {
+      auto res = Generate(&program, {}, ep);
+      if (!res.error.empty()) {
+        state.SkipWithError(res.error.c_str());
+      }
+    }
+  }
+}
+
+TINT_BENCHMARK_WGSL_PROGRAMS(GenerateGLSL);
+
+}  // namespace
+}  // namespace tint::writer::glsl

src/writer/hlsl/generator_bench.cc

@@ -0,0 +1,40 @@
+// Copyright 2022 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <string>
+
+#include "src/benchmark/benchmark.h"
+
+namespace tint::writer::hlsl {
+namespace {
+
+void GenerateHLSL(::benchmark::State& state, std::string input_name) {
+  auto res = benchmark::LoadProgram(input_name);
+  if (auto err = std::get_if<benchmark::Error>(&res)) {
+    state.SkipWithError(err->msg.c_str());
+    return;
+  }
+  auto& program = std::get<benchmark::ProgramAndFile>(res).program;
+  for (auto _ : state) {
+    auto res = Generate(&program, {});
+    if (!res.error.empty()) {
+      state.SkipWithError(res.error.c_str());
+    }
+  }
+}
+
+TINT_BENCHMARK_WGSL_PROGRAMS(GenerateHLSL);
+
+}  // namespace
+}  // namespace tint::writer::hlsl

src/writer/msl/generator_bench.cc

@@ -0,0 +1,40 @@
+// Copyright 2022 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <string>
+
+#include "src/benchmark/benchmark.h"
+
+namespace tint::writer::msl {
+namespace {
+
+void GenerateMSL(::benchmark::State& state, std::string input_name) {
+  auto res = benchmark::LoadProgram(input_name);
+  if (auto err = std::get_if<benchmark::Error>(&res)) {
+    state.SkipWithError(err->msg.c_str());
+    return;
+  }
+  auto& program = std::get<benchmark::ProgramAndFile>(res).program;
+  for (auto _ : state) {
+    auto res = Generate(&program, {});
+    if (!res.error.empty()) {
+      state.SkipWithError(res.error.c_str());
+    }
+  }
+}
+
+TINT_BENCHMARK_WGSL_PROGRAMS(GenerateMSL);
+
+}  // namespace
+}  // namespace tint::writer::msl

src/writer/spirv/generator_bench.cc

@@ -0,0 +1,40 @@
+// Copyright 2022 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <string>
+
+#include "src/benchmark/benchmark.h"
+
+namespace tint::writer::spirv {
+namespace {
+
+void GenerateSPIRV(::benchmark::State& state, std::string input_name) {
+  auto res = benchmark::LoadProgram(input_name);
+  if (auto err = std::get_if<benchmark::Error>(&res)) {
+    state.SkipWithError(err->msg.c_str());
+    return;
+  }
+  auto& program = std::get<benchmark::ProgramAndFile>(res).program;
+  for (auto _ : state) {
+    auto res = Generate(&program, {});
+    if (!res.error.empty()) {
+      state.SkipWithError(res.error.c_str());
+    }
+  }
+}
+
+TINT_BENCHMARK_WGSL_PROGRAMS(GenerateSPIRV);
+
+}  // namespace
+}  // namespace tint::writer::spirv

src/writer/wgsl/generator_bench.cc

@@ -0,0 +1,40 @@
+// Copyright 2022 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <string>
+
+#include "src/benchmark/benchmark.h"
+
+namespace tint::writer::wgsl {
+namespace {
+
+void GenerateWGSL(::benchmark::State& state, std::string input_name) {
+  auto res = benchmark::LoadProgram(input_name);
+  if (auto err = std::get_if<benchmark::Error>(&res)) {
+    state.SkipWithError(err->msg.c_str());
+    return;
+  }
+  auto& program = std::get<benchmark::ProgramAndFile>(res).program;
+  for (auto _ : state) {
+    auto res = Generate(&program, {});
+    if (!res.error.empty()) {
+      state.SkipWithError(res.error.c_str());
+    }
+  }
+}
+
+TINT_BENCHMARK_WGSL_PROGRAMS(GenerateWGSL);
+
+}  // namespace
+}  // namespace tint::writer::wgsl