Add runtime benchmarks for Fixed*Map data structures

This adds a new set of tests tagged "benchmark" in bazel, which are filtered out of the default `test_suite` target. This means that they will only be built/run manually.
teslamotors · Jul 12, 2024 · dcff589 · dcff589
1 parent 3a45dbd
commit dcff589
Show file tree

Hide file tree

Showing 8 changed files with 557 additions and 114 deletions.
diff --git a/BUILD.bazel b/BUILD.bazel
@@ -1157,20 +1157,6 @@ cc_test(
     copts = ["-std=c++20"],
 )
 
-cc_test(
-    name = "fixed_map_perf_test",
-    srcs = ["test/fixed_map_perf_test.cpp"],
-    deps = [
-        ":consteval_compare",
-        ":fixed_index_based_storage",
-        ":fixed_map",
-        ":fixed_red_black_tree",
-        "@com_google_googletest//:gtest_main",
-        "@com_google_benchmark//:benchmark_main",
-    ],
-    copts = ["-std=c++20"],
-)
-
 cc_test(
     name = "fixed_list_test",
     srcs = ["test/fixed_list_test.cpp"],
@@ -1621,6 +1607,57 @@ cc_test(
     copts = ["-std=c++20"],
 )
 
+cc_library(
+    name = "benchmark_map_utils",
+    hdrs = ["test/benchmarks/map_utils.hpp"],
+)
+
+cc_test(
+    name = "map_copy_bench",
+    srcs = ["test/benchmarks/map_copy.cpp"],
+    deps = [
+        ":consteval_compare",
+        ":fixed_map",
+        ":fixed_unordered_map",
+        ":mock_testing_types",
+        ":benchmark_map_utils",
+        "@com_google_benchmark//:benchmark_main",
+    ],
+    copts = ["-std=c++20"],
+    tags = ["benchmark"],
+)
+
+cc_test(
+    name = "map_lookup_bench",
+    srcs = ["test/benchmarks/map_lookup.cpp"],
+    deps = [
+        ":consteval_compare",
+        ":fixed_map",
+        ":fixed_unordered_map",
+        ":mock_testing_types",
+        ":benchmark_map_utils",
+        "@com_google_benchmark//:benchmark_main",
+    ],
+    copts = ["-std=c++20"],
+    tags = ["benchmark"],
+)
+
+cc_test(
+    name = "map_clear_bench",
+    srcs = ["test/benchmarks/map_clear.cpp"],
+    deps = [
+        ":consteval_compare",
+        ":fixed_map",
+        ":fixed_unordered_map",
+        ":mock_testing_types",
+        ":benchmark_map_utils",
+        "@com_google_benchmark//:benchmark_main",
+    ],
+    copts = ["-std=c++20"],
+    tags = ["benchmark"],
+)
+
 test_suite(
     name = "all_tests",
+    tags = ["-benchmark"],
 )
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -171,8 +171,6 @@ if(BUILD_TESTS)
     add_test_dependencies(fixed_list_test)
     add_executable(fixed_map_test test/fixed_map_test.cpp)
     add_test_dependencies(fixed_map_test)
-    add_executable(fixed_map_perf_test test/fixed_map_perf_test.cpp)
-    add_test_dependencies(fixed_map_perf_test)
     add_executable(fixed_red_black_tree_test test/fixed_red_black_tree_test.cpp)
     add_test_dependencies(fixed_red_black_tree_test)
     add_executable(fixed_red_black_tree_view_test test/fixed_red_black_tree_view_test.cpp)
@@ -231,6 +229,13 @@ if(BUILD_TESTS)
     add_test_dependencies(tuples_test)
     add_executable(type_name_test test/type_name_test.cpp)
     add_test_dependencies(type_name_test)
+
+    add_executable(map_copy_bench test/benchmarks/map_copy.cpp)
+    add_test_dependencies(map_copy_bench)
+    add_executable(map_lookup_bench test/benchmarks/map_lookup.cpp)
+    add_test_dependencies(map_lookup_bench)
+    add_executable(map_clear_bench test/benchmarks/map_clear.cpp)
+    add_test_dependencies(map_clear_bench)
 endif()
 
 option(FIXED_CONTAINERS_OPT_INSTALL "Enable install target" ${PROJECT_IS_TOP_LEVEL})

diff --git a/test/benchmarks/map_clear.cpp b/test/benchmarks/map_clear.cpp
@@ -0,0 +1,113 @@
+#include "fixed_containers/fixed_map.hpp"
+#include "fixed_containers/fixed_unordered_map.hpp"
+
+#include <benchmark/benchmark.h>
+
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <map>
+#include <unordered_map>
+
+namespace fixed_containers
+{
+
+namespace
+{
+template <typename MapType>
+void benchmark_map_copy(benchmark::State& state)
+{
+    const int64_t nelem = state.range(0);
+    MapType instance = {};
+
+    using KeyType = typename MapType::key_type;
+    for (int64_t i = 0; i < nelem; i++)
+    {
+        instance.try_emplace(static_cast<KeyType>(i));
+    }
+
+    for (auto _ : state)
+    {
+        MapType instance2{instance};
+        benchmark::DoNotOptimize(instance2);
+    }
+}
+
+template <typename MapType>
+void benchmark_map_copy_then_clear(benchmark::State& state)
+{
+    using KeyType = typename MapType::key_type;
+    MapType instance{};
+    const int64_t nelem = state.range(0);
+    for (int64_t i = 0; i < nelem; i++)
+    {
+        instance.try_emplace(static_cast<KeyType>(i));
+    }
+
+    for (auto _ : state)
+    {
+        MapType instance2{instance};
+        instance2.clear();
+        benchmark::DoNotOptimize(instance2);
+    }
+}
+
+template <typename MapType>
+void benchmark_map_copy_then_reconstruct(benchmark::State& state)
+{
+    using KeyType = typename MapType::key_type;
+    MapType instance{};
+    const int64_t nelem = state.range(0);
+    for (int64_t i = 0; i < nelem; i++)
+    {
+        instance.try_emplace(static_cast<KeyType>(i));
+    }
+
+    for (auto _ : state)
+    {
+        MapType instance2{instance};
+        instance2 = {};
+        benchmark::DoNotOptimize(instance2);
+    }
+}
+
+template <typename ArrType>
+void benchmark_array_clear(benchmark::State& state)
+{
+    ArrType instance{};
+
+    for (auto _ : state)
+    {
+        instance.fill(0);
+        benchmark::DoNotOptimize(instance);
+    }
+}
+}  // namespace
+
+constexpr std::size_t MAX_SIZE = 8 << 13;
+
+BENCHMARK(benchmark_map_copy<std::map<int, int>>)->Range(16, MAX_SIZE);
+BENCHMARK(benchmark_map_copy_then_clear<std::map<int, int>>)->Range(16, MAX_SIZE);
+BENCHMARK(benchmark_map_copy_then_reconstruct<std::map<int, int>>)->Range(16, MAX_SIZE);
+
+BENCHMARK(benchmark_map_copy<std::unordered_map<int, int>>)->Range(16, MAX_SIZE);
+BENCHMARK(benchmark_map_copy_then_clear<std::unordered_map<int, int>>)->Range(16, MAX_SIZE);
+BENCHMARK(benchmark_map_copy_then_reconstruct<std::unordered_map<int, int>>)->Range(16, MAX_SIZE);
+
+BENCHMARK(benchmark_map_copy<FixedMap<int, int, MAX_SIZE>>)->Range(16, MAX_SIZE);
+BENCHMARK(benchmark_map_copy_then_clear<FixedMap<int, int, MAX_SIZE>>)->Range(16, MAX_SIZE);
+BENCHMARK(benchmark_map_copy_then_reconstruct<FixedMap<int, int, MAX_SIZE>>)->Range(16, MAX_SIZE);
+
+BENCHMARK(benchmark_map_copy<FixedUnorderedMap<int, int, MAX_SIZE>>)->Range(16, MAX_SIZE);
+BENCHMARK(benchmark_map_copy_then_clear<FixedUnorderedMap<int, int, MAX_SIZE>>)
+    ->Range(16, MAX_SIZE);
+BENCHMARK(benchmark_map_copy_then_reconstruct<FixedUnorderedMap<int, int, MAX_SIZE>>)
+    ->Range(16, MAX_SIZE);
+
+// more-or-less the theoretical best performance we could possibly get for a full FixedUnorderedMap
+// (just 0 out every bucket)
+BENCHMARK(benchmark_array_clear<std::array<long, (MAX_SIZE * 130ULL) / 100>>);
+
+}  // namespace fixed_containers
+
+BENCHMARK_MAIN();
diff --git a/test/benchmarks/map_copy.cpp b/test/benchmarks/map_copy.cpp
@@ -0,0 +1,116 @@
+#include "map_utils.hpp"
+
+#include "../mock_testing_types.hpp"
+#include "fixed_containers/fixed_unordered_map.hpp"
+
+#include <benchmark/benchmark.h>
+
+#include <cstdint>
+
+namespace fixed_containers
+{
+
+namespace
+{
+template <typename MapType>
+void benchmark_map_copy_fresh(benchmark::State& state)
+{
+    const int64_t nelem = state.range(0);
+    MapType instance = {};
+
+    using KeyType = typename MapType::key_type;
+    for (int64_t i = 0; i < nelem; i++)
+    {
+        instance.try_emplace(static_cast<KeyType>(i));
+    }
+
+    for (auto _ : state)
+    {
+        MapType instance2{instance};
+        benchmark::DoNotOptimize(instance2);
+    }
+}
+
+template <typename MapType>
+void benchmark_map_iterate_copy_fresh(benchmark::State& state)
+{
+    const int64_t nelem = state.range(0);
+    MapType instance = {};
+
+    using KeyType = typename MapType::key_type;
+    for (int64_t i = 0; i < nelem; i++)
+    {
+        instance.try_emplace(static_cast<KeyType>(i));
+    }
+
+    for (auto _ : state)
+    {
+        MapType instance2{};
+        for (auto elem : instance)
+        {
+            instance2.try_emplace(elem.first, elem.second);
+        }
+        benchmark::DoNotOptimize(instance2);
+    }
+}
+
+template <typename MapType>
+void benchmark_map_copy_shuffled(benchmark::State& state)
+{
+    const int64_t nelem = state.range(0);
+    auto instance = map_benchmarks::make_shuffled_map<MapType>();
+
+    using KeyType = typename MapType::key_type;
+    for (int64_t i = 0; i < nelem; i++)
+    {
+        instance.try_emplace(static_cast<KeyType>(i));
+    }
+
+    for (auto _ : state)
+    {
+        MapType instance2{instance};
+        benchmark::DoNotOptimize(instance2);
+    }
+}
+
+template <typename MapType>
+void benchmark_map_iterate_copy_shuffled(benchmark::State& state)
+{
+    const int64_t nelem = state.range(0);
+    auto instance = map_benchmarks::make_shuffled_map<MapType>();
+
+    using KeyType = typename MapType::key_type;
+    for (int64_t i = 0; i < nelem; i++)
+    {
+        instance.try_emplace(static_cast<KeyType>(i));
+    }
+
+    for (auto _ : state)
+    {
+        MapType instance2{};
+        for (auto elem : instance)
+        {
+            instance2.try_emplace(elem.first, elem.second);
+        }
+        benchmark::DoNotOptimize(instance2);
+    }
+}
+}  // namespace
+
+BENCHMARK(
+    benchmark_map_copy_fresh<FixedUnorderedMap<int, MockNonTrivialCopyConstructible, 8 << 14>>)
+    ->DenseRange(1024, 8 << 14, 1024);
+BENCHMARK(benchmark_map_iterate_copy_fresh<
+              FixedUnorderedMap<int, MockNonTrivialCopyConstructible, 8 << 14>>)
+    ->DenseRange(1024, 8 << 14, 1024);
+
+BENCHMARK(
+    benchmark_map_copy_shuffled<FixedUnorderedMap<int, MockNonTrivialCopyConstructible, 8 << 14>>)
+    ->DenseRange(1024, 8 << 14, 1024);
+BENCHMARK(benchmark_map_iterate_copy_shuffled<
+              FixedUnorderedMap<int, MockNonTrivialCopyConstructible, 8 << 14>>)
+    ->DenseRange(1024, 8 << 14, 1024);
+
+}  // namespace fixed_containers
+
+BENCHMARK_MAIN();