test(benchmark): introduce FFTBatch benchmark

Cargo.Bazel.lock

@@ -1,5 +1,5 @@
 {
- "checksum": "d9a69b7b7c2e8f7447076811098173ebe669c22af14223e2e9dfeb933bf91d26",
+ "checksum": "13a3e635e211fd5829278f0398081863ed1429e454e4c51cbab29f530430fca1",
  "crates": {
  "addchain 0.2.0": {
  "name": "addchain",
@@ -13465,6 +13465,56 @@
  },
  "license": "Apache-2.0 OR MIT"
  },
+ "plonky3_batch_fft_benchmark 0.0.1": {
+ "name": "plonky3_batch_fft_benchmark",
+ "version": "0.0.1",
+ "repository": null,
+ "targets": [
+ {
+ "Library": {
+ "crate_name": "plonky3_batch_fft_benchmark",
+ "crate_root": "src/lib.rs",
+ "srcs": [
+ "**/*.rs"
+ ]
+ }
+ }
+ ],
+ "library_target_name": "plonky3_batch_fft_benchmark",
+ "common_attrs": {
+ "compile_data_glob": [
+ "**"
+ ],
+ "deps": {
+ "common": [
+ {
+ "id": "ff 0.13.0",
+ "target": "ff"
+ },
+ {
+ "id": "p3-baby-bear 0.1.3-succinct",
+ "target": "p3_baby_bear"
+ },
+ {
+ "id": "p3-dft 0.1.3-succinct",
+ "target": "p3_dft"
+ },
+ {
+ "id": "p3-matrix 0.1.3-succinct",
+ "target": "p3_matrix"
+ },
+ {
+ "id": "rand 0.8.5",
+ "target": "rand"
+ }
+ ],
+ "selects": {}
+ },
+ "edition": "2021",
+ "version": "0.0.1"
+ },
+ "license": "MIT OR Apache-2.0"
+ },
  "plonky3_poseidon2_benchmark 0.0.1": {
  "name": "plonky3_poseidon2_benchmark",
  "version": "0.0.1",
@@ -22045,6 +22095,7 @@
  "halo2_fft_benchmark 0.0.1": "benchmark/fft/halo2",
  "halo2_msm_benchmark 0.0.1": "benchmark/msm/halo2",
  "horizen_poseidon2_benchmark 0.0.1": "benchmark/poseidon2/horizen",
+ "plonky3_batch_fft_benchmark 0.0.1": "benchmark/fft_batch/plonky3",
  "plonky3_poseidon2_benchmark 0.0.1": "benchmark/poseidon2/plonky3",
  "tachyon_plonky3 0.0.1": "vendors/plonky3",
  "tachyon_rs 0.0.1": "tachyon/rs",

Cargo.toml

@@ -6,6 +6,7 @@ members = [
  "benchmark/fft/arkworks",
  "benchmark/fft/bellman",
  "benchmark/fft/halo2",
+ "benchmark/fft_batch/plonky3",
  "benchmark/poseidon/arkworks",
  "benchmark/poseidon2/horizen",
  "benchmark/poseidon2/plonky3",

WORKSPACE

@@ -57,6 +57,7 @@ crates_repository(
  "//benchmark/fft/arkworks:Cargo.toml",
  "//benchmark/fft/bellman:Cargo.toml",
  "//benchmark/fft/halo2:Cargo.toml",
+ "//benchmark/fft_batch/plonky3:Cargo.toml",
  "//benchmark/poseidon/arkworks:Cargo.toml",
  "//benchmark/poseidon2/horizen:Cargo.toml",
  "//benchmark/poseidon2/plonky3:Cargo.toml",

benchmark/fft_batch/BUILD.bazel

@@ -0,0 +1,60 @@
+load(
+ "//bazel:tachyon_cc.bzl",
+ "tachyon_cc_binary",
+ "tachyon_cc_library",
+)
+
+tachyon_cc_library(
+ name = "config",
+ testonly = True,
+ srcs = ["config.cc"],
+ hdrs = ["config.h"],
+ deps = [
+ "//tachyon/base/console",
+ "//tachyon/base/flag:flag_parser",
+ ],
+)
+
+tachyon_cc_library(
+ name = "benchmark_reporter",
+ testonly = True,
+ srcs = ["benchmark_reporter.cc"],
+ hdrs = ["benchmark_reporter.h"],
+ deps = [
+ "//benchmark:simple_benchmark_reporter",
+ "//tachyon/base/containers:container_util",
+ "//tachyon/base/strings:string_number_conversions",
+ ],
+)
+
+tachyon_cc_library(
+ name = "benchmark_runner",
+ testonly = True,
+ hdrs = ["benchmark_runner.h"],
+ deps = [
+ ":benchmark_reporter",
+ ":config",
+ "//tachyon/base:logging",
+ "//tachyon/base/containers:container_util",
+ "//tachyon/base/time",
+ "//tachyon/c/base:type_traits_forward",
+ "//tachyon/math/matrix:matrix_types",
+ "//tachyon/math/polynomials/univariate:radix2_evaluation_domain",
+ ],
+)
+
+tachyon_cc_binary(
+ name = "benchmark",
+ testonly = True,
+ srcs = ["benchmark.cc"],
+ deps = [
+ ":benchmark_reporter",
+ ":benchmark_runner",
+ ":config",
+ "//benchmark/fft_batch/plonky3",
+ "//tachyon/c/math/finite_fields/baby_bear",
+ "//tachyon/math/finite_fields/baby_bear",
+ "//tachyon/math/matrix:matrix_types",
+ "//tachyon/math/polynomials/univariate:radix2_evaluation_domain",
+ ],
+)

benchmark/fft_batch/README.md

@@ -0,0 +1,56 @@
+# FFTBatch Benchmark
+
+```
+Run on 13th Gen Intel(R) Core(TM) i9-13900K (32 X 5500 MHz CPU s)
+CPU Caches:
+ L1 Data 48 KiB (x16)
+ L1 Instruction 32 KiB (x16)
+ L2 Unified 2048 KiB (x16)
+ L3 Unified 36864 KiB (x1)
+
+Run on Apple M3 Pro (12 X 4050 MHz)
+CPU Caches:
+ L1 Data 64 KiB (x12)
+ L1 Instruction 128 KiB (x12)
+ L2 Unified 4096 KiB (x12)
+```
+
+```shell
+bazel run -c opt --//:has_openmp --//:has_rtti --//:has_matplotlib //benchmark/fft_batch:benchmark -- -d 21 -p baby_bear --vendor plonky3
+```
+
+#### On Intel i9-13900K
+
+| Repetition | Tachyon | Plonky3 |
+| :--------- | ------- | ----------- |
+| 0 | 1.49794 | **0.211087** |
+| 1 | 1.14541 | **0.209432** |
+| 2 | 1.19469 | **0.212014** |
+| 3 | 1.19882 | **0.209351** |
+| 4 | 1.13655 | **0.210366** |
+| 5 | 0.901086 | **0.209983** |
+| 6 | 1.19522 | **0.211067** |
+| 7 | 1.18459 | **0.209839** |
+| 8 | 0.900512 | **0.209745** |
+| 9 | 1.21875 | **0.208887** |
+| avg | 1.15736 | **0.210177** |
+
+![image](/benchmark/fft_batch/benchmark_ubuntu_i9.png)
+
+#### On Mac M3 Pro
+
+| Repetition | Tachyon | Plonky3 |
+| :--------- | ------- | ------------ |
+| 0 | 2.53273 | **0.146832** |
+| 1 | 2.48872 | **0.158379** |
+| 2 | 2.52125 | **0.147676** |
+| 3 | 2.53502 | **0.162602** |
+| 4 | 2.50802 | **0.144537** |
+| 5 | 2.57135 | **0.145066** |
+| 6 | 2.4639 | **0.144031** |
+| 7 | 2.53492 | **0.150916** |
+| 8 | 2.52831 | **0.149961** |
+| 9 | 2.53022 | **0.167055** |
+| avg | 2.52144 | **0.151705** |
+
+![image](/benchmark/fft_batch/benchmark_mac_m3.png)

benchmark/fft_batch/benchmark.cc

@@ -0,0 +1,75 @@
+#include <iostream>
+
+// clang-format off
+#include "benchmark/fft_batch/benchmark_reporter.h"
+#include "benchmark/fft_batch/benchmark_runner.h"
+#include "benchmark/fft_batch/config.h"
+// clang-format on
+#include "tachyon/base/logging.h"
+#include "tachyon/c/math/finite_fields/baby_bear/baby_bear.h"
+#include "tachyon/c/math/finite_fields/baby_bear/baby_bear_type_traits.h"
+#include "tachyon/math/finite_fields/baby_bear/baby_bear.h"
+
+namespace tachyon::benchmark::fft_batch {
+
+extern "C" void* run_fft_batch_plonky3_baby_bear(uint64_t* duration,
+ math::BabyBear* data,
+ size_t n_log,
+ size_t batch_size);
+
+template <typename F, typename Fn>
+void Run(BenchmarkReporter& reporter, const tachyon::Config& config,
+ Fn plonky3_fn) {
+ F::Init();
+
+ BenchmarkRunner<F> runner(&reporter, &config);
+ runner.Prepare();
+
+ math::RowMajorMatrix<F> result = runner.Run();
+ math::RowMajorMatrix<F> result_vendor;
+ for (const Config::Vendor vendor : config.vendors()) {
+ switch (vendor) {
+ case Config::Vendor::kPlonky3:
+ result_vendor = runner.RunExternal(plonky3_fn);
+ break;
+ }
+ }
+ if (config.check_results()) {
+ NOTIMPLEMENTED() << "Check results not implemented";
+ CHECK_EQ(result, result_vendor) << "Result not matched";
+ }
+}
+
+int RealMain(int argc, char** argv) {
+ Config config;
+ if (!config.Parse(argc, argv)) {
+ return 1;
+ }
+
+ BenchmarkReporter reporter("FFTBatch Benchmark (" +
+ std::to_string(config.degree()) + ", " +
+ std::to_string(config.batch_size()) + ")",
+ config.repeating_num());
+
+ for (const Config::Vendor vendor : config.vendors()) {
+ reporter.AddVendor(Config::VendorToString(vendor));
+ }
+
+ switch (config.prime_field()) {
+ case Config::PrimeField::kBabyBear: {
+ Run<math::BabyBear>(reporter, config, run_fft_batch_plonky3_baby_bear);
+ break;
+ }
+ }
+
+ reporter.AddAverageToLastRow();
+ reporter.Show();
+
+ return 0;
+}
+
+} // namespace tachyon::benchmark::fft_batch
+
+int main(int argc, char** argv) {
+ return tachyon::benchmark::fft_batch::RealMain(argc, argv);
+}

benchmark/fft_batch/benchmark_reporter.cc

@@ -0,0 +1,33 @@
+#include "benchmark/fft_batch/benchmark_reporter.h"
+
+#include "tachyon/base/containers/container_util.h"
+#include "tachyon/base/strings/string_number_conversions.h"
+
+namespace tachyon::benchmark::fft_batch {
+
+BenchmarkReporter::BenchmarkReporter(std::string_view title,
+ size_t repeating_num)
+ : SimpleBenchmarkReporter(title, "Trial number") {
+ targets_ = base::CreateVector(
+ repeating_num, [](size_t i) { return base::NumberToString(i); });
+ targets_.push_back("avg");
+
+ AddVendor("tachyon");
+ times_.resize(repeating_num + 1);
+}
+
+void BenchmarkReporter::AddVendor(std::string_view name) {
+ column_headers_.push_back(std::string(name));
+}
+
+void BenchmarkReporter::AddAverageToLastRow() {
+ for (size_t i = 0; i < column_headers_.size(); ++i) {
+ double total = 0;
+ for (size_t j = 0; j < targets_.size() - 1; ++j) {
+ total += times_[j][i];
+ }
+ AddTime(targets_.size() - 1, total / (targets_.size() - 1));
+ }
+}
+
+} // namespace tachyon::benchmark::fft_batch

benchmark/fft_batch/benchmark_reporter.h

@@ -0,0 +1,23 @@
+#ifndef BENCHMARK_FFT_BATCH_BENCHMARK_REPORTER_H_
+#define BENCHMARK_FFT_BATCH_BENCHMARK_REPORTER_H_
+
+#include <string>
+
+#include "benchmark/simple_benchmark_reporter.h"
+
+namespace tachyon::benchmark::fft_batch {
+
+class BenchmarkReporter : public SimpleBenchmarkReporter {
+ public:
+ BenchmarkReporter(std::string_view title, size_t repeating_num);
+ BenchmarkReporter(const BenchmarkReporter& other) = delete;
+ BenchmarkReporter& operator=(const BenchmarkReporter& other) = delete;
+
+ void AddVendor(std::string_view name);
+
+ void AddAverageToLastRow();
+};
+
+} // namespace tachyon::benchmark::fft_batch
+
+#endif // BENCHMARK_FFT_BATCH_BENCHMARK_REPORTER_H_