From c2951f3747de28247820d30d7c51059d616e7e89 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Thu, 9 Nov 2023 10:21:51 -0500 Subject: [PATCH 01/11] plumbing a webassembly::interface object for benchmarking BLS host functions directly --- benchmark/CMakeLists.txt | 3 ++- libraries/chain/include/eosio/chain/transaction_context.hpp | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index a9a76190ca..e6dd5b939f 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -1,7 +1,8 @@ file(GLOB BENCHMARK "*.cpp") add_executable( benchmark ${BENCHMARK} ) -target_link_libraries( benchmark fc Boost::program_options bn256) +target_link_libraries( benchmark eosio_testing fc Boost::program_options bn256) target_include_directories( benchmark PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}" + "${CMAKE_CURRENT_BINARY_DIR}/../unittests/include" ) diff --git a/libraries/chain/include/eosio/chain/transaction_context.hpp b/libraries/chain/include/eosio/chain/transaction_context.hpp index 430defce27..f9b719c760 100644 --- a/libraries/chain/include/eosio/chain/transaction_context.hpp +++ b/libraries/chain/include/eosio/chain/transaction_context.hpp @@ -92,6 +92,7 @@ namespace eosio { namespace chain { friend struct controller_impl; friend class apply_context; + friend struct interface_in_benchmark; // defined in benchmark/bls.cpp void add_ram_usage( account_name account, int64_t ram_delta ); From be7dcc08665abd264c74b05e1a2559653834d544 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Thu, 9 Nov 2023 10:47:06 -0500 Subject: [PATCH 02/11] benchmark bls_g1_add, bls_g2_add, bls_g1_mul, bls_g2_mul --- benchmark/benchmark.cpp | 1 + benchmark/benchmark.hpp | 1 + benchmark/bls.cpp | 232 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 234 insertions(+) create mode 100644 benchmark/bls.cpp diff --git a/benchmark/benchmark.cpp b/benchmark/benchmark.cpp index e38d55237a..4ee2b76687 100644 --- a/benchmark/benchmark.cpp +++ b/benchmark/benchmark.cpp @@ -15,6 +15,7 @@ std::map> features { { "key", key_benchmarking }, { "hash", hash_benchmarking }, { "blake2", blake2_benchmarking }, + { "bls", bls_benchmarking } }; // values to control cout format diff --git a/benchmark/benchmark.hpp b/benchmark/benchmark.hpp index aed818fb5d..94848a0442 100644 --- a/benchmark/benchmark.hpp +++ b/benchmark/benchmark.hpp @@ -19,6 +19,7 @@ void modexp_benchmarking(); void key_benchmarking(); void hash_benchmarking(); void blake2_benchmarking(); +void bls_benchmarking(); void benchmarking(std::string name, const std::function& func); diff --git a/benchmark/bls.cpp b/benchmark/bls.cpp new file mode 100644 index 0000000000..564ec21c74 --- /dev/null +++ b/benchmark/bls.cpp @@ -0,0 +1,232 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace eosio; +using namespace eosio::chain; +using namespace eosio::testing; +using namespace bls12_381; + +namespace eosio::chain { +// placed in eosio::chain name space so that interface_in_benchmark +// can access transaction_context's private member max_transaction_time_subjective. +// interface_in_benchmark is a friend of transaction_context. + +// To benchmark host functions directly without going through CDT +// wrappers, we need to contruct a eosio::chain::webassembly::interface +// object, as host functions are implemented in eosio::chain::webassembly::interface +struct interface_in_benchmark { + interface_in_benchmark() { + // prevent logging from messing up benchmark results display + fc::logger::get(DEFAULT_LOGGER).set_log_level(fc::log_level::off); + + // create a chain + fc::temp_directory tempdir; + auto conf_genesis = tester::default_config( tempdir ); + auto& cfg = conf_genesis.second.initial_configuration; + // configure large cpu usgaes so expensive BLS functions like pairing + // can run a reasonable number of times within a trasaction time + cfg.max_block_cpu_usage = 500'000; + cfg.max_transaction_cpu_usage = 480'000; + cfg.min_transaction_cpu_usage = 1; + chain = std::make_unique(conf_genesis.first, conf_genesis.second); + chain->execute_setup_policy( setup_policy::full ); + + // get hold controller + control = chain->control.get(); + + // create account and deploy contract for a temp transaction + chain->create_accounts( {"payloadless"_n} ); + chain->set_code( "payloadless"_n, test_contracts::payloadless_wasm() ); + chain->set_abi( "payloadless"_n, test_contracts::payloadless_abi() ); + + // construct a signed transaction + fc::variant pretty_trx = fc::mutable_variant_object() + ("actions", fc::variants({ + fc::mutable_variant_object() + ("account", name("payloadless"_n)) + ("name", "doit") + ("authorization", fc::variants({ + fc::mutable_variant_object() + ("actor", name("payloadless"_n)) + ("permission", name(config::active_name)) + })) + ("data", fc::mutable_variant_object() + ) + }) + ); + trx = std::make_unique(); + abi_serializer::from_variant(pretty_trx, *trx, chain->get_resolver(), abi_serializer::create_yield_function( chain->abi_serializer_max_time )); + chain->set_transaction_headers(*trx); + trx->sign( chain->get_private_key( "payloadless"_n, "active" ), control->get_chain_id() ); + + // construct a packed transaction + ptrx = std::make_unique(*trx, eosio::chain::packed_transaction::compression_type::zlib); + + // build transaction context from the packed transaction + timer = std::make_unique(); + trx_timer = std::make_unique(*timer); + trx_ctx = std::make_unique(*control, *ptrx, ptrx->id(), std::move(*trx_timer)); + trx_ctx->max_transaction_time_subjective = fc::microseconds::maximum(); + trx_ctx->init_for_input_trx( ptrx->get_unprunable_size(), ptrx->get_prunable_size() ); + trx_ctx->exec(); + + // build apply context from the control and transaction context + apply_ctx = std::make_unique(*control, *trx_ctx, 1); + + // finally build the interface + interface = std::make_unique(*apply_ctx); + } + + std::unique_ptr chain; + controller* control; + std::unique_ptr trx; + std::unique_ptr ptrx; + std::unique_ptr timer; + std::unique_ptr trx_timer; + std::unique_ptr trx_ctx; + std::unique_ptr apply_ctx; + std::unique_ptr interface; +}; +} // namespace eosio::chain + + +namespace benchmark { + +// utilility to create a random scalar +std::array random_scalar() +{ + std::random_device rd; + std::mt19937_64 gen(rd()); + std::uniform_int_distribution dis; + + return { + dis(gen) % bls12_381::fp::Q[0], + dis(gen) % bls12_381::fp::Q[1], + dis(gen) % bls12_381::fp::Q[2], + dis(gen) % bls12_381::fp::Q[3] + }; +} + +// utilility to create a random g1 +bls12_381::g1 random_g1() +{ + std::array k = random_scalar(); + return bls12_381::g1::one().mulScalar(k); +} + +// utilility to create a random g2 +bls12_381::g2 random_g2() +{ + std::array k = random_scalar(); + return bls12_381::g2::one().mulScalar(k); +} + +void benchmark_bls_g1_add() { + // prepare g1 operand in Jacobian LE format + g1 p = random_g1(); + std::vector buf(144); + p.toJacobianBytesLE(std::span((uint8_t*)buf.data(), 144), true); + eosio::chain::span op1(buf.data(), buf.size()); + + // prepare result operand + std::vector result_buf(144); + eosio::chain::span result(result_buf.data(), result_buf.size()); + + // set up bls_g1_add to be benchmarked + interface_in_benchmark interface; + auto g1_add_func = [&]() { + interface.interface->bls_g1_add(op1, op1, result); + }; + + benchmarking("bls_g1_add", g1_add_func); +} + +void benchmark_bls_g2_add() { + // prepare g2 operand in Jacobian LE format + g2 p = random_g2(); + std::vector buf(288); + p.toJacobianBytesLE(std::span((uint8_t*)buf.data(), 288), true); + eosio::chain::span op(buf.data(), buf.size()); + + // prepare result operand + std::vector result_buf(288); + eosio::chain::span result(result_buf.data(), result_buf.size()); + + // set up bls_g2_add to be benchmarked + interface_in_benchmark interface; + auto benchmarked_func = [&]() { + interface.interface->bls_g2_add(op, op, result); + }; + + benchmarking("bls_g2_add", benchmarked_func); +} + +void benchmark_bls_g1_mul() { + // prepare g1 operand + g1 p = random_g1(); + std::vector buf(144); + p.toJacobianBytesLE(std::span((uint8_t*)buf.data(), 144), true); + eosio::chain::span point(buf.data(), buf.size()); + + // prepare scalar operand + std::array s = random_scalar(); + std::vector scalar_buf(32); + scalar::toBytesLE(s, std::span((uint8_t*)scalar_buf.data(), 32)); + eosio::chain::span scalar(scalar_buf.data(), scalar_buf.size()); + + // prepare result operand + std::vector result_buf(144); + eosio::chain::span result(result_buf.data(), result_buf.size()); + + // set up bls_g1_mul to be benchmarked + interface_in_benchmark interface; + auto benchmarked_func = [&]() { + interface.interface->bls_g1_mul(point, scalar, result); + }; + + benchmarking("bls_g1_mul", benchmarked_func); +} + +void benchmark_bls_g2_mul() { + g2 p = random_g2(); + std::vector buf(288); + p.toJacobianBytesLE(std::span((uint8_t*)buf.data(), 288), true); + eosio::chain::span point(buf.data(), buf.size()); + + // prepare scalar operand + std::array s = random_scalar(); + std::vector scalar_buf(32); + scalar::toBytesLE(s, std::span((uint8_t*)scalar_buf.data(), 32)); + eosio::chain::span scalar(scalar_buf.data(), scalar_buf.size()); + + // prepare result operand + std::vector result_buf(288); + eosio::chain::span result(result_buf.data(), result_buf.size()); + + // set up bls_g2_mul to be benchmarked + interface_in_benchmark interface; + auto benchmarked_func = [&]() { + interface.interface->bls_g2_mul(point, scalar, result); + }; + + benchmarking("bls_g2_mul", benchmarked_func); +} + +void bls_benchmarking() { + benchmark_bls_g1_add(); + benchmark_bls_g2_add(); + benchmark_bls_g1_mul(); + benchmark_bls_g2_mul(); +} +} // namespace benchmark From 6f3b7debda259e8f777687b677fbe933ef6db911 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Thu, 9 Nov 2023 11:34:13 -0500 Subject: [PATCH 03/11] support benchmarked function specific limit of number of runs --- benchmark/benchmark.cpp | 18 ++++++++++++++---- benchmark/benchmark.hpp | 3 ++- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/benchmark/benchmark.cpp b/benchmark/benchmark.cpp index 4ee2b76687..aa8726cf2a 100644 --- a/benchmark/benchmark.cpp +++ b/benchmark/benchmark.cpp @@ -63,10 +63,20 @@ bytes to_bytes(const std::string& source) { return output; }; -void benchmarking(std::string name, const std::function& func) { - uint64_t total {0}, min {std::numeric_limits::max()}, max {0}; +void benchmarking(const std::string& name, + const std::function& func, + uint32_t max_num_runs) { - for (auto i = 0U; i < num_runs; ++i) { + uint64_t total{0}; + uint64_t min{std::numeric_limits::max()}; + uint64_t max{0}; + + // some benchmarked functions are expensive and run in a transaction + // (like BLS pairing). Limit actual number of runs to the function + // specific maximum such that transaction deadline is not exceeded. + uint32_t actual_num_runs = (num_runs > max_num_runs) ? max_num_runs : num_runs; + + for (auto i = 0U; i < actual_num_runs; ++i) { auto start_time = std::chrono::high_resolution_clock::now(); func(); auto end_time = std::chrono::high_resolution_clock::now(); @@ -77,7 +87,7 @@ void benchmarking(std::string name, const std::function& func) { max = std::max(max, duration); } - print_results(name, num_runs, total, min, max); + print_results(name, actual_num_runs, total, min, max); } } // benchmark diff --git a/benchmark/benchmark.hpp b/benchmark/benchmark.hpp index 94848a0442..8f1935ef18 100644 --- a/benchmark/benchmark.hpp +++ b/benchmark/benchmark.hpp @@ -3,6 +3,7 @@ #include #include #include +#include #include @@ -21,6 +22,6 @@ void hash_benchmarking(); void blake2_benchmarking(); void bls_benchmarking(); -void benchmarking(std::string name, const std::function& func); +void benchmarking(const std::string& name, const std::function& func, uint32_t max_num_runs = std::numeric_limits::max()); } // benchmark From b2e214dc3b5b72be12a040b0306812c63133518e Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Thu, 9 Nov 2023 12:24:22 -0500 Subject: [PATCH 04/11] benchmark bls_pairing, bls_g1_exp, bls_g2_exp --- benchmark/bls.cpp | 124 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) diff --git a/benchmark/bls.cpp b/benchmark/bls.cpp index 564ec21c74..6a78b428f6 100644 --- a/benchmark/bls.cpp +++ b/benchmark/bls.cpp @@ -118,6 +118,10 @@ std::array random_scalar() }; } +// some functions like pairing/exp are expensive. set a hard limit to +// the number of runs such that the transanction does not reach deadline +constexpr uint32_t num_runs_limit = 200; + // utilility to create a random g1 bls12_381::g1 random_g1() { @@ -223,10 +227,130 @@ void benchmark_bls_g2_mul() { benchmarking("bls_g2_mul", benchmarked_func); } +void benchmark_bls_g1_exp(std::string test_name, uint32_t num_points) { + // prepare g1 points operand + std::vector g1_buf(144*num_points); + for (auto i=0u; i < num_points; ++i) { + g1 p = random_g1(); + p.toJacobianBytesLE(std::span((uint8_t*)g1_buf.data() + i * 144, 144), true); + } + chain::span g1_points(g1_buf.data(), g1_buf.size()); + + // prepare scalars operand + std::vector scalars_buf(32*num_points); + for (auto i=0u; i < num_points; ++i) { + std::array s = random_scalar(); + scalar::toBytesLE(s, std::span((uint8_t*)scalars_buf.data() + i*32, 32)); + } + chain::span scalars(scalars_buf.data(), scalars_buf.size()); + + // prepare result operand + std::vector result_buf(144); + eosio::chain::span result(result_buf.data(), result_buf.size()); + + // set up bls_g1_exp to be benchmarked + interface_in_benchmark interface; + auto benchmarked_func = [&]() { + interface.interface->bls_g1_exp(g1_points, scalars, num_points, result); + }; + + benchmarking(test_name, benchmarked_func, num_runs_limit); +} + +void benchmark_bls_g1_exp_one_point() { + benchmark_bls_g1_exp("bls_g1_exp 1 point", 1); +} + +void benchmark_bls_g1_exp_three_point() { + benchmark_bls_g1_exp("bls_g1_exp 3 points", 3); +} + +void benchmark_bls_g2_exp(std::string test_name, uint32_t num_points) { + // prepare g2 points operand + std::vector g2_buf(288*num_points); + for (auto i=0u; i < num_points; ++i) { + g2 p = random_g2(); + p.toJacobianBytesLE(std::span((uint8_t*)g2_buf.data() + i * 288, 288), true); + } + eosio::chain::span g2_points(g2_buf.data(), g2_buf.size()); + + // prepare scalars operand + std::vector scalars_buf(32*num_points); + for (auto i=0u; i < num_points; ++i) { + std::array s = random_scalar(); + scalar::toBytesLE(s, std::span((uint8_t*)scalars_buf.data() + i*32, 32)); + } + eosio::chain::span scalars(scalars_buf.data(), scalars_buf.size()); + + // prepare result operand + std::vector result_buf(288); + eosio::chain::span result(result_buf.data(), result_buf.size()); + + // set up bls_g2_exp to be benchmarked + interface_in_benchmark interface; + auto benchmarked_func = [&]() { + interface.interface->bls_g2_exp(g2_points, scalars, num_points, result); + }; + + benchmarking(test_name, benchmarked_func, num_runs_limit); +} + +void benchmark_bls_g2_exp_one_point() { + benchmark_bls_g2_exp("bls_g2_exp 1 point", 1); +} + +void benchmark_bls_g2_exp_three_point() { + benchmark_bls_g2_exp("bls_g2_exp 3 points", 3); +} + +void benchmark_bls_pairing(std::string test_name, uint32_t num_pairs) { + // prepare g1 operand + std::vector g1_buf(144*num_pairs); + //g1_buf.reserve(144*num_pairs); + for (auto i=0u; i < num_pairs; ++i) { + g1 p = random_g1(); + p.toJacobianBytesLE(std::span((uint8_t*)g1_buf.data() + i * 144, 144), true); + } + eosio::chain::span g1_points(g1_buf.data(), g1_buf.size()); + + // prepare g2 operand + std::vector g2_buf(288*num_pairs); + for (auto i=0u; i < num_pairs; ++i) { + g2 p2 = random_g2(); + p2.toJacobianBytesLE(std::span((uint8_t*)g2_buf.data() + i * 288, (288)), true); + } + eosio::chain::span g2_points(g2_buf.data(), g2_buf.size()); + + // prepare result operand + std::vector result_buf(576); + eosio::chain::span result(result_buf.data(), result_buf.size()); + + // set up bls_pairing to be benchmarked + interface_in_benchmark interface; + auto benchmarked_func = [&]() { + interface.interface->bls_pairing(g1_points, g2_points, num_pairs, result); + }; + + benchmarking(test_name, benchmarked_func, num_runs_limit); +} + +void benchmark_bls_pairing_one_pair() { + benchmark_bls_pairing("bls_pairing 1 pair", 1); +} + +void benchmark_bls_pairing_three_pair() { + benchmark_bls_pairing("bls_pairing 3 pairs", 3); +} void bls_benchmarking() { benchmark_bls_g1_add(); benchmark_bls_g2_add(); benchmark_bls_g1_mul(); benchmark_bls_g2_mul(); + benchmark_bls_pairing_one_pair(); + benchmark_bls_pairing_three_pair(); + benchmark_bls_g1_exp_one_point(); + benchmark_bls_g1_exp_three_point(); + benchmark_bls_g2_exp_one_point(); + benchmark_bls_g2_exp_three_point(); } } // namespace benchmark From 63b90eb908fe6a526814d632b9b9422ce9ae6ed8 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Thu, 9 Nov 2023 12:38:58 -0500 Subject: [PATCH 05/11] benchmark bls_g1_map, bls_g2_map, and bls_fp_mod --- benchmark/bls.cpp | 81 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 71 insertions(+), 10 deletions(-) diff --git a/benchmark/bls.cpp b/benchmark/bls.cpp index 6a78b428f6..75c69aef17 100644 --- a/benchmark/bls.cpp +++ b/benchmark/bls.cpp @@ -1,14 +1,8 @@ #include -#include -#include -#include -#include #include #include -#include -#include +#include #include -#include #include #include @@ -17,14 +11,18 @@ using namespace eosio::chain; using namespace eosio::testing; using namespace bls12_381; +// benchmark BLS host functions directly without relying on CDT wrappers. +// to run a benchmarking session, in the build directory, type +// benchmark/benchmark -f bls + namespace eosio::chain { // placed in eosio::chain name space so that interface_in_benchmark // can access transaction_context's private member max_transaction_time_subjective. // interface_in_benchmark is a friend of transaction_context. -// To benchmark host functions directly without going through CDT -// wrappers, we need to contruct a eosio::chain::webassembly::interface -// object, as host functions are implemented in eosio::chain::webassembly::interface +// To benchmark host functions directly without CDT wrappers, +// we need to contruct a eosio::chain::webassembly::interface object, +// as host functions are implemented in eosio::chain::webassembly::interface struct interface_in_benchmark { interface_in_benchmark() { // prevent logging from messing up benchmark results display @@ -341,6 +339,66 @@ void benchmark_bls_pairing_one_pair() { void benchmark_bls_pairing_three_pair() { benchmark_bls_pairing("bls_pairing 3 pairs", 3); } + +void benchmark_bls_g1_map() { + // prepare e operand. Must be fp LE. + std::vector e_buf = {0xc9, 0x3f,0x81,0x7b, 0x15, 0x9b, 0xdf, 0x84, 0x04, 0xdc, 0x37, 0x85, 0x14, 0xf8, 0x45, 0x19, 0x2b, 0xba, 0xe4, 0xfa, 0xac, 0x7f, 0x4a, 0x56, 0x89, 0x24, 0xf2, 0xd9, 0x72, 0x51, 0x25, 0x00, 0x04, 0x89, 0x40, 0x8f, 0xd7, 0x96, 0x46, 0x1c, 0x28, 0x89, 0x00, 0xad, 0xd0, 0x0d, 0x46, 0x18}; + eosio::chain::span e((char*)e_buf.data(), e_buf.size()); + + // prepare result operand + std::vector result_buf(144); + eosio::chain::span result(result_buf.data(), result_buf.size()); + + // set up bls_g1_map to be benchmarked + interface_in_benchmark interface; + auto benchmarked_func = [&]() { + interface.interface->bls_g1_map(e, result); + }; + + benchmarking("bls_g1_map", benchmarked_func); +} + +void benchmark_bls_g2_map() { + // prepare e operand. Must be fp2 LE. + std::vector e_buf = {0xd4, 0xf2, 0xcf, 0xec, 0x99, 0x38, 0x78, 0x09, 0x57, 0x4f, 0xcc, 0x2d, 0xba, 0x10, 0x56, 0x03, 0xd9, 0x50, 0xd4, 0x90, 0xe2, 0xbe, 0xbe, 0x0c, 0x21, 0x2c, 0x05, 0xe1, 0x6b, 0x78, 0x47, 0x45, 0xef, 0x4f, 0xe8, 0xe7, 0x0b, 0x55, 0x4d, 0x0a, 0x52, 0xfe, 0x0b, 0xed, 0x5e, 0xa6, 0x69, 0x0a, 0xde, 0x23, 0x48, 0xeb, 0x89, 0x72, 0xa9, 0x67, 0x40, 0xa4, 0x30, 0xdf, 0x16, 0x2d, 0x92, 0x0e, 0x17, 0x5f, 0x59, 0x23, 0xa7, 0x6d, 0x18, 0x65, 0x0e, 0xa2, 0x4a, 0x8e, 0xc0, 0x6d, 0x41, 0x4c, 0x6d, 0x1d, 0x21, 0x8d, 0x67, 0x3d, 0xac, 0x36, 0x19, 0xa1, 0xa5, 0xc1, 0x42, 0x78, 0x57, 0x08}; + eosio::chain::span e((char*)e_buf.data(), e_buf.size()); + + // prepare result operand + std::vector result_buf(288); + eosio::chain::span result(result_buf.data(), result_buf.size()); + + // set up bls_g2_map to be benchmarked + interface_in_benchmark interface; + auto benchmarked_func = [&]() { + interface.interface->bls_g2_map(e, result); + }; + + benchmarking("bls_g2_map", benchmarked_func); +} + +void benchmark_bls_fp_mod() { + // prepare scalar operand + std::vector scalar_buf(64); + // random_scalar returns 32 bytes. need to call it twice + for (auto i=0u; i < 2; ++i) { + std::array s = random_scalar(); + scalar::toBytesLE(s, std::span((uint8_t*)scalar_buf.data() + i*32, 32)); + } + chain::span scalar(scalar_buf.data(), scalar_buf.size()); + + // prepare result operand + std::vector result_buf(48); + eosio::chain::span result(result_buf.data(), result_buf.size()); + + // set up bls_fp_mod to be benchmarked + interface_in_benchmark interface; + auto benchmarked_func = [&]() { + interface.interface->bls_fp_mod(scalar, result); + }; + + benchmarking("bls_fp_mod", benchmarked_func); +} + void bls_benchmarking() { benchmark_bls_g1_add(); benchmark_bls_g2_add(); @@ -352,5 +410,8 @@ void bls_benchmarking() { benchmark_bls_g1_exp_three_point(); benchmark_bls_g2_exp_one_point(); benchmark_bls_g2_exp_three_point(); + benchmark_bls_g1_map(); + benchmark_bls_g2_map(); + benchmark_bls_fp_mod(); } } // namespace benchmark From 181d24259060d9f7ea3e52a745b93cf18647b5a0 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Thu, 9 Nov 2023 14:08:04 -0500 Subject: [PATCH 06/11] print number of runs right aligned since now the numbers can be different between benchmarked functions --- benchmark/benchmark.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/benchmark.cpp b/benchmark/benchmark.cpp index aa8726cf2a..bf93c9c239 100644 --- a/benchmark/benchmark.cpp +++ b/benchmark/benchmark.cpp @@ -47,10 +47,10 @@ void print_results(std::string name, uint32_t runs, uint64_t total, uint64_t min std::cout.imbue(std::locale("")); std::cout << std::setw(name_width) << std::left << name - << std::setw(runs_width) << runs // std::fixed for not printing 1234 in 1.234e3. // setprecision(0) for not printing fractions << std::right << std::fixed << std::setprecision(0) + << std::setw(runs_width) << runs << std::setw(time_width) << total/runs << std::setw(ns_width) << " ns" << std::setw(time_width) << min << std::setw(ns_width) << " ns" << std::setw(time_width) << max << std::setw(ns_width) << " ns" From 4b65a4f14f64a90f431115857d3d92c8c48c74dc Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Thu, 9 Nov 2023 16:22:19 -0500 Subject: [PATCH 07/11] use a single eosio::benchmark for BLS while still being a friend of eosio::chain::transaction_context --- benchmark/alt_bn_128.cpp | 2 +- benchmark/benchmark.cpp | 2 +- benchmark/benchmark.hpp | 2 +- benchmark/blake2.cpp | 2 +- benchmark/bls.cpp | 9 +-------- benchmark/hash.cpp | 2 +- benchmark/key.cpp | 2 +- benchmark/main.cpp | 6 +++--- benchmark/modexp.cpp | 2 +- .../chain/include/eosio/chain/transaction_context.hpp | 6 +++++- 10 files changed, 16 insertions(+), 19 deletions(-) diff --git a/benchmark/alt_bn_128.cpp b/benchmark/alt_bn_128.cpp index 5ccb1e09be..da1bda15f9 100644 --- a/benchmark/alt_bn_128.cpp +++ b/benchmark/alt_bn_128.cpp @@ -4,7 +4,7 @@ #include -namespace benchmark { +namespace eosio::benchmark { using bytes = std::vector; using g1g2_pair = std::vector; diff --git a/benchmark/benchmark.cpp b/benchmark/benchmark.cpp index bf93c9c239..c72040ab8f 100644 --- a/benchmark/benchmark.cpp +++ b/benchmark/benchmark.cpp @@ -5,7 +5,7 @@ #include -namespace benchmark { +namespace eosio::benchmark { // update this map when a new feature is supported // key is the name and value is the function doing benchmarking diff --git a/benchmark/benchmark.hpp b/benchmark/benchmark.hpp index 8f1935ef18..6ac14e4c47 100644 --- a/benchmark/benchmark.hpp +++ b/benchmark/benchmark.hpp @@ -7,7 +7,7 @@ #include -namespace benchmark { +namespace eosio::benchmark { using bytes = std::vector; void set_num_runs(uint32_t runs); diff --git a/benchmark/blake2.cpp b/benchmark/blake2.cpp index 19aa59d771..a5c971a492 100644 --- a/benchmark/blake2.cpp +++ b/benchmark/blake2.cpp @@ -2,7 +2,7 @@ #include -namespace benchmark { +namespace eosio::benchmark { void blake2_benchmarking() { uint32_t _rounds = 0x0C; diff --git a/benchmark/bls.cpp b/benchmark/bls.cpp index 75c69aef17..97219d338d 100644 --- a/benchmark/bls.cpp +++ b/benchmark/bls.cpp @@ -15,10 +15,7 @@ using namespace bls12_381; // to run a benchmarking session, in the build directory, type // benchmark/benchmark -f bls -namespace eosio::chain { -// placed in eosio::chain name space so that interface_in_benchmark -// can access transaction_context's private member max_transaction_time_subjective. -// interface_in_benchmark is a friend of transaction_context. +namespace eosio::benchmark { // To benchmark host functions directly without CDT wrappers, // we need to contruct a eosio::chain::webassembly::interface object, @@ -96,10 +93,6 @@ struct interface_in_benchmark { std::unique_ptr apply_ctx; std::unique_ptr interface; }; -} // namespace eosio::chain - - -namespace benchmark { // utilility to create a random scalar std::array random_scalar() diff --git a/benchmark/hash.cpp b/benchmark/hash.cpp index 9bb4aef2b4..9193fae80d 100644 --- a/benchmark/hash.cpp +++ b/benchmark/hash.cpp @@ -10,7 +10,7 @@ using namespace fc; -namespace benchmark { +namespace eosio::benchmark { void hash_benchmarking() { std::string small_message = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ01"; diff --git a/benchmark/key.cpp b/benchmark/key.cpp index 3d6973d806..cdc8707d0c 100644 --- a/benchmark/key.cpp +++ b/benchmark/key.cpp @@ -9,7 +9,7 @@ using namespace fc::crypto; using namespace fc; using namespace std::literals; -namespace benchmark { +namespace eosio::benchmark { void k1_sign_benchmarking() { auto payload = "Test Cases"; diff --git a/benchmark/main.cpp b/benchmark/main.cpp index eadc35ede4..f8d6663eed 100644 --- a/benchmark/main.cpp +++ b/benchmark/main.cpp @@ -12,7 +12,7 @@ int main(int argc, char* argv[]) { uint32_t num_runs = 1; std::string feature_name; - auto features = benchmark::get_features(); + auto features = eosio::benchmark::get_features(); options_description cli ("benchmark command line options"); cli.add_options() @@ -61,8 +61,8 @@ int main(int argc, char* argv[]) { std::cerr << "unknown exception" << std::endl; } - benchmark::set_num_runs(num_runs); - benchmark::print_header(); + eosio::benchmark::set_num_runs(num_runs); + eosio::benchmark::print_header(); if (feature_name.empty()) { for (auto& [name, f]: features) { diff --git a/benchmark/modexp.cpp b/benchmark/modexp.cpp index 14d8443331..7bf5fbedeb 100644 --- a/benchmark/modexp.cpp +++ b/benchmark/modexp.cpp @@ -5,7 +5,7 @@ #include -namespace benchmark { +namespace eosio::benchmark { void modexp_benchmarking() { std::mt19937 r(0x11223344); diff --git a/libraries/chain/include/eosio/chain/transaction_context.hpp b/libraries/chain/include/eosio/chain/transaction_context.hpp index f9b719c760..806e75d9dd 100644 --- a/libraries/chain/include/eosio/chain/transaction_context.hpp +++ b/libraries/chain/include/eosio/chain/transaction_context.hpp @@ -4,6 +4,10 @@ #include #include +namespace eosio::benchmark { + struct interface_in_benchmark; // for benchmark testing +} + namespace eosio { namespace chain { struct transaction_checktime_timer { @@ -92,7 +96,7 @@ namespace eosio { namespace chain { friend struct controller_impl; friend class apply_context; - friend struct interface_in_benchmark; // defined in benchmark/bls.cpp + friend struct benchmark::interface_in_benchmark; // defined in benchmark/bls.cpp void add_ram_usage( account_name account, int64_t ram_delta ); From ec1e594c7ca75b35a76d9c766044001902378812 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Thu, 9 Nov 2023 16:43:36 -0500 Subject: [PATCH 08/11] remove little used control as a member of interface_in_benchmark, use chain->control.get() directly instead --- benchmark/bls.cpp | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/benchmark/bls.cpp b/benchmark/bls.cpp index 97219d338d..9d1a874bfc 100644 --- a/benchmark/bls.cpp +++ b/benchmark/bls.cpp @@ -37,9 +37,6 @@ struct interface_in_benchmark { chain = std::make_unique(conf_genesis.first, conf_genesis.second); chain->execute_setup_policy( setup_policy::full ); - // get hold controller - control = chain->control.get(); - // create account and deploy contract for a temp transaction chain->create_accounts( {"payloadless"_n} ); chain->set_code( "payloadless"_n, test_contracts::payloadless_wasm() ); @@ -63,7 +60,7 @@ struct interface_in_benchmark { trx = std::make_unique(); abi_serializer::from_variant(pretty_trx, *trx, chain->get_resolver(), abi_serializer::create_yield_function( chain->abi_serializer_max_time )); chain->set_transaction_headers(*trx); - trx->sign( chain->get_private_key( "payloadless"_n, "active" ), control->get_chain_id() ); + trx->sign( chain->get_private_key( "payloadless"_n, "active" ), chain->control.get()->get_chain_id() ); // construct a packed transaction ptrx = std::make_unique(*trx, eosio::chain::packed_transaction::compression_type::zlib); @@ -71,20 +68,19 @@ struct interface_in_benchmark { // build transaction context from the packed transaction timer = std::make_unique(); trx_timer = std::make_unique(*timer); - trx_ctx = std::make_unique(*control, *ptrx, ptrx->id(), std::move(*trx_timer)); + trx_ctx = std::make_unique(*chain->control.get(), *ptrx, ptrx->id(), std::move(*trx_timer)); trx_ctx->max_transaction_time_subjective = fc::microseconds::maximum(); trx_ctx->init_for_input_trx( ptrx->get_unprunable_size(), ptrx->get_prunable_size() ); trx_ctx->exec(); // build apply context from the control and transaction context - apply_ctx = std::make_unique(*control, *trx_ctx, 1); + apply_ctx = std::make_unique(*chain->control.get(), *trx_ctx, 1); // finally build the interface interface = std::make_unique(*apply_ctx); } std::unique_ptr chain; - controller* control; std::unique_ptr trx; std::unique_ptr ptrx; std::unique_ptr timer; From a53e6fc9295579a8e5c68538e4c6fd032a5d6e0e Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Thu, 9 Nov 2023 16:58:23 -0500 Subject: [PATCH 09/11] set max block and transaction cpu to 999'999'999 such that expensive functions will never reach deadline --- benchmark/bls.cpp | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/benchmark/bls.cpp b/benchmark/bls.cpp index 9d1a874bfc..b1cc09260e 100644 --- a/benchmark/bls.cpp +++ b/benchmark/bls.cpp @@ -30,9 +30,9 @@ struct interface_in_benchmark { auto conf_genesis = tester::default_config( tempdir ); auto& cfg = conf_genesis.second.initial_configuration; // configure large cpu usgaes so expensive BLS functions like pairing - // can run a reasonable number of times within a trasaction time - cfg.max_block_cpu_usage = 500'000; - cfg.max_transaction_cpu_usage = 480'000; + // can finish within a trasaction time + cfg.max_block_cpu_usage = 999'999'999; + cfg.max_transaction_cpu_usage = 999'999'990; cfg.min_transaction_cpu_usage = 1; chain = std::make_unique(conf_genesis.first, conf_genesis.second); chain->execute_setup_policy( setup_policy::full ); @@ -105,10 +105,6 @@ std::array random_scalar() }; } -// some functions like pairing/exp are expensive. set a hard limit to -// the number of runs such that the transanction does not reach deadline -constexpr uint32_t num_runs_limit = 200; - // utilility to create a random g1 bls12_381::g1 random_g1() { @@ -241,7 +237,7 @@ void benchmark_bls_g1_exp(std::string test_name, uint32_t num_points) { interface.interface->bls_g1_exp(g1_points, scalars, num_points, result); }; - benchmarking(test_name, benchmarked_func, num_runs_limit); + benchmarking(test_name, benchmarked_func); } void benchmark_bls_g1_exp_one_point() { @@ -279,7 +275,7 @@ void benchmark_bls_g2_exp(std::string test_name, uint32_t num_points) { interface.interface->bls_g2_exp(g2_points, scalars, num_points, result); }; - benchmarking(test_name, benchmarked_func, num_runs_limit); + benchmarking(test_name, benchmarked_func); } void benchmark_bls_g2_exp_one_point() { @@ -318,7 +314,7 @@ void benchmark_bls_pairing(std::string test_name, uint32_t num_pairs) { interface.interface->bls_pairing(g1_points, g2_points, num_pairs, result); }; - benchmarking(test_name, benchmarked_func, num_runs_limit); + benchmarking(test_name, benchmarked_func); } void benchmark_bls_pairing_one_pair() { From f54724fb2967afebbc7f559672173f595a3f2e44 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Thu, 9 Nov 2023 17:22:08 -0500 Subject: [PATCH 10/11] revert benchmarked function specific limit of number of runs changes, as they are not needed --- benchmark/benchmark.cpp | 14 +++----------- benchmark/benchmark.hpp | 2 +- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/benchmark/benchmark.cpp b/benchmark/benchmark.cpp index c72040ab8f..5c142c4162 100644 --- a/benchmark/benchmark.cpp +++ b/benchmark/benchmark.cpp @@ -63,20 +63,12 @@ bytes to_bytes(const std::string& source) { return output; }; -void benchmarking(const std::string& name, - const std::function& func, - uint32_t max_num_runs) { - +void benchmarking(const std::string& name, const std::function& func) { uint64_t total{0}; uint64_t min{std::numeric_limits::max()}; uint64_t max{0}; - // some benchmarked functions are expensive and run in a transaction - // (like BLS pairing). Limit actual number of runs to the function - // specific maximum such that transaction deadline is not exceeded. - uint32_t actual_num_runs = (num_runs > max_num_runs) ? max_num_runs : num_runs; - - for (auto i = 0U; i < actual_num_runs; ++i) { + for (auto i = 0U; i < num_runs; ++i) { auto start_time = std::chrono::high_resolution_clock::now(); func(); auto end_time = std::chrono::high_resolution_clock::now(); @@ -87,7 +79,7 @@ void benchmarking(const std::string& name, max = std::max(max, duration); } - print_results(name, actual_num_runs, total, min, max); + print_results(name, num_runs, total, min, max); } } // benchmark diff --git a/benchmark/benchmark.hpp b/benchmark/benchmark.hpp index 6ac14e4c47..51fe4b6af9 100644 --- a/benchmark/benchmark.hpp +++ b/benchmark/benchmark.hpp @@ -22,6 +22,6 @@ void hash_benchmarking(); void blake2_benchmarking(); void bls_benchmarking(); -void benchmarking(const std::string& name, const std::function& func, uint32_t max_num_runs = std::numeric_limits::max()); +void benchmarking(const std::string& name, const std::function& func); } // benchmark From a96d6dea49c6ecea0c32b9e0c67ecc49fbea130e Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Thu, 9 Nov 2023 18:19:27 -0500 Subject: [PATCH 11/11] improve and add more comments --- benchmark/bls.cpp | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/benchmark/bls.cpp b/benchmark/bls.cpp index b1cc09260e..e5b5ad3843 100644 --- a/benchmark/bls.cpp +++ b/benchmark/bls.cpp @@ -11,18 +11,20 @@ using namespace eosio::chain; using namespace eosio::testing; using namespace bls12_381; -// benchmark BLS host functions directly without relying on CDT wrappers. -// to run a benchmarking session, in the build directory, type -// benchmark/benchmark -f bls +// Benchmark BLS host functions without relying on CDT wrappers. +// +// To run a benchmarking session, in the build directory, type +// benchmark/benchmark -f bls namespace eosio::benchmark { // To benchmark host functions directly without CDT wrappers, -// we need to contruct a eosio::chain::webassembly::interface object, -// as host functions are implemented in eosio::chain::webassembly::interface +// we need to contruct an eosio::chain::webassembly::interface object, +// because host functions are implemented in +// eosio::chain::webassembly::interface class. struct interface_in_benchmark { interface_in_benchmark() { - // prevent logging from messing up benchmark results display + // prevent logging from interwined with output benchmark results fc::logger::get(DEFAULT_LOGGER).set_log_level(fc::log_level::off); // create a chain @@ -71,12 +73,12 @@ struct interface_in_benchmark { trx_ctx = std::make_unique(*chain->control.get(), *ptrx, ptrx->id(), std::move(*trx_timer)); trx_ctx->max_transaction_time_subjective = fc::microseconds::maximum(); trx_ctx->init_for_input_trx( ptrx->get_unprunable_size(), ptrx->get_prunable_size() ); - trx_ctx->exec(); + trx_ctx->exec(); // this is required to generate action traces to be used by apply_context constructor // build apply context from the control and transaction context apply_ctx = std::make_unique(*chain->control.get(), *trx_ctx, 1); - // finally build the interface + // finally construct the interface interface = std::make_unique(*apply_ctx); } @@ -119,6 +121,7 @@ bls12_381::g2 random_g2() return bls12_381::g2::one().mulScalar(k); } +// bls_g1_add benchmarking void benchmark_bls_g1_add() { // prepare g1 operand in Jacobian LE format g1 p = random_g1(); @@ -139,6 +142,7 @@ void benchmark_bls_g1_add() { benchmarking("bls_g1_add", g1_add_func); } +// bls_g2_add benchmarking void benchmark_bls_g2_add() { // prepare g2 operand in Jacobian LE format g2 p = random_g2(); @@ -159,6 +163,7 @@ void benchmark_bls_g2_add() { benchmarking("bls_g2_add", benchmarked_func); } +// bls_g1_mul benchmarking void benchmark_bls_g1_mul() { // prepare g1 operand g1 p = random_g1(); @@ -185,6 +190,7 @@ void benchmark_bls_g1_mul() { benchmarking("bls_g1_mul", benchmarked_func); } +// bls_g2_mul benchmarking void benchmark_bls_g2_mul() { g2 p = random_g2(); std::vector buf(288); @@ -210,6 +216,7 @@ void benchmark_bls_g2_mul() { benchmarking("bls_g2_mul", benchmarked_func); } +// bls_g1_exp benchmarking utility void benchmark_bls_g1_exp(std::string test_name, uint32_t num_points) { // prepare g1 points operand std::vector g1_buf(144*num_points); @@ -240,14 +247,17 @@ void benchmark_bls_g1_exp(std::string test_name, uint32_t num_points) { benchmarking(test_name, benchmarked_func); } +// bls_g1_exp benchmarking with 1 input point void benchmark_bls_g1_exp_one_point() { benchmark_bls_g1_exp("bls_g1_exp 1 point", 1); } +// bls_g1_exp benchmarking with 3 input points void benchmark_bls_g1_exp_three_point() { benchmark_bls_g1_exp("bls_g1_exp 3 points", 3); } +// bls_g2_exp benchmarking utility void benchmark_bls_g2_exp(std::string test_name, uint32_t num_points) { // prepare g2 points operand std::vector g2_buf(288*num_points); @@ -278,14 +288,17 @@ void benchmark_bls_g2_exp(std::string test_name, uint32_t num_points) { benchmarking(test_name, benchmarked_func); } +// bls_g2_exp benchmarking with 1 input point void benchmark_bls_g2_exp_one_point() { benchmark_bls_g2_exp("bls_g2_exp 1 point", 1); } +// bls_g2_exp benchmarking with 3 input points void benchmark_bls_g2_exp_three_point() { benchmark_bls_g2_exp("bls_g2_exp 3 points", 3); } +// bls_pairing benchmarking utility void benchmark_bls_pairing(std::string test_name, uint32_t num_pairs) { // prepare g1 operand std::vector g1_buf(144*num_pairs); @@ -317,14 +330,17 @@ void benchmark_bls_pairing(std::string test_name, uint32_t num_pairs) { benchmarking(test_name, benchmarked_func); } +// bls_pairing benchmarking with 1 input pair void benchmark_bls_pairing_one_pair() { benchmark_bls_pairing("bls_pairing 1 pair", 1); } +// bls_pairing benchmarking with 3 input pairs void benchmark_bls_pairing_three_pair() { benchmark_bls_pairing("bls_pairing 3 pairs", 3); } +// bls_g1_map benchmarking void benchmark_bls_g1_map() { // prepare e operand. Must be fp LE. std::vector e_buf = {0xc9, 0x3f,0x81,0x7b, 0x15, 0x9b, 0xdf, 0x84, 0x04, 0xdc, 0x37, 0x85, 0x14, 0xf8, 0x45, 0x19, 0x2b, 0xba, 0xe4, 0xfa, 0xac, 0x7f, 0x4a, 0x56, 0x89, 0x24, 0xf2, 0xd9, 0x72, 0x51, 0x25, 0x00, 0x04, 0x89, 0x40, 0x8f, 0xd7, 0x96, 0x46, 0x1c, 0x28, 0x89, 0x00, 0xad, 0xd0, 0x0d, 0x46, 0x18}; @@ -343,6 +359,7 @@ void benchmark_bls_g1_map() { benchmarking("bls_g1_map", benchmarked_func); } +// bls_g2_map benchmarking void benchmark_bls_g2_map() { // prepare e operand. Must be fp2 LE. std::vector e_buf = {0xd4, 0xf2, 0xcf, 0xec, 0x99, 0x38, 0x78, 0x09, 0x57, 0x4f, 0xcc, 0x2d, 0xba, 0x10, 0x56, 0x03, 0xd9, 0x50, 0xd4, 0x90, 0xe2, 0xbe, 0xbe, 0x0c, 0x21, 0x2c, 0x05, 0xe1, 0x6b, 0x78, 0x47, 0x45, 0xef, 0x4f, 0xe8, 0xe7, 0x0b, 0x55, 0x4d, 0x0a, 0x52, 0xfe, 0x0b, 0xed, 0x5e, 0xa6, 0x69, 0x0a, 0xde, 0x23, 0x48, 0xeb, 0x89, 0x72, 0xa9, 0x67, 0x40, 0xa4, 0x30, 0xdf, 0x16, 0x2d, 0x92, 0x0e, 0x17, 0x5f, 0x59, 0x23, 0xa7, 0x6d, 0x18, 0x65, 0x0e, 0xa2, 0x4a, 0x8e, 0xc0, 0x6d, 0x41, 0x4c, 0x6d, 0x1d, 0x21, 0x8d, 0x67, 0x3d, 0xac, 0x36, 0x19, 0xa1, 0xa5, 0xc1, 0x42, 0x78, 0x57, 0x08}; @@ -361,6 +378,7 @@ void benchmark_bls_g2_map() { benchmarking("bls_g2_map", benchmarked_func); } +// bls_fp_mod benchmarking void benchmark_bls_fp_mod() { // prepare scalar operand std::vector scalar_buf(64); @@ -384,6 +402,7 @@ void benchmark_bls_fp_mod() { benchmarking("bls_fp_mod", benchmarked_func); } +// register benchmarking functions void bls_benchmarking() { benchmark_bls_g1_add(); benchmark_bls_g2_add();