From 833e1c703c9b7010b9669180cead2c178bc3c7c3 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Mon, 15 Jan 2024 13:07:10 -0500 Subject: [PATCH 1/8] sign and broadcast vote message --- libraries/chain/controller.cpp | 39 +++++++++++++++++++ .../eosio/chain/block_header_state.hpp | 2 +- .../chain/include/eosio/chain/controller.hpp | 2 + .../include/eosio/chain/plugin_interface.hpp | 5 ++- plugins/net_plugin/net_plugin.cpp | 9 +++++ 5 files changed, 54 insertions(+), 3 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index d5b2857a14..633bd6ea14 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -884,6 +884,7 @@ struct controller_impl { named_thread_pool thread_pool; deep_mind_handler* deep_mind_logger = nullptr; bool okay_to_print_integrity_hash_on_stop = false; + bls_key_map_t finalizer_keys_on_the_node; thread_local static platform_timer timer; // a copy for main thread and each read-only thread #if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED) @@ -2825,6 +2826,34 @@ struct controller_impl { } } FC_CAPTURE_AND_RETHROW() } /// apply_block + void create_and_send_vote_msg(const block_header_state& bhs) { +#warning use decide_vote() for strong after it is implementd by https://github.com/AntelopeIO/leap/issues/2070 + bool strong = true; + + // A vote is created and signed by each finalizer configured on the node that + // in active finalizer policy +#warning possible optimization: when setting finalizer policy, update finalizer_keys_on_the_node to indicate if it is active to save going over bhs.finalizer_policy->finalizers + // go over each active finalizer and check if it is configured on the node + for (const auto& f: bhs.finalizer_policy->finalizers) { + auto it = finalizer_keys_on_the_node.find( f.public_key ); + if( it != finalizer_keys_on_the_node.end() ) { + const auto& private_key = it->second; + + // signing can take long time, do it asynchronously + auto sig_fut = post_async_task(thread_pool.get_executor(), + [&]() { + const auto& digest = bhs.compute_finalizer_digest(); + return private_key.sign(std::vector(digest.data(), digest.data() + digest.data_size())); }); + + // construct the vote message + hs_vote_message vote{ bhs.id, strong, private_key.get_public_key(), sig_fut.get() }; + + // net plugin subscribed this signal. it will broadcast the vote message + // on receiving the signal + emit( self.voted_block, vote ); + } + } + } // thread safe, expected to be called from thread other than the main thread block_state_legacy_ptr create_block_state_i( const block_id_type& id, const signed_block_ptr& b, const block_header_state_legacy& prev ) { @@ -3407,6 +3436,12 @@ struct controller_impl { wasmif.code_block_num_last_used(code_hash, vm_type, vm_version, block_num); } + void set_finalizer_keys_on_the_node(const bls_pub_priv_key_map_t finalizer_keys) { + for (const auto& k : finalizer_keys) { + finalizer_keys_on_the_node[bls_public_key{k.first}] = bls_private_key{k.second}; + } + } + bool irreversible_mode() const { return read_mode == db_read_mode::IRREVERSIBLE; } const block_id_type& fork_db_head_block_id() const { return block_data.fork_db_head_block_id(irreversible_mode()); } uint32_t fork_db_head_block_num() const { return block_data.fork_db_head_block_num(irreversible_mode()); } @@ -4461,6 +4496,10 @@ void controller::code_block_num_last_used(const digest_type& code_hash, uint8_t return my->code_block_num_last_used(code_hash, vm_type, vm_version, block_num); } +void controller::set_finalizer_keys_on_the_node(const bls_pub_priv_key_map_t finalizer_keys) { + my->set_finalizer_keys_on_the_node(finalizer_keys); +} + /// Protocol feature activation handlers: template<> diff --git a/libraries/chain/include/eosio/chain/block_header_state.hpp b/libraries/chain/include/eosio/chain/block_header_state.hpp index da9e525041..d87d837faf 100644 --- a/libraries/chain/include/eosio/chain/block_header_state.hpp +++ b/libraries/chain/include/eosio/chain/block_header_state.hpp @@ -61,7 +61,7 @@ struct block_header_state { flat_map finalizer_policies; // ------ functions ----------------------------------------------------------------- - digest_type compute_finalizer_digest() const; + digest_type compute_finalizer_digest() const { return id; }; block_timestamp_type timestamp() const { return header.timestamp; } account_name producer() const { return header.producer; } const block_id_type& previous() const { return header.previous; } diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index b563e4a5ca..746479d2cc 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -362,6 +362,7 @@ namespace eosio::chain { signal accepted_block; signal irreversible_block; signal)> applied_transaction; + signal voted_block; const apply_handler* find_apply_handler( account_name contract, scope_name scope, action_name act )const; wasm_interface& get_wasm_interface(); @@ -383,6 +384,7 @@ namespace eosio::chain { void set_to_read_window(); bool is_write_window() const; void code_block_num_last_used(const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, uint32_t block_num); + void set_finalizer_keys_on_the_node(const bls_pub_priv_key_map_t finalizer_keys); private: friend class apply_context; diff --git a/plugins/chain_interface/include/eosio/chain/plugin_interface.hpp b/plugins/chain_interface/include/eosio/chain/plugin_interface.hpp index 4fe1c247d4..b6f29abea8 100644 --- a/plugins/chain_interface/include/eosio/chain/plugin_interface.hpp +++ b/plugins/chain_interface/include/eosio/chain/plugin_interface.hpp @@ -16,9 +16,10 @@ namespace eosio::chain::plugin_interface { namespace channels { using rejected_block = channel_decl; using accepted_block_header = channel_decl; - using accepted_block = channel_decl; - using irreversible_block = channel_decl; + using accepted_block = channel_decl; + using irreversible_block = channel_decl; using applied_transaction = channel_decl; + using voted_block = channel_decl; } namespace methods { diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index bf8f7d0a07..f44865ac18 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -534,6 +534,7 @@ namespace eosio { void on_accepted_block_header( const signed_block_ptr& block, const block_id_type& id ); void on_accepted_block(); + void on_voted_block ( const hs_vote_message& vote ); void transaction_ack(const std::pair&); void on_irreversible_block( const block_id_type& id, uint32_t block_num ); @@ -3933,6 +3934,11 @@ namespace eosio { on_active_schedule(chain_plug->chain().active_producers()); } + // called from application thread + void net_plugin_impl::on_voted_block(const hs_vote_message& vote) { + bcast_hs_message(std::nullopt, chain::hs_message{ vote }); + } + void net_plugin_impl::bcast_hs_message( const std::optional& exclude_peer, const chain::hs_message& msg ) { fc_dlog(logger, "sending hs msg: ${msg}", ("msg", msg)); @@ -4359,6 +4365,9 @@ namespace eosio { my->on_irreversible_block( id, block->block_num() ); } ); + cc.voted_block.connect( [my = shared_from_this()]( const hs_vote_message& vote ) { + my->on_voted_block(vote); + } ); } { From 0bcb9b3f0e9e10d4c280bb7d7b922958aa6fad80 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Mon, 15 Jan 2024 15:28:56 -0500 Subject: [PATCH 2/8] do not sign asynchronously and add & to the missing parameter of set_finalizer_keys_on_the_node --- libraries/chain/controller.cpp | 15 +++++---------- .../chain/include/eosio/chain/controller.hpp | 2 +- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 633bd6ea14..5eb8ec899d 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -2832,21 +2832,16 @@ struct controller_impl { // A vote is created and signed by each finalizer configured on the node that // in active finalizer policy -#warning possible optimization: when setting finalizer policy, update finalizer_keys_on_the_node to indicate if it is active to save going over bhs.finalizer_policy->finalizers - // go over each active finalizer and check if it is configured on the node for (const auto& f: bhs.finalizer_policy->finalizers) { auto it = finalizer_keys_on_the_node.find( f.public_key ); if( it != finalizer_keys_on_the_node.end() ) { const auto& private_key = it->second; + const auto& digest = bhs.compute_finalizer_digest(); - // signing can take long time, do it asynchronously - auto sig_fut = post_async_task(thread_pool.get_executor(), - [&]() { - const auto& digest = bhs.compute_finalizer_digest(); - return private_key.sign(std::vector(digest.data(), digest.data() + digest.data_size())); }); + auto sig = private_key.sign(std::vector(digest.data(), digest.data() + digest.data_size())); // construct the vote message - hs_vote_message vote{ bhs.id, strong, private_key.get_public_key(), sig_fut.get() }; + hs_vote_message vote{ bhs.id, strong, private_key.get_public_key(), sig }; // net plugin subscribed this signal. it will broadcast the vote message // on receiving the signal @@ -3436,7 +3431,7 @@ struct controller_impl { wasmif.code_block_num_last_used(code_hash, vm_type, vm_version, block_num); } - void set_finalizer_keys_on_the_node(const bls_pub_priv_key_map_t finalizer_keys) { + void set_finalizer_keys_on_the_node(const bls_pub_priv_key_map_t& finalizer_keys) { for (const auto& k : finalizer_keys) { finalizer_keys_on_the_node[bls_public_key{k.first}] = bls_private_key{k.second}; } @@ -4496,7 +4491,7 @@ void controller::code_block_num_last_used(const digest_type& code_hash, uint8_t return my->code_block_num_last_used(code_hash, vm_type, vm_version, block_num); } -void controller::set_finalizer_keys_on_the_node(const bls_pub_priv_key_map_t finalizer_keys) { +void controller::set_finalizer_keys_on_the_node(const bls_pub_priv_key_map_t& finalizer_keys) { my->set_finalizer_keys_on_the_node(finalizer_keys); } diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index 746479d2cc..1d04570be0 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -384,7 +384,7 @@ namespace eosio::chain { void set_to_read_window(); bool is_write_window() const; void code_block_num_last_used(const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, uint32_t block_num); - void set_finalizer_keys_on_the_node(const bls_pub_priv_key_map_t finalizer_keys); + void set_finalizer_keys_on_the_node(const bls_pub_priv_key_map_t& finalizer_keys); private: friend class apply_context; From dd7de545338a0c75908c5a4d080d1e81e75340d5 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Mon, 15 Jan 2024 15:29:34 -0500 Subject: [PATCH 3/8] call set_finalizer_keys_on_the_node at the startup of producer plugin --- plugins/producer_plugin/producer_plugin.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index 559d6e53af..a6b36bfafa 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -1340,6 +1340,9 @@ void producer_plugin_impl::plugin_startup() { EOS_ASSERT(_producers.empty() || chain_plug->accept_transactions(), plugin_config_exception, "node cannot have any producer-name configured because no block production is possible with no [api|p2p]-accepted-transactions"); + chain.set_finalizer_keys_on_the_node(_finalizer_keys); + +#warning TODO remove create_pacemaker chain.create_pacemaker(_producers, std::move(_finalizer_keys), hotstuff_logger); _finalizer_keys.clear(); From 8d9e230ed8cb0bc52358a4b04b84d0f97f0ac937 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Mon, 15 Jan 2024 15:30:44 -0500 Subject: [PATCH 4/8] add a TODO warning to compute_finalizer_digest --- libraries/chain/include/eosio/chain/block_header_state.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/chain/include/eosio/chain/block_header_state.hpp b/libraries/chain/include/eosio/chain/block_header_state.hpp index d87d837faf..b8289a972f 100644 --- a/libraries/chain/include/eosio/chain/block_header_state.hpp +++ b/libraries/chain/include/eosio/chain/block_header_state.hpp @@ -61,6 +61,7 @@ struct block_header_state { flat_map finalizer_policies; // ------ functions ----------------------------------------------------------------- +#warning TDDO https://github.com/AntelopeIO/leap/issues/2080 digest_type compute_finalizer_digest() const { return id; }; block_timestamp_type timestamp() const { return header.timestamp; } account_name producer() const { return header.producer; } From bb457b7d61cf290dd20b4b5d3a1e2333508f425d Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Tue, 16 Jan 2024 15:13:25 -0500 Subject: [PATCH 5/8] change finalizer_keys_on_the_node to node_finalizer_keys --- libraries/chain/controller.cpp | 14 +++++++------- libraries/chain/include/eosio/chain/controller.hpp | 2 +- plugins/producer_plugin/producer_plugin.cpp | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 5eb8ec899d..7f604e8726 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -884,7 +884,7 @@ struct controller_impl { named_thread_pool thread_pool; deep_mind_handler* deep_mind_logger = nullptr; bool okay_to_print_integrity_hash_on_stop = false; - bls_key_map_t finalizer_keys_on_the_node; + bls_key_map_t node_finalizer_keys; thread_local static platform_timer timer; // a copy for main thread and each read-only thread #if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED) @@ -2833,8 +2833,8 @@ struct controller_impl { // A vote is created and signed by each finalizer configured on the node that // in active finalizer policy for (const auto& f: bhs.finalizer_policy->finalizers) { - auto it = finalizer_keys_on_the_node.find( f.public_key ); - if( it != finalizer_keys_on_the_node.end() ) { + auto it = node_finalizer_keys.find( f.public_key ); + if( it != node_finalizer_keys.end() ) { const auto& private_key = it->second; const auto& digest = bhs.compute_finalizer_digest(); @@ -3431,9 +3431,9 @@ struct controller_impl { wasmif.code_block_num_last_used(code_hash, vm_type, vm_version, block_num); } - void set_finalizer_keys_on_the_node(const bls_pub_priv_key_map_t& finalizer_keys) { + void set_node_finalizer_keys(const bls_pub_priv_key_map_t& finalizer_keys) { for (const auto& k : finalizer_keys) { - finalizer_keys_on_the_node[bls_public_key{k.first}] = bls_private_key{k.second}; + node_finalizer_keys[bls_public_key{k.first}] = bls_private_key{k.second}; } } @@ -4491,8 +4491,8 @@ void controller::code_block_num_last_used(const digest_type& code_hash, uint8_t return my->code_block_num_last_used(code_hash, vm_type, vm_version, block_num); } -void controller::set_finalizer_keys_on_the_node(const bls_pub_priv_key_map_t& finalizer_keys) { - my->set_finalizer_keys_on_the_node(finalizer_keys); +void controller::set_node_finalizer_keys(const bls_pub_priv_key_map_t& finalizer_keys) { + my->set_node_finalizer_keys(finalizer_keys); } /// Protocol feature activation handlers: diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index 1d04570be0..3793c72158 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -384,7 +384,7 @@ namespace eosio::chain { void set_to_read_window(); bool is_write_window() const; void code_block_num_last_used(const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, uint32_t block_num); - void set_finalizer_keys_on_the_node(const bls_pub_priv_key_map_t& finalizer_keys); + void set_node_finalizer_keys(const bls_pub_priv_key_map_t& finalizer_keys); private: friend class apply_context; diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index a6b36bfafa..6049dda887 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -492,7 +492,7 @@ class producer_plugin_impl : public std::enable_shared_from_this _signature_providers; - chain::bls_pub_priv_key_map_t _finalizer_keys; // public, private + chain::bls_pub_priv_key_map_t _finalizer_keys; // public, private std::set _producers; boost::asio::deadline_timer _timer; block_timing_util::producer_watermarks _producer_watermarks; @@ -1340,7 +1340,7 @@ void producer_plugin_impl::plugin_startup() { EOS_ASSERT(_producers.empty() || chain_plug->accept_transactions(), plugin_config_exception, "node cannot have any producer-name configured because no block production is possible with no [api|p2p]-accepted-transactions"); - chain.set_finalizer_keys_on_the_node(_finalizer_keys); + chain.set_node_finalizer_keys(_finalizer_keys); #warning TODO remove create_pacemaker chain.create_pacemaker(_producers, std::move(_finalizer_keys), hotstuff_logger); From d05cb02b1e4ca1c5d902219fe5d88a5ffec07d69 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Tue, 16 Jan 2024 16:35:09 -0500 Subject: [PATCH 6/8] resolve merge conflict --- libraries/chain/controller.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index e2c6993df4..b6b1260628 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -2984,7 +2984,7 @@ struct controller_impl { // A vote is created and signed by each finalizer configured on the node that // in active finalizer policy - for (const auto& f: bhs.finalizer_policy->finalizers) { + for (const auto& f: bhs.active_finalizer_policy->finalizers) { auto it = node_finalizer_keys.find( f.public_key ); if( it != node_finalizer_keys.end() ) { const auto& private_key = it->second; From 7977340271b1f05530e9749a24f2f492bea11b08 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Tue, 16 Jan 2024 22:54:42 -0500 Subject: [PATCH 7/8] send out vote and integrate received QC into the claimed block after block state is created --- libraries/chain/controller.cpp | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index b6b1260628..3227243d54 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -317,6 +317,15 @@ struct block_data_t { }, v); } + block_state_ptr fork_db_fetch_bsp_by_num(uint32_t block_num) const { + return std::visit(overloaded{ + [](const block_data_legacy_t&) -> block_state_ptr { return nullptr; }, + [&](const block_data_new_t& bd) -> block_state_ptr { + auto bsp = bd.fork_db.search_on_branch(bd.fork_db.head()->id(), block_num); + return bsp; } + }, v); + } + template R apply(F& f) { if constexpr (std::is_same_v) @@ -2978,22 +2987,22 @@ struct controller_impl { } FC_CAPTURE_AND_RETHROW(); } /// apply_block - void create_and_send_vote_msg(const block_header_state& bhs) { + void create_and_send_vote_msg(const block_state_ptr& bsp) { #warning use decide_vote() for strong after it is implementd by https://github.com/AntelopeIO/leap/issues/2070 bool strong = true; // A vote is created and signed by each finalizer configured on the node that // in active finalizer policy - for (const auto& f: bhs.active_finalizer_policy->finalizers) { + for (const auto& f: bsp->active_finalizer_policy->finalizers) { auto it = node_finalizer_keys.find( f.public_key ); if( it != node_finalizer_keys.end() ) { const auto& private_key = it->second; - const auto& digest = bhs.compute_finalizer_digest(); + const auto& digest = bsp->compute_finalizer_digest(); auto sig = private_key.sign(std::vector(digest.data(), digest.data() + digest.data_size())); // construct the vote message - hs_vote_message vote{ bhs.id, strong, private_key.get_public_key(), sig }; + hs_vote_message vote{ bsp->id(), strong, private_key.get_public_key(), sig }; // net plugin subscribed this signal. it will broadcast the vote message // on receiving the signal @@ -3025,6 +3034,17 @@ struct controller_impl { return block_token{bsp}; } + void integrate_received_qc_to_block(const block_id_type& id, const signed_block_ptr& b) { +#warning validate received QC before integrate it: https://github.com/AntelopeIO/leap/issues/2071 + // extract QC from block extensions + auto block_exts = b->validate_and_extract_extensions(); + if( block_exts.count(quorum_certificate_extension::extension_id()) > 0 ) { + const auto& qc_ext = std::get(block_exts. lower_bound(quorum_certificate_extension::extension_id())->second); + auto last_qc_block_bsp = block_data.fork_db_fetch_bsp_by_num( qc_ext.qc.block_height ); + last_qc_block_bsp->valid_qc = qc_ext.qc.qc; + } + } + // thread safe, expected to be called from thread other than the main thread block_token create_block_state_i( const block_id_type& id, const signed_block_ptr& b, const block_header_state& prev ) { auto trx_mroot = calculate_trx_merkle( b->transactions, true ); @@ -3045,6 +3065,12 @@ struct controller_impl { EOS_ASSERT( id == bsp->id(), block_validate_exception, "provided id ${id} does not match block id ${bid}", ("id", id)("bid", bsp->id()) ); + + create_and_send_vote_msg(bsp); + + // integrate the received QC into the claimed block + integrate_received_qc_to_block(id, b); + return block_token{bsp}; } From 447c773a43a9c6bcc16c2f0af53f8ca518e9106a Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Wed, 17 Jan 2024 09:39:44 -0500 Subject: [PATCH 8/8] add a TODO warning --- libraries/chain/controller.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 3227243d54..3f0b4f3c8a 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -3041,6 +3041,7 @@ struct controller_impl { if( block_exts.count(quorum_certificate_extension::extension_id()) > 0 ) { const auto& qc_ext = std::get(block_exts. lower_bound(quorum_certificate_extension::extension_id())->second); auto last_qc_block_bsp = block_data.fork_db_fetch_bsp_by_num( qc_ext.qc.block_height ); +#warning a mutex might be needed for updating valid_pc last_qc_block_bsp->valid_qc = qc_ext.qc.qc; } }