Merge pull request #1926 from AntelopeIO/GH-1916-refactor

IF: Refactor chain_pacemaker ownership

CMakeModules/EosioTester.cmake.in

@@ -47,6 +47,7 @@ find_library(libfc fc @CMAKE_INSTALL_FULL_LIBDIR@ NO_DEFAULT_PATH)
 find_library(libsecp256k1 secp256k1 @CMAKE_INSTALL_FULL_LIBDIR@ NO_DEFAULT_PATH)
 find_library(libbn256 bn256 @CMAKE_INSTALL_FULL_LIBDIR@ NO_DEFAULT_PATH)
 find_library(libbls12-381 bls12-381 @CMAKE_INSTALL_FULL_LIBDIR@ NO_DEFAULT_PATH)
+find_library(libhotstuff hotstuff @CMAKE_INSTALL_FULL_LIBDIR@ NO_DEFAULT_PATH)
 
 find_library(libwasm WASM @CMAKE_INSTALL_FULL_LIBDIR@ NO_DEFAULT_PATH)
 find_library(libwast WAST @CMAKE_INSTALL_FULL_LIBDIR@ NO_DEFAULT_PATH)
@@ -72,6 +73,7 @@ add_library(EosioChain INTERFACE)
 
 target_link_libraries(EosioChain INTERFACE
    ${libchain}
+   ${libhotstuff}
    ${libfc}
    ${libwast}
    ${libwasm}

CMakeModules/EosioTesterBuild.cmake.in

@@ -44,6 +44,7 @@ find_library(libfc fc @CMAKE_BINARY_DIR@/libraries/libfc NO_DEFAULT_PATH)
 find_library(libsecp256k1 secp256k1 @CMAKE_BINARY_DIR@/libraries/libfc/secp256k1 NO_DEFAULT_PATH)
 find_library(libbn256 bn256 @CMAKE_BINARY_DIR@/libraries/libfc/libraries/bn256/src NO_DEFAULT_PATH)
 find_library(libbls12-381 bls12-381 @CMAKE_BINARY_DIR@/libraries/libfc/libraries/bls12-381 NO_DEFAULT_PATH)
+find_library(libhotstuff hotstuff @CMAKE_BINARY_DIR@/libraries/hotstuff NO_DEFAULT_PATH)
 
 find_library(libwasm WASM @CMAKE_BINARY_DIR@/libraries/wasm-jit/Source/WASM NO_DEFAULT_PATH)
 find_library(libwast WAST @CMAKE_BINARY_DIR@/libraries/wasm-jit/Source/WAST NO_DEFAULT_PATH)
@@ -69,6 +70,7 @@ add_library(EosioChain INTERFACE)
 
 target_link_libraries(EosioChain INTERFACE
    ${libchain}
+   ${libhotstuff}
    ${libfc}
    ${libwast}
    ${libwasm}

libraries/chain/CMakeLists.txt

@@ -144,7 +144,7 @@ add_library( eosio_chain
 add_library(boost_numeric_ublas INTERFACE)
 add_library(Boost::numeric_ublas ALIAS boost_numeric_ublas)
 
-target_link_libraries( eosio_chain PUBLIC bn256 fc chainbase eosio_rapidjson Logging IR WAST WASM
+target_link_libraries( eosio_chain PUBLIC hotstuff bn256 fc chainbase eosio_rapidjson Logging IR WAST WASM
                        softfloat builtins ${CHAIN_EOSVM_LIBRARIES} ${LLVM_LIBS} ${CHAIN_RT_LINKAGE}
                        Boost::signals2 Boost::hana Boost::property_tree Boost::multi_index Boost::asio Boost::lockfree
                        Boost::assign Boost::accumulators

libraries/chain/block_header_state.cpp

@@ -128,6 +128,7 @@ namespace eosio { namespace chain {
          // fork_database will prefer hotstuff blocks over dpos blocks
          result.dpos_irreversible_blocknum          = hs_dpos_irreversible_blocknum;
          // Change to active on the next().next() producer block_num
+         // TODO: use calculated hotstuff lib instead of block_num
          if( pending_schedule.schedule.producers.size() &&
              block_num >= detail::get_next_next_round_block_num(when, pending_schedule.schedule_lib_num)) {
             result.active_schedule = pending_schedule.schedule;

libraries/chain/controller.cpp

@@ -28,6 +28,8 @@
 #include <eosio/chain/deep_mind.hpp>
 #include <eosio/chain/finalizer_set.hpp>
 #include <eosio/chain/finalizer_authority.hpp>
+#include <eosio/hotstuff/hotstuff.hpp>
+#include <eosio/hotstuff/chain_pacemaker.hpp>
 
 #include <chainbase/chainbase.hpp>
 #include <eosio/vm/allocator.hpp>
@@ -238,6 +240,7 @@ struct controller_impl {
    std::optional<pending_state>    pending;
    block_state_ptr                 head;
    fork_database                   fork_db;
+   std::optional<hotstuff::chain_pacemaker> pacemaker;
    std::atomic<uint32_t>           hs_irreversible_block_num{0};
    resource_limits_manager         resource_limits;
    subjective_billing              subjective_bill;
@@ -363,11 +366,6 @@ struct controller_impl {
    SET_APP_HANDLER( eosio, eosio, deleteauth );
    SET_APP_HANDLER( eosio, eosio, linkauth );
    SET_APP_HANDLER( eosio, eosio, unlinkauth );
-/*
-   SET_APP_HANDLER( eosio, eosio, postrecovery );
-   SET_APP_HANDLER( eosio, eosio, passrecovery );
-   SET_APP_HANDLER( eosio, eosio, vetorecovery );
-*/
 
    SET_APP_HANDLER( eosio, eosio, canceldelay );
    }
@@ -1990,6 +1988,7 @@ struct controller_impl {
 
          if( s == controller::block_status::incomplete ) {
             log_irreversible();
+            pacemaker->beat();
          }
       } catch (...) {
          // dont bother resetting pending, instead abort the block
@@ -3325,10 +3324,35 @@ int64_t controller::set_proposed_producers( vector<producer_authority> producers
    return version;
 }
 
+void controller::create_pacemaker(std::set<chain::account_name> my_producers, hotstuff::bls_pub_priv_key_map_t finalizer_keys, fc::logger& hotstuff_logger) {
+   EOS_ASSERT( !my->pacemaker, misc_exception, "duplicate chain_pacemaker initialization" );
+   my->pacemaker.emplace(this, std::move(my_producers), std::move(finalizer_keys), hotstuff_logger);
+}
+
+void controller::register_pacemaker_bcast_function(std::function<void(const std::optional<uint32_t>&, const hotstuff::hs_message&)> bcast_hs_message) {
+   EOS_ASSERT( my->pacemaker, misc_exception, "chain_pacemaker not created" );
+   my->pacemaker->register_bcast_function(std::move(bcast_hs_message));
+}
+
+void controller::register_pacemaker_warn_function(std::function<void(uint32_t, hotstuff::hs_message_warning)> warn_hs_message) {
+   EOS_ASSERT( my->pacemaker, misc_exception, "chain_pacemaker not created" );
+   my->pacemaker->register_warn_function(std::move(warn_hs_message));
+}
+
 void controller::set_proposed_finalizers( const finalizer_set& fin_set ) {
    my->set_proposed_finalizers(fin_set);
 }
 
+void controller::get_finalizer_state( hotstuff::finalizer_state& fs ) const {
+   EOS_ASSERT( my->pacemaker, misc_exception, "chain_pacemaker not created" );
+   my->pacemaker->get_state(fs);
+}
+
+// called from net threads
+void controller::notify_hs_message( const uint32_t connection_id, const hotstuff::hs_message& msg ) {
+   my->pacemaker->on_hs_msg(connection_id, msg);
+};
+
 const producer_authority_schedule&    controller::active_producers()const {
    if( !(my->pending) )
       return  my->head->active_schedule;

libraries/chain/include/eosio/chain/controller.hpp

@@ -3,13 +3,15 @@
 #include <eosio/chain/block_log.hpp>
 #include <eosio/chain/trace.hpp>
 #include <eosio/chain/genesis_state.hpp>
-#include <chainbase/pinnable_mapped_file.hpp>
-#include <boost/signals2/signal.hpp>
-
 #include <eosio/chain/snapshot.hpp>
 #include <eosio/chain/protocol_feature_manager.hpp>
 #include <eosio/chain/webassembly/eos-vm-oc/config.hpp>
 
+#include <chainbase/pinnable_mapped_file.hpp>
+
+#include <boost/signals2/signal.hpp>
+
+
 namespace chainbase {
    class database;
 }
@@ -19,6 +21,13 @@ namespace boost { namespace asio {
 
 namespace eosio { namespace vm { class wasm_allocator; }}
 
+namespace eosio::hotstuff {
+   struct hs_message;
+   struct finalizer_state;
+   enum class hs_message_warning;
+   using bls_pub_priv_key_map_t = std::map<std::string, std::string>;
+}
+
 namespace eosio { namespace chain {
 
    struct finalizer_set;
@@ -294,8 +303,14 @@ namespace eosio { namespace chain {
 
          int64_t set_proposed_producers( vector<producer_authority> producers );
 
+         void create_pacemaker(std::set<chain::account_name> my_producers, hotstuff::bls_pub_priv_key_map_t finalizer_keys, fc::logger& hotstuff_logger);
+         void register_pacemaker_bcast_function(std::function<void(const std::optional<uint32_t>&, const hotstuff::hs_message&)> bcast_hs_message);
+         void register_pacemaker_warn_function(std::function<void(uint32_t, hotstuff::hs_message_warning)> warn_hs_message);
          // called by host function set_finalizers
          void set_proposed_finalizers( const finalizer_set& fin_set );
+         void get_finalizer_state( hotstuff::finalizer_state& fs ) const;
+         // called from net threads
+         void notify_hs_message( const uint32_t connection_id, const hotstuff::hs_message& msg );
 
          bool light_validation_allowed() const;
          bool skip_auth_check()const;

libraries/chain/webassembly/privileged.cpp

@@ -196,7 +196,7 @@ namespace eosio { namespace chain { namespace webassembly {
 
       // system contract should perform a duplicate check and fthreshold check before calling
       EOS_ASSERT( finalizers.size() == unique_finalizer_keys.size(), wasm_execution_error, "Duplicate finalizer bls key in finalizer set" );
-      EOS_ASSERT( finset.fthreshold > f_weight_sum / 2, wasm_execution_error, "Finalizer set threshold cannot be met by finalizer weights" );
+      EOS_ASSERT( finset.fthreshold >= f_weight_sum / 2, wasm_execution_error, "Finalizer set threshold cannot be met by finalizer weights" );
 
       context.control.set_proposed_finalizers( finset );
    }

libraries/hotstuff/CMakeLists.txt

@@ -15,3 +15,10 @@ target_include_directories( hotstuff
                           )
 
 add_subdirectory( test )
+
+install(TARGETS hotstuff
+        LIBRARY DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR} COMPONENT dev EXCLUDE_FROM_ALL
+        ARCHIVE DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR} COMPONENT dev EXCLUDE_FROM_ALL)
+install(DIRECTORY include/eosio/hotstuff/
+        DESTINATION ${CMAKE_INSTALL_FULL_INCLUDEDIR}/eosio/hotstuff
+        COMPONENT dev EXCLUDE_FROM_ALL)

libraries/hotstuff/chain_pacemaker.cpp

@@ -104,7 +104,7 @@ namespace eosio { namespace hotstuff {
 #warning TODO get a data directory str passed into the chain_pacemaker ctor and use it to compose the absolute filepathname that is passed to qc_chain ctor
    chain_pacemaker::chain_pacemaker(controller* chain,
                                     std::set<account_name> my_producers,
-                                    bls_key_map_t finalizer_keys,
+                                    bls_pub_priv_key_map_t finalizer_keys,
                                     fc::logger& logger)
       : _chain(chain),
         _qc_chain("default", this, std::move(my_producers), std::move(finalizer_keys), logger, eosio::chain::config::safetydb_filename),
@@ -119,13 +119,13 @@ namespace eosio { namespace hotstuff {
       _head_block_state = chain->head_block_state();
    }
 
-   void chain_pacemaker::register_bcast_function(std::function<void(const std::optional<uint32_t>&, const chain::hs_message&)> broadcast_hs_message) {
+   void chain_pacemaker::register_bcast_function(std::function<void(const std::optional<uint32_t>&, const hs_message&)> broadcast_hs_message) {
       FC_ASSERT(broadcast_hs_message, "on_hs_message must be provided");
       // no need to std::lock_guard g( _hotstuff_global_mutex ); here since pre-comm init
       bcast_hs_message = std::move(broadcast_hs_message);
    }
 
-   void chain_pacemaker::register_warn_function(std::function<void(const uint32_t, const chain::hs_message_warning&)> warning_hs_message) {
+   void chain_pacemaker::register_warn_function(std::function<void(uint32_t, const hs_message_warning&)> warning_hs_message) {
       FC_ASSERT(warning_hs_message, "must provide callback");
       // no need to std::lock_guard g( _hotstuff_global_mutex ); here since pre-comm init
       warn_hs_message = std::move(warning_hs_message);
@@ -198,7 +198,7 @@ namespace eosio { namespace hotstuff {
       return p_auth.producer_name;
    }
 
-   const finalizer_set& chain_pacemaker::get_finalizer_set(){
+   const finalizer_set& chain_pacemaker::get_finalizer_set() {
       return _active_finalizer_set;
    }
 
@@ -221,29 +221,29 @@ namespace eosio { namespace hotstuff {
    }
 
    void chain_pacemaker::send_hs_proposal_msg(const hs_proposal_message& msg, const std::string& id, const std::optional<uint32_t>& exclude_peer) {
-      bcast_hs_message(exclude_peer, msg);
+      bcast_hs_message(exclude_peer, {msg});
    }
 
    void chain_pacemaker::send_hs_vote_msg(const hs_vote_message& msg, const std::string& id, const std::optional<uint32_t>& exclude_peer) {
-      bcast_hs_message(exclude_peer, msg);
+      bcast_hs_message(exclude_peer, {msg});
    }
 
    void chain_pacemaker::send_hs_new_view_msg(const hs_new_view_message& msg, const std::string& id, const std::optional<uint32_t>& exclude_peer) {
-      bcast_hs_message(exclude_peer, msg);
+      bcast_hs_message(exclude_peer, {msg});
    }
 
-   void chain_pacemaker::send_hs_message_warning(const uint32_t sender_peer, const chain::hs_message_warning code) {
+   void chain_pacemaker::send_hs_message_warning(uint32_t sender_peer, const hs_message_warning code) {
       warn_hs_message(sender_peer, code);
 
    }
 
    // called from net threads
-   void chain_pacemaker::on_hs_msg(const uint32_t connection_id, const eosio::chain::hs_message &msg) {
+   void chain_pacemaker::on_hs_msg(const uint32_t connection_id, const hs_message &msg) {
       std::visit(overloaded{
             [this, connection_id](const hs_vote_message& m) { on_hs_vote_msg(connection_id, m); },
             [this, connection_id](const hs_proposal_message& m) { on_hs_proposal_msg(connection_id, m); },
             [this, connection_id](const hs_new_view_message& m) { on_hs_new_view_msg(connection_id, m); },
-      }, msg);
+      }, msg.msg);
    }
 
    // called from net threads

libraries/hotstuff/include/eosio/hotstuff/base_pacemaker.hpp

@@ -1,15 +1,10 @@
 #pragma once
 
+#include <eosio/hotstuff/hotstuff.hpp>
 #include <eosio/chain/types.hpp>
 #include <eosio/chain/name.hpp>
-#include <eosio/chain/hotstuff.hpp>
-
-#include <fc/crypto/bls_utils.hpp>
-
 #include <eosio/chain/finalizer_set.hpp>
 
-#include <vector>
-
 namespace eosio::hotstuff {
 
    // Abstract pacemaker; a reference of this type will only be used by qc_chain, as qc_chain
@@ -31,14 +26,14 @@ namespace eosio::hotstuff {
       virtual chain::name get_proposer() = 0;
       virtual chain::name get_leader() = 0;
       virtual chain::name get_next_leader() = 0;
-      virtual const eosio::chain::finalizer_set& get_finalizer_set() = 0;
+      virtual const chain::finalizer_set& get_finalizer_set() = 0;
 
       //outbound communications; 'id' is the producer name (can be ignored if/when irrelevant to the implementer)
-      virtual void send_hs_proposal_msg(const chain::hs_proposal_message& msg, const std::string& id, const std::optional<uint32_t>& exclude_peer = std::nullopt) = 0;
-      virtual void send_hs_vote_msg(const chain::hs_vote_message& msg, const std::string& id, const std::optional<uint32_t>& exclude_peer = std::nullopt) = 0;
-      virtual void send_hs_new_view_msg(const chain::hs_new_view_message& msg, const std::string& id, const std::optional<uint32_t>& exclude_peer = std::nullopt) = 0;
+      virtual void send_hs_proposal_msg(const hs_proposal_message& msg, const std::string& id, const std::optional<uint32_t>& exclude_peer = std::nullopt) = 0;
+      virtual void send_hs_vote_msg(const hs_vote_message& msg, const std::string& id, const std::optional<uint32_t>& exclude_peer = std::nullopt) = 0;
+      virtual void send_hs_new_view_msg(const hs_new_view_message& msg, const std::string& id, const std::optional<uint32_t>& exclude_peer = std::nullopt) = 0;
 
-      virtual void send_hs_message_warning(const uint32_t sender_peer, const chain::hs_message_warning code) = 0;
+      virtual void send_hs_message_warning(uint32_t sender_peer, hs_message_warning code) = 0;
 
    };
 

libraries/hotstuff/include/eosio/hotstuff/chain_pacemaker.hpp

@@ -20,10 +20,10 @@ namespace eosio::hotstuff {
 
       chain_pacemaker(controller* chain,
                       std::set<account_name> my_producers,
-                      chain::bls_key_map_t finalizer_keys,
+                      bls_pub_priv_key_map_t finalizer_keys,
                       fc::logger& logger);
-      void register_bcast_function(std::function<void(const std::optional<uint32_t>&, const chain::hs_message&)> broadcast_hs_message);
-      void register_warn_function(std::function<void(const uint32_t, const chain::hs_message_warning&)> warning_hs_message);
+      void register_bcast_function(std::function<void(const std::optional<uint32_t>&, const hs_message&)> broadcast_hs_message);
+      void register_warn_function(std::function<void(uint32_t, const hs_message_warning&)> warning_hs_message);
 
       void beat();
 
@@ -33,20 +33,20 @@ namespace eosio::hotstuff {
 
       //base_pacemaker interface functions
 
-      name get_proposer();
-      name get_leader() ;
-      name get_next_leader() ;
-      const finalizer_set&  get_finalizer_set();
+      name get_proposer() final;
+      name get_leader() final;
+      name get_next_leader() final;
+      const finalizer_set&  get_finalizer_set() final;
 
-      block_id_type get_current_block_id();
+      block_id_type get_current_block_id() final;
 
-      uint32_t get_quorum_threshold();
+      uint32_t get_quorum_threshold() final;
 
-      void send_hs_proposal_msg(const hs_proposal_message& msg, const std::string& id, const std::optional<uint32_t>& exclude_peer);
-      void send_hs_vote_msg(const hs_vote_message& msg, const std::string& id, const std::optional<uint32_t>& exclude_peer);
-      void send_hs_new_view_msg(const hs_new_view_message& msg, const std::string& id, const std::optional<uint32_t>& exclude_peer);
+      void send_hs_proposal_msg(const hs_proposal_message& msg, const std::string& id, const std::optional<uint32_t>& exclude_peer) final;
+      void send_hs_vote_msg(const hs_vote_message& msg, const std::string& id, const std::optional<uint32_t>& exclude_peer) final;
+      void send_hs_new_view_msg(const hs_new_view_message& msg, const std::string& id, const std::optional<uint32_t>& exclude_peer) final;
 
-      void send_hs_message_warning(const uint32_t sender_peer, const chain::hs_message_warning code);
+      void send_hs_message_warning(uint32_t sender_peer, hs_message_warning code) final;
 
    private:
       void on_accepted_block( const block_state_ptr& blk );
@@ -80,8 +80,8 @@ namespace eosio::hotstuff {
       boost::signals2::scoped_connection _irreversible_block_connection;
 
       qc_chain                _qc_chain;
-      std::function<void(const std::optional<uint32_t>&, const chain::hs_message&)> bcast_hs_message;
-      std::function<void(const uint32_t, const chain::hs_message_warning&)> warn_hs_message;
+      std::function<void(const std::optional<uint32_t>&, const hs_message&)> bcast_hs_message = [](const std::optional<uint32_t>&, const hs_message&){};
+      std::function<void(uint32_t, const hs_message_warning&)> warn_hs_message = [](uint32_t, const hs_message_warning&){};
 
       uint32_t                _quorum_threshold = 15; //FIXME/TODO: calculate from schedule
       fc::logger&             _logger;