diff --git a/libraries/chain/include/eosio/chain/config.hpp b/libraries/chain/include/eosio/chain/config.hpp index c21c62db63..ec88d6fb63 100644 --- a/libraries/chain/include/eosio/chain/config.hpp +++ b/libraries/chain/include/eosio/chain/config.hpp @@ -12,7 +12,7 @@ const static auto reversible_blocks_dir_name = "reversible"; const static auto default_state_dir_name = "state"; const static auto forkdb_filename = "fork_db.dat"; -const static auto qcdb_filename = "qc_db.dat"; +const static auto safetydb_filename = "safety_db.dat"; const static auto default_state_size = 1*1024*1024*1024ll; const static auto default_state_guard_size = 128*1024*1024ll; diff --git a/libraries/chain/include/eosio/chain/hotstuff.hpp b/libraries/chain/include/eosio/chain/hotstuff.hpp index 8d31a4a317..0f765747b0 100644 --- a/libraries/chain/include/eosio/chain/hotstuff.hpp +++ b/libraries/chain/include/eosio/chain/hotstuff.hpp @@ -13,10 +13,33 @@ namespace eosio::chain { using hs_bitset = boost::dynamic_bitset; using bls_key_map_t = std::map; + inline digest_type get_digest_to_sign(const block_id_type& block_id, uint8_t phase_counter, const fc::sha256& final_on_qc) { + digest_type h1 = digest_type::hash( std::make_pair( std::cref(block_id), phase_counter ) ); + digest_type h2 = digest_type::hash( std::make_pair( std::cref(h1), std::cref(final_on_qc) ) ); + return h2; + } + inline uint64_t compute_height(uint32_t block_height, uint32_t phase_counter) { return (uint64_t{block_height} << 32) | phase_counter; } + struct view_number { + view_number() : bheight(0), pcounter(0) {} + explicit view_number(uint32_t block_height, uint8_t phase_counter) : bheight(block_height), pcounter(phase_counter) {} + auto operator<=>(const view_number&) const = default; + friend std::ostream& operator<<(std::ostream& os, const view_number& vn) { + os << "view_number(" << vn.bheight << ", " << vn.pcounter << ")\n"; + } + + uint32_t block_height() const { return bheight; } + uint8_t phase_counter() const { return pcounter; } + uint64_t get_key() const { return compute_height(bheight, pcounter); } + std::string to_string() const { return std::to_string(bheight) + "::" + std::to_string(pcounter); } + + uint32_t bheight; + uint8_t pcounter; + }; + struct extended_schedule { producer_authority_schedule producer_schedule; std::map bls_pub_keys; @@ -42,8 +65,12 @@ namespace eosio::chain { quorum_certificate_message justify; //justification uint8_t phase_counter = 0; + digest_type get_proposal_id() const { return get_digest_to_sign(block_id, phase_counter, final_on_qc); }; + uint32_t block_num() const { return block_header::num_from_id(block_id); } - uint64_t get_height() const { return compute_height(block_header::num_from_id(block_id), phase_counter); }; + uint64_t get_key() const { return compute_height(block_header::num_from_id(block_id), phase_counter); }; + + view_number get_view_number() const { return view_number(block_header::num_from_id(block_id), phase_counter); }; }; struct hs_new_block_message { @@ -72,7 +99,7 @@ namespace eosio::chain { fc::sha256 b_finality_violation; block_id_type block_exec; block_id_type pending_proposal_block; - uint32_t v_height = 0; + eosio::chain::view_number v_height; eosio::chain::quorum_certificate_message high_qc; eosio::chain::quorum_certificate_message current_qc; eosio::chain::extended_schedule schedule; @@ -88,7 +115,8 @@ namespace eosio::chain { } //eosio::chain -// // @ignore quorum_met + +FC_REFLECT(eosio::chain::view_number, (bheight)(pcounter)); FC_REFLECT(eosio::chain::quorum_certificate_message, (proposal_id)(active_finalizers)(active_agg_sig)); FC_REFLECT(eosio::chain::extended_schedule, (producer_schedule)(bls_pub_keys)); FC_REFLECT(eosio::chain::hs_vote_message, (proposal_id)(finalizer_key)(sig)); diff --git a/libraries/hotstuff/chain_pacemaker.cpp b/libraries/hotstuff/chain_pacemaker.cpp index eac294893a..8e58fe97bb 100644 --- a/libraries/hotstuff/chain_pacemaker.cpp +++ b/libraries/hotstuff/chain_pacemaker.cpp @@ -101,12 +101,13 @@ namespace eosio { namespace hotstuff { #endif //=============================================================================================== +#warning TODO get a data directory str passed into the chain_pacemaker ctor and use it to compose the absolute filepathname that is passed to qc_chain ctor chain_pacemaker::chain_pacemaker(controller* chain, std::set my_producers, bls_key_map_t finalizer_keys, fc::logger& logger) : _chain(chain), - _qc_chain("default", this, std::move(my_producers), std::move(finalizer_keys), logger), + _qc_chain("default", this, std::move(my_producers), std::move(finalizer_keys), logger, eosio::chain::config::safetydb_filename), _logger(logger) { _accepted_block_connection = chain->accepted_block.connect( [this]( const block_state_ptr& blk ) { diff --git a/libraries/hotstuff/include/eosio/hotstuff/qc_chain.hpp b/libraries/hotstuff/include/eosio/hotstuff/qc_chain.hpp index 23ba715666..a2dae39a7a 100644 --- a/libraries/hotstuff/include/eosio/hotstuff/qc_chain.hpp +++ b/libraries/hotstuff/include/eosio/hotstuff/qc_chain.hpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -21,6 +22,7 @@ #include +#include #include #include @@ -30,6 +32,52 @@ namespace eosio::hotstuff { + template class state_db_manager { + public: + static constexpr uint64_t magic = 0x0123456789abcdef; + static bool write(fc::cfile& pfile, const StateObjectType& sobj) { + if (!pfile.is_open()) + return false; + pfile.seek(0); + pfile.truncate(); + pfile.write((char*)(&magic), sizeof(magic)); + auto data = fc::raw::pack(sobj); + pfile.write(data.data(), data.size()); + pfile.flush(); + return true; + } + static bool read(const std::string& file_path, StateObjectType& sobj) { + if (!std::filesystem::exists(file_path)) + return false; + fc::cfile pfile; + pfile.set_file_path(file_path); + pfile.open("rb"); + pfile.seek_end(0); + if (pfile.tellp() <= 0) + return false; + pfile.seek(0); + try { + uint64_t read_magic; + pfile.read((char*)(&read_magic), sizeof(read_magic)); + if (read_magic != magic) + return false; + auto datastream = pfile.create_datastream(); + StateObjectType read_sobj; + fc::raw::unpack(datastream, read_sobj); + sobj = std::move(read_sobj); + return true; + } catch (...) { + return false; + } + } + static bool write(const std::string& file_path, const StateObjectType& sobj) { + fc::cfile pfile; + pfile.set_file_path(file_path); + pfile.open(fc::cfile::truncate_rw_mode); + return write(pfile, sobj); + } + }; + using boost::multi_index_container; using namespace boost::multi_index; using namespace eosio::chain; @@ -85,6 +133,7 @@ namespace eosio::hotstuff { bool is_quorum_met() const { return quorum_met; } void set_quorum_met() { quorum_met = true; } + private: friend struct fc::reflector; fc::sha256 proposal_id; @@ -109,7 +158,8 @@ namespace eosio::hotstuff { qc_chain(std::string id, base_pacemaker* pacemaker, std::set my_producers, chain::bls_key_map_t finalizer_keys, - fc::logger& logger); + fc::logger& logger, + std::string safety_state_file); uint64_t get_state_version() const { return _state_version; } // no lock required @@ -132,6 +182,8 @@ namespace eosio::hotstuff { private: + void write_safety_state_file(); + const hs_proposal_message* get_proposal(const fc::sha256& proposal_id); // returns nullptr if not found // returns false if proposal with that same ID already exists at the store of its height @@ -141,9 +193,6 @@ namespace eosio::hotstuff { hs_bitset update_bitset(const hs_bitset& finalizer_set, const fc::crypto::blslib::bls_public_key& finalizer_key); - //get digest to sign from proposal data - digest_type get_digest_to_sign(const block_id_type& block_id, uint8_t phase_counter, const fc::sha256& final_on_qc); - void reset_qc(const fc::sha256& proposal_id); bool evaluate_quorum(const hs_bitset& finalizers, const fc::crypto::blslib::bls_signature& agg_sig, const hs_proposal_message& proposal); //evaluate quorum for a proposal @@ -164,7 +213,7 @@ namespace eosio::hotstuff { void process_new_view(const std::optional& connection_id, const hs_new_view_message& msg); void process_new_block(const std::optional& connection_id, const hs_new_block_message& msg); - hs_vote_message sign_proposal(const hs_proposal_message& proposal, const fc::crypto::blslib::bls_private_key& finalizer_priv_key); + hs_vote_message sign_proposal(const hs_proposal_message& proposal, const fc::crypto::blslib::bls_public_key& finalizer_pub_key, const fc::crypto::blslib::bls_private_key& finalizer_priv_key); //verify that a proposal descends from another bool extends(const fc::sha256& descendant, const fc::sha256& ancestor); @@ -202,20 +251,23 @@ namespace eosio::hotstuff { }; bool _chained_mode = false; + block_id_type _block_exec; block_id_type _pending_proposal_block; + safety_state _safety_state; fc::sha256 _b_leaf; - fc::sha256 _b_lock; fc::sha256 _b_exec; fc::sha256 _b_finality_violation; quorum_certificate _high_qc; quorum_certificate _current_qc; - uint32_t _v_height = 0; base_pacemaker* _pacemaker = nullptr; std::set _my_producers; chain::bls_key_map_t _my_finalizer_keys; std::string _id; + std::string _safety_state_file; // if empty, safety state persistence is turned off + fc::cfile _safety_state_file_handle; + mutable std::atomic _state_version = 1; fc::logger& _logger; @@ -239,11 +291,11 @@ namespace eosio::hotstuff { indexed_by< hashed_unique< tag, - BOOST_MULTI_INDEX_MEMBER(hs_proposal_message,fc::sha256,proposal_id) + BOOST_MULTI_INDEX_MEMBER(hs_proposal_message, fc::sha256,proposal_id) >, ordered_non_unique< tag, - BOOST_MULTI_INDEX_CONST_MEM_FUN(hs_proposal_message,uint64_t,get_height) + BOOST_MULTI_INDEX_CONST_MEM_FUN(hs_proposal_message, uint64_t, get_key) > > > proposal_store_type; diff --git a/libraries/hotstuff/include/eosio/hotstuff/state.hpp b/libraries/hotstuff/include/eosio/hotstuff/state.hpp new file mode 100644 index 0000000000..589f3a2188 --- /dev/null +++ b/libraries/hotstuff/include/eosio/hotstuff/state.hpp @@ -0,0 +1,61 @@ +#include + +#include + +namespace eosio::hotstuff { + + using namespace eosio::chain; + + struct safety_state { + + void set_v_height(const fc::crypto::blslib::bls_public_key& finalizer_key, const eosio::chain::view_number v_height) { + _states[finalizer_key].first = v_height; + } + + void set_b_lock(const fc::crypto::blslib::bls_public_key& finalizer_key, const fc::sha256& b_lock) { + _states[finalizer_key].second = b_lock; + } + + std::pair get_safety_state(const fc::crypto::blslib::bls_public_key& finalizer_key) const { + auto s = _states.find(finalizer_key); + if (s != _states.end()) return s->second; + else return {}; + } + + eosio::chain::view_number get_v_height(const fc::crypto::blslib::bls_public_key& finalizer_key) const { + auto s = _states.find(finalizer_key); + if (s != _states.end()) return s->second.first; + else return {}; + }; + + fc::sha256 get_b_lock(const fc::crypto::blslib::bls_public_key& finalizer_key) const { + auto s_itr = _states.find(finalizer_key); + if (s_itr != _states.end()) return s_itr->second.second; + else return {}; + }; + + //todo : implement safety state default / sorting + + std::pair get_safety_state() const { + auto s = _states.begin(); + if (s != _states.end()) return s->second; + else return {}; + } + + eosio::chain::view_number get_v_height() const { + auto s = _states.begin(); + if (s != _states.end()) return s->second.first; + else return {}; + }; + + fc::sha256 get_b_lock() const { + auto s_itr = _states.begin(); + if (s_itr != _states.end()) return s_itr->second.second; + else return {}; + }; + + std::map> _states; + }; +} + +FC_REFLECT(eosio::hotstuff::safety_state, (_states)) diff --git a/libraries/hotstuff/qc_chain.cpp b/libraries/hotstuff/qc_chain.cpp index 92eda18f96..0a6d3f5e23 100644 --- a/libraries/hotstuff/qc_chain.cpp +++ b/libraries/hotstuff/qc_chain.cpp @@ -4,6 +4,14 @@ namespace eosio::hotstuff { + void qc_chain::write_safety_state_file() { + if (_safety_state_file.empty()) + return; + if (!_safety_state_file_handle.is_open()) + _safety_state_file_handle.open(fc::cfile::create_or_update_rw_mode); + state_db_manager::write(_safety_state_file_handle, _safety_state); + } + const hs_proposal_message* qc_chain::get_proposal(const fc::sha256& proposal_id) { #ifdef QC_CHAIN_SIMPLE_PROPOSAL_STORE if (proposal_id == NULL_PROPOSAL_ID) @@ -31,7 +39,7 @@ namespace eosio::hotstuff { bool qc_chain::insert_proposal(const hs_proposal_message& proposal) { #ifdef QC_CHAIN_SIMPLE_PROPOSAL_STORE - uint64_t proposal_height = proposal.get_height(); + uint64_t proposal_height = proposal.get_key(); ps_height_iterator psh_it = _proposal_stores_by_height.find( proposal_height ); if (psh_it == _proposal_stores_by_height.end()) { _proposal_stores_by_height.emplace( proposal_height, proposal_store() ); @@ -56,12 +64,12 @@ namespace eosio::hotstuff { void qc_chain::get_state(finalizer_state& fs) const { fs.chained_mode = _chained_mode; fs.b_leaf = _b_leaf; - fs.b_lock = _b_lock; + fs.b_lock = _safety_state.get_b_lock(); fs.b_exec = _b_exec; fs.b_finality_violation = _b_finality_violation; fs.block_exec = _block_exec; fs.pending_proposal_block = _pending_proposal_block; - fs.v_height = _v_height; + fs.v_height = _safety_state.get_v_height(); fs.high_qc = _high_qc.to_msg(); fs.current_qc = _current_qc.to_msg(); #ifdef QC_CHAIN_SIMPLE_PROPOSAL_STORE @@ -111,12 +119,6 @@ namespace eosio::hotstuff { throw std::runtime_error("qc_chain internal error: finalizer_key not found"); } - digest_type qc_chain::get_digest_to_sign(const block_id_type& block_id, uint8_t phase_counter, const fc::sha256& final_on_qc){ - digest_type h1 = digest_type::hash( std::make_pair( block_id, phase_counter ) ); - digest_type h2 = digest_type::hash( std::make_pair( h1, final_on_qc ) ); - return h2; - } - std::vector qc_chain::get_qc_chain(const fc::sha256& proposal_id) { std::vector ret_arr; if ( const hs_proposal_message* b2 = get_proposal( proposal_id ) ) { @@ -137,7 +139,7 @@ namespace eosio::hotstuff { b_new.parent_id = _b_leaf; b_new.phase_counter = phase_counter; b_new.justify = _high_qc.to_msg(); //or null if no _high_qc upon activation or chain launch - if (!b_new.justify.proposal_id.empty()) { + if (!b_new.justify.proposal_id.empty()){ std::vector current_qc_chain = get_qc_chain(b_new.justify.proposal_id); size_t chain_length = std::distance(current_qc_chain.begin(), current_qc_chain.end()); if (chain_length>=2){ @@ -195,7 +197,8 @@ namespace eosio::hotstuff { keys.push_back(c_finalizers[i].public_key); fc::crypto::blslib::bls_public_key agg_key = fc::crypto::blslib::aggregate(keys); - digest_type digest = get_digest_to_sign(proposal.block_id, proposal.phase_counter, proposal.final_on_qc); + digest_type digest = proposal.get_proposal_id(); //get_digest_to_sign(proposal.block_id, proposal.phase_counter, proposal.final_on_qc); + std::vector h = std::vector(digest.data(), digest.data() + 32); bool ok = fc::crypto::blslib::verify(agg_key, h, agg_sig); return ok; @@ -218,13 +221,22 @@ namespace eosio::hotstuff { base_pacemaker* pacemaker, std::set my_producers, bls_key_map_t finalizer_keys, - fc::logger& logger) + fc::logger& logger, + std::string safety_state_file) : _pacemaker(pacemaker), _my_producers(std::move(my_producers)), _my_finalizer_keys(std::move(finalizer_keys)), _id(std::move(id)), + _safety_state_file(safety_state_file), _logger(logger) { + //todo : read liveness state / select initialization heuristics ? + + if (!_safety_state_file.empty()) { + _safety_state_file_handle.set_file_path(safety_state_file); + state_db_manager::read(_safety_state_file, _safety_state); + } + _high_qc.reset({}, 21); // TODO: use active schedule size _current_qc.reset({}, 21); // TODO: use active schedule size @@ -253,10 +265,13 @@ namespace eosio::hotstuff { } - hs_vote_message qc_chain::sign_proposal(const hs_proposal_message& proposal, const fc::crypto::blslib::bls_private_key& finalizer_priv_key){ - _v_height = proposal.get_height(); + hs_vote_message qc_chain::sign_proposal(const hs_proposal_message& proposal, + const fc::crypto::blslib::bls_public_key& finalizer_pub_key, + const fc::crypto::blslib::bls_private_key& finalizer_priv_key) + { + _safety_state.set_v_height(finalizer_pub_key, proposal.get_view_number()); - digest_type digest = get_digest_to_sign(proposal.block_id, proposal.phase_counter, proposal.final_on_qc); + digest_type digest = proposal.get_proposal_id(); //get_digest_to_sign(proposal.block_id, proposal.phase_counter, proposal.final_on_qc); std::vector h = std::vector(digest.data(), digest.data() + 32); @@ -297,7 +312,7 @@ namespace eosio::hotstuff { } #ifdef QC_CHAIN_SIMPLE_PROPOSAL_STORE - ps_height_iterator psh_it = _proposal_stores_by_height.find( proposal.get_height() ); + ps_height_iterator psh_it = _proposal_stores_by_height.find( proposal.get_key() ); if (psh_it != _proposal_stores_by_height.end()) { proposal_store & pstore = psh_it->second; @@ -307,8 +322,8 @@ namespace eosio::hotstuff { hs_proposal_message & existing_proposal = ps_it->second; #else //height is not necessarily unique, so we iterate over all prior proposals at this height - auto hgt_itr = _proposal_store.get().lower_bound( proposal.get_height() ); - auto end_itr = _proposal_store.get().upper_bound( proposal.get_height() ); + auto hgt_itr = _proposal_store.get().lower_bound( proposal.get_key() ); + auto end_itr = _proposal_store.get().upper_bound( proposal.get_key() ); while (hgt_itr != end_itr) { const hs_proposal_message & existing_proposal = *hgt_itr; @@ -360,7 +375,7 @@ namespace eosio::hotstuff { if (mfk_itr!=_my_finalizer_keys.end()) { - hs_vote_message v_msg = sign_proposal(proposal, mfk_itr->second); + hs_vote_message v_msg = sign_proposal(proposal, mfk_itr->first, mfk_itr->second); fc_tlog(_logger, " === ${id} signed proposal : block_num ${block_num} phase ${phase_counter} : proposal_id ${proposal_id}", ("id", _id) @@ -382,6 +397,8 @@ namespace eosio::hotstuff { //update internal state update(proposal); + write_safety_state_file(); + //propagate this proposal since it was new to us send_hs_proposal_msg(connection_id, proposal); @@ -424,7 +441,7 @@ namespace eosio::hotstuff { seen_votes_store_type::nth_index<0>::type::iterator itr = _seen_votes_store.get().find( p->proposal_id ); bool propagate = false; if (itr == _seen_votes_store.get().end()) { - seen_votes sv = { p->proposal_id, p->get_height(), { vote.finalizer_key } }; + seen_votes sv = { p->proposal_id, p->get_key(), { vote.finalizer_key } }; _seen_votes_store.insert(sv); propagate = true; } else { @@ -497,10 +514,15 @@ namespace eosio::hotstuff { reset_qc(proposal_candidate.proposal_id); fc_tlog(_logger, " === ${id} setting _pending_proposal_block to null (process_vote)", ("id", _id)); + _pending_proposal_block = {}; _b_leaf = proposal_candidate.proposal_id; + //todo : asynchronous? + //write_state(_liveness_state_file , _liveness_state); + send_hs_proposal_msg( std::nullopt, proposal_candidate ); + fc_tlog(_logger, " === ${id} _b_leaf updated (process_vote): ${proposal_id}", ("proposal_id", proposal_candidate.proposal_id)("id", _id)); } } @@ -573,6 +595,9 @@ namespace eosio::hotstuff { _pending_proposal_block = {}; _b_leaf = proposal_candidate.proposal_id; + //todo : asynchronous? + //write_state(_liveness_state_file , _liveness_state); + send_hs_proposal_msg( std::nullopt, proposal_candidate ); fc_tlog(_logger, " === ${id} _b_leaf updated (on_beat): ${proposal_id}", ("proposal_id", proposal_candidate.proposal_id)("id", _id)); @@ -641,7 +666,6 @@ namespace eosio::hotstuff { } // Invoked when we could perhaps make a proposal to the network (or to ourselves, if we are the leader). - // Called from the main application thread void qc_chain::on_beat(){ // Non-proposing leaders do not care about on_beat(), because leaders react to a block proposal @@ -687,10 +711,14 @@ namespace eosio::hotstuff { // if new high QC is higher than current, update to new - if (_high_qc.get_proposal_id().empty()) { + if (_high_qc.get_proposal_id().empty()){ + _high_qc = high_qc; _b_leaf = _high_qc.get_proposal_id(); + //todo : asynchronous? + //write_state(_liveness_state_file , _liveness_state); + fc_tlog(_logger, " === ${id} _b_leaf updated (update_high_qc) : ${proposal_id}", ("proposal_id", _high_qc.get_proposal_id())("id", _id)); // avoid looping message propagation when receiving a new-view message with a high_qc.get_proposal_id().empty(). @@ -701,23 +729,25 @@ namespace eosio::hotstuff { } else { const hs_proposal_message *old_high_qc_prop = get_proposal( _high_qc.get_proposal_id() ); const hs_proposal_message *new_high_qc_prop = get_proposal( high_qc.get_proposal_id() ); + if (old_high_qc_prop == nullptr) return false; if (new_high_qc_prop == nullptr) return false; - if (new_high_qc_prop->get_height() > old_high_qc_prop->get_height() + if (new_high_qc_prop->get_view_number() > old_high_qc_prop->get_view_number() && is_quorum_met(high_qc, *new_high_qc_prop)) { - // "The caller does not need this updated on their high_qc structure" -- g - //high_qc.quorum_met = true; - - fc_tlog(_logger, " === updated high qc, now is : #${get_height} ${proposal_id}", ("get_height", new_high_qc_prop->get_height())("proposal_id", new_high_qc_prop->proposal_id)); + fc_tlog(_logger, " === updated high qc, now is : #${view_number} ${proposal_id}", ("view_number", new_high_qc_prop->get_view_number())("proposal_id", new_high_qc_prop->proposal_id)); _high_qc = high_qc; _high_qc.set_quorum_met(); _b_leaf = _high_qc.get_proposal_id(); + //todo : asynchronous? + //write_state(_liveness_state_file , _liveness_state); + fc_tlog(_logger, " === ${id} _b_leaf updated (update_high_qc) : ${proposal_id}", ("proposal_id", _high_qc.get_proposal_id())("id", _id)); + return true; } } @@ -765,7 +795,8 @@ namespace eosio::hotstuff { fc::sha256 upcoming_commit; - if (proposal.justify.proposal_id.empty() && _b_lock.empty()) { + if (proposal.justify.proposal_id.empty() && _safety_state.get_b_lock().empty()) { + final_on_qc_check = true; //if chain just launched or feature just activated } else { @@ -800,27 +831,29 @@ namespace eosio::hotstuff { } } - if (proposal.get_height() > _v_height) { + if (proposal.get_view_number() > _safety_state.get_v_height()) { monotony_check = true; } - if (!_b_lock.empty()) { + if (!_safety_state.get_b_lock().empty()){ //Safety check : check if this proposal extends the chain I'm locked on - if (extends(proposal.proposal_id, _b_lock)) { + if (extends(proposal.proposal_id, _safety_state.get_b_lock())) { safety_check = true; } //Liveness check : check if the height of this proposal's justification is higher than the height of the proposal I'm locked on. This allows restoration of liveness if a replica is locked on a stale block. - if (proposal.justify.proposal_id.empty() && _b_lock.empty()) { + + if (proposal.justify.proposal_id.empty() && _safety_state.get_b_lock().empty()) { + liveness_check = true; //if there is no justification on the proposal and I am not locked on anything, means the chain just launched or feature just activated } else { - const hs_proposal_message *b_lock = get_proposal( _b_lock ); - EOS_ASSERT( b_lock != nullptr , chain_exception, "expected hs_proposal ${id} not found", ("id", _b_lock) ); + const hs_proposal_message *b_lock = get_proposal( _safety_state.get_b_lock() ); + EOS_ASSERT( b_lock != nullptr , chain_exception, "expected hs_proposal ${id} not found", ("id", _safety_state.get_b_lock()) ); const hs_proposal_message *prop_justification = get_proposal( proposal.justify.proposal_id ); EOS_ASSERT( prop_justification != nullptr , chain_exception, "expected hs_proposal ${id} not found", ("id", proposal.justify.proposal_id) ); - if (prop_justification->get_height() > b_lock->get_height()) { + if (prop_justification->get_view_number() > b_lock->get_view_number()) { liveness_check = true; } } @@ -884,8 +917,8 @@ namespace eosio::hotstuff { size_t chain_length = std::distance(current_qc_chain.begin(), current_qc_chain.end()); - const hs_proposal_message *b_lock = get_proposal( _b_lock ); - EOS_ASSERT( b_lock != nullptr || _b_lock.empty(), chain_exception, "expected hs_proposal ${id} not found", ("id", _b_lock) ); + const hs_proposal_message *b_lock = get_proposal( _safety_state.get_b_lock() ); + EOS_ASSERT( b_lock != nullptr || _safety_state.get_b_lock().empty() , chain_exception, "expected hs_proposal ${id} not found", ("id", _safety_state.get_b_lock()) ); //fc_tlog(_logger, " === update_high_qc : proposal.justify ==="); update_high_qc(quorum_certificate{proposal.justify, 21}); // TODO: use active schedule size @@ -911,7 +944,7 @@ namespace eosio::hotstuff { fc_tlog(_logger, " === ${id} _b_lock ${_b_lock} b_1 height ${b_1_height}", ("id", _id) - ("_b_lock", _b_lock) + ("_b_lock", _safety_state.get_b_lock()) ("b_1_height", b_1.block_num()) ("b_1_phase", b_1.phase_counter)); @@ -921,9 +954,13 @@ namespace eosio::hotstuff { ("b_lock_phase", b_lock->phase_counter)); } - if (_b_lock.empty() || b_1.get_height() > b_lock->get_height()) { + if (_safety_state.get_b_lock().empty() || b_1.get_view_number() > b_lock->get_view_number()){ + fc_tlog(_logger, "setting _b_lock to ${proposal_id}", ("proposal_id",b_1.proposal_id )); - _b_lock = b_1.proposal_id; //commit phase on b1 + + for (const auto& f_itr : _my_finalizer_keys) { + _safety_state.set_b_lock(f_itr.first, b_1.proposal_id); //commit phase on b1 + } fc_tlog(_logger, " === ${id} _b_lock updated : ${proposal_id}", ("proposal_id", b_1.proposal_id)("id", _id)); } @@ -946,12 +983,12 @@ namespace eosio::hotstuff { //direct parent relationship verification if (b_2.parent_id == b_1.proposal_id && b_1.parent_id == b.proposal_id){ - if (!_b_exec.empty()) { + if (!_b_exec.empty()){ const hs_proposal_message *b_exec = get_proposal( _b_exec ); EOS_ASSERT( b_exec != nullptr , chain_exception, "expected hs_proposal ${id} not found", ("id", _b_exec) ); - if (b_exec->get_height() >= b.get_height() && b_exec->proposal_id != b.proposal_id){ + if (b_exec->get_view_number() >= b.get_view_number() && b_exec->proposal_id != b.proposal_id){ fc_elog(_logger, " *** ${id} finality violation detected at height ${block_num}, phase : ${phase}. Proposal ${proposal_id_1} conflicts with ${proposal_id_2}", ("id", _id) @@ -967,14 +1004,17 @@ namespace eosio::hotstuff { } } - commit(b); + commit(b); //todo : ensure that block is marked irreversible / lib is updated etc. + + //todo : asynchronous? + //write_state(_liveness_state_file , _liveness_state); fc_tlog(_logger, " === last executed proposal : #${block_num} ${block_id}", ("block_num", b.block_num())("block_id", b.block_id)); _b_exec = b.proposal_id; //decide phase on b _block_exec = b.block_id; - gc_proposals( b.get_height()-1); + gc_proposals( b.get_key()-1); } else { fc_elog(_logger, " *** ${id} could not verify direct parent relationship", ("id",_id)); @@ -1003,7 +1043,7 @@ namespace eosio::hotstuff { ph_iterator ph_it = _proposal_height.find( p.proposal_id ); EOS_ASSERT( ph_it != _proposal_height.end(), chain_exception, "gc_proposals internal error: no proposal height entry"); uint64_t proposal_height = ph_it->second; - EOS_ASSERT(proposal_height == p.get_height(), chain_exception, "gc_proposals internal error: mismatched proposal height record"); // this check is unnecessary + EOS_ASSERT(proposal_height == p.get_key(), chain_exception, "gc_proposals internal error: mismatched proposal height record"); // this check is unnecessary _proposal_height.erase( ph_it ); ++ps_it; } @@ -1056,7 +1096,8 @@ void qc_chain::commit(const hs_proposal_message& initial_proposal) { ("prop_id_2", p->block_num())("phase_2", p->phase_counter)); } - bool exec_height_check = _b_exec.empty() || last_exec_prop->get_height() < p->get_height(); + + bool exec_height_check = _b_exec.empty() || last_exec_prop->get_view_number() < p->get_view_number(); if (exec_height_check) { proposal_chain.push_back(p); // add proposal to vector for further processing p = get_proposal(p->parent_id); // process parent if non-null diff --git a/libraries/hotstuff/test/CMakeLists.txt b/libraries/hotstuff/test/CMakeLists.txt index b147f10496..eda3efb0c5 100644 --- a/libraries/hotstuff/test/CMakeLists.txt +++ b/libraries/hotstuff/test/CMakeLists.txt @@ -1,4 +1,4 @@ -add_executable( test_hotstuff test_hotstuff.cpp test_pacemaker.cpp) +add_executable( test_hotstuff test_hotstuff.cpp test_hotstuff_state.cpp hotstuff_tools.cpp test_pacemaker.cpp) target_link_libraries( test_hotstuff hotstuff fc Boost::unit_test_framework) add_test(NAME test_hotstuff COMMAND test_hotstuff WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) diff --git a/libraries/hotstuff/test/hotstuff_tools.cpp b/libraries/hotstuff/test/hotstuff_tools.cpp new file mode 100644 index 0000000000..e2d3e994c9 --- /dev/null +++ b/libraries/hotstuff/test/hotstuff_tools.cpp @@ -0,0 +1,56 @@ +#include +#include +#include + +#include + +#include + +BOOST_AUTO_TEST_CASE(view_number_tests) try { + + eosio::hotstuff::hs_proposal_message hspm_1; + eosio::hotstuff::hs_proposal_message hspm_2; + eosio::hotstuff::hs_proposal_message hspm_3; + eosio::hotstuff::hs_proposal_message hspm_4; + eosio::hotstuff::hs_proposal_message hspm_5; + + hspm_1.block_id = eosio::chain::block_id_type("0b93846ba73bdfdc9b2383863b64f8f921c8a2379d6dde4e05bdd2e434e9392a"); //UX Network block #194217067 + hspm_1.phase_counter = 0; + + hspm_2.block_id = eosio::chain::block_id_type("0b93846ba73bdfdc9b2383863b64f8f921c8a2379d6dde4e05bdd2e434e9392a"); //UX Network block #194217067 + hspm_2.phase_counter = 1; + + hspm_3.block_id = eosio::chain::block_id_type("0b93846cf55a3ecbcd8f9bd86866b1aecc2e8bd981e40c92609ce3a68dbd0824"); //UX Network block #194217068 + hspm_3.phase_counter = 0; + + hspm_4.block_id = eosio::chain::block_id_type("0b93846cf55a3ecbcd8f9bd86866b1aecc2e8bd981e40c92609ce3a68dbd0824"); //UX Network block #194217068 + hspm_4.phase_counter = 1; + + hspm_5.block_id = eosio::chain::block_id_type("0b93846cf55a3ecbcd8f9bd86866b1aecc2e8bd981e40c92609ce3a68dbd0824"); //UX Network block #194217068 + hspm_5.phase_counter = 2; + + eosio::hotstuff::view_number vn_1 = hspm_1.get_view_number(); + eosio::hotstuff::view_number vn_2 = hspm_2.get_view_number(); + eosio::hotstuff::view_number vn_3 = hspm_3.get_view_number(); + eosio::hotstuff::view_number vn_4 = hspm_4.get_view_number(); + eosio::hotstuff::view_number vn_5 = hspm_5.get_view_number(); + + //test getters + BOOST_CHECK_EQUAL(vn_1.block_height(), 194217067); + BOOST_CHECK_EQUAL(vn_1.phase_counter(), 0); + + BOOST_CHECK_NE(vn_1, vn_2); + BOOST_CHECK_LT(vn_1, vn_2); + BOOST_CHECK_LT(vn_2, vn_3); + BOOST_CHECK_LT(vn_3, vn_4); + BOOST_CHECK_LT(vn_4, vn_5); + BOOST_CHECK_LE(vn_4, vn_5); + BOOST_CHECK_LE(vn_2, vn_3); + +//test constructor + + eosio::hotstuff::view_number vn_6 = eosio::hotstuff::view_number(194217068, 2); + + BOOST_CHECK_EQUAL(vn_5, vn_6); + +} FC_LOG_AND_RETHROW(); diff --git a/libraries/hotstuff/test/test_hotstuff.cpp b/libraries/hotstuff/test/test_hotstuff.cpp index ed1d72c958..135dce76fd 100644 --- a/libraries/hotstuff/test/test_hotstuff.cpp +++ b/libraries/hotstuff/test/test_hotstuff.cpp @@ -76,12 +76,11 @@ class hotstuff_test_handler { for (size_t i = 0 ; i < replicas.size() ; i++){ fc::crypto::blslib::bls_private_key sk = fc::crypto::blslib::bls_private_key(replica_keys[i]); bls_key_map_t keys{{sk.get_public_key(), sk}}; - qc_chain *qcc_ptr = new qc_chain(replica_keys[i].to_string(), &tpm, {replicas[i]}, keys, hotstuff_logger); + qc_chain *qcc_ptr = new qc_chain(replica_keys[i].to_string(), &tpm, {replicas[i]}, keys, hotstuff_logger, std::string()); std::shared_ptr qcc_shared_ptr(qcc_ptr); _qc_chains.push_back( std::make_pair(replicas[i], qcc_shared_ptr) ); tpm.register_qc_chain(replicas[i], qcc_shared_ptr ); } - } void print_msgs(std::vector msgs ){ @@ -179,7 +178,6 @@ BOOST_AUTO_TEST_CASE(hotstuff_bitset) try { } FC_LOG_AND_RETHROW(); - static std::vector map_to_sks(std::vector keys){ std::vector sks; std::transform(keys.cbegin(), keys.cend(), std::back_inserter(sks), diff --git a/libraries/hotstuff/test/test_hotstuff_state.cpp b/libraries/hotstuff/test/test_hotstuff_state.cpp new file mode 100644 index 0000000000..97c2a3cb4e --- /dev/null +++ b/libraries/hotstuff/test/test_hotstuff_state.cpp @@ -0,0 +1,152 @@ +#include + +#include +#include + +#include + +#include +#include + +#include + +using std::cout; + +BOOST_AUTO_TEST_SUITE(test_hotstuff_state) + +const std::string file_path_1("temp_hs_safety"); +//const std::string file_path_2("temp_hs_liveness"); + +BOOST_AUTO_TEST_CASE(write_safety_state_to_file) try { + + eosio::hotstuff::hs_proposal_message hspm_1; + eosio::hotstuff::hs_proposal_message hspm_2; + + hspm_1.block_id = eosio::chain::block_id_type("0b93846cf55a3ecbcd8f9bd86866b1aecc2e8bd981e40c92609ce3a68dbd0824"); //UX Network block #194217067 + hspm_1.final_on_qc = eosio::chain::block_id_type(); + hspm_1.phase_counter = 2; + + eosio::hotstuff::view_number v_height = hspm_1.get_view_number(); + + hspm_2.block_id = eosio::chain::block_id_type("0b93846ba73bdfdc9b2383863b64f8f921c8a2379d6dde4e05bdd2e434e9392a"); //UX Network block #194217067 + hspm_2.final_on_qc = eosio::chain::block_id_type(); + hspm_2.phase_counter = 0; + + fc::sha256 b_lock = eosio::hotstuff::get_digest_to_sign(hspm_2.block_id, hspm_2.phase_counter, hspm_2.final_on_qc); + + eosio::hotstuff::safety_state ss; + + ss.set_v_height(fc::crypto::blslib::bls_public_key{}, v_height); + ss.set_b_lock(fc::crypto::blslib::bls_public_key{}, b_lock); + + BOOST_CHECK( eosio::hotstuff::state_db_manager::write(file_path_1, ss) ); + + //fc::cfile pfile; + //pfile.set_file_path(file_path_1); + //pfile.open(fc::cfile::truncate_rw_mode); + //pfile.write("force garbage to fail read_safety_state_from_file", 20); + //pfile.close(); + +} FC_LOG_AND_RETHROW(); + +BOOST_AUTO_TEST_CASE(read_safety_state_from_file) try { + + eosio::hotstuff::safety_state ss; + + BOOST_CHECK( eosio::hotstuff::state_db_manager::read(file_path_1, ss) ); + + std::remove(file_path_1.c_str()); + + //test correct values + eosio::hotstuff::hs_proposal_message hspm_1; + eosio::hotstuff::hs_proposal_message hspm_2; + + hspm_1.block_id = eosio::chain::block_id_type("0b93846cf55a3ecbcd8f9bd86866b1aecc2e8bd981e40c92609ce3a68dbd0824"); //UX Network block #194217067 + hspm_1.final_on_qc = eosio::chain::block_id_type(); + hspm_1.phase_counter = 2; + + eosio::hotstuff::view_number v_height = hspm_1.get_view_number(); + + hspm_2.block_id = eosio::chain::block_id_type("0b93846ba73bdfdc9b2383863b64f8f921c8a2379d6dde4e05bdd2e434e9392a"); //UX Network block #194217067 + hspm_2.final_on_qc = eosio::chain::block_id_type(); + hspm_2.phase_counter = 0; + + fc::sha256 b_lock = eosio::hotstuff::get_digest_to_sign(hspm_2.block_id, hspm_2.phase_counter, hspm_2.final_on_qc); + + //std::pair ss = get_safety_state(eosio::chain::name{""}); + + BOOST_CHECK_EQUAL(ss.get_v_height(fc::crypto::blslib::bls_public_key{}), v_height); + BOOST_CHECK_EQUAL(ss.get_b_lock(fc::crypto::blslib::bls_public_key{}), b_lock); + +} FC_LOG_AND_RETHROW(); + +#warning TODO decide on liveness state file then implement it in qc_chain and then test it here +/*BOOST_AUTO_TEST_CASE(write_liveness_state_to_file) try { + + eosio::hotstuff::hs_proposal_message hspm_1; + eosio::hotstuff::hs_proposal_message hspm_2; + + hspm_1.block_id = eosio::chain::block_id_type("0b93846cf55a3ecbcd8f9bd86866b1aecc2e8bd981e40c92609ce3a68dbd0824"); //UX Network block #194217068 + hspm_1.final_on_qc = eosio::chain::block_id_type(); + hspm_1.phase_counter = 2; + + fc::sha256 b_exec = eosio::hotstuff::get_digest_to_sign(hspm_1.block_id, hspm_1.phase_counter, hspm_1.final_on_qc); + + hspm_2.block_id = eosio::chain::block_id_type("0b93846ba73bdfdc9b2383863b64f8f921c8a2379d6dde4e05bdd2e434e9392a"); //UX Network block #194217067 + hspm_2.final_on_qc = eosio::chain::block_id_type(); + hspm_2.phase_counter = 1; + + fc::sha256 b_leaf = eosio::hotstuff::get_digest_to_sign(hspm_2.block_id, hspm_2.phase_counter, hspm_2.final_on_qc); + + //mock quorum_certificate + eosio::hotstuff::quorum_certificate high_qc; + + high_qc.proposal_id = fc::sha256("0b93846cf55a3ecbcd8f9bd86866b1aecc2e8bd981e40c92609ce3a68dbd0824"); + high_qc.active_finalizers = 1245; + high_qc.active_agg_sig = fc::crypto::blslib::bls_signature("SIG_BLS_23PuSu1B72cPe6wxGkKjAaaZqA1Ph79zSoW7omsKKUrnprbA3cJCJVhT48QKUG6ofjYTTg4BA4TrVENWyrxjTomwLX6TGdVg2RYhKH7Kk9X23K5ohuhKQcWQ6AwJJGVSbSp4"); + + eosio::hotstuff::liveness_state ls(high_qc, b_leaf, b_exec); + + eosio::hotstuff::write_state(file_path_2, ls); + +} FC_LOG_AND_RETHROW(); + +BOOST_AUTO_TEST_CASE(read_liveness_state_from_file) try { + + eosio::hotstuff::liveness_state ls; + + eosio::hotstuff::read_state(file_path_2, ls); + + std::remove(file_path_2.c_str()); + + //test correct values + + eosio::hotstuff::hs_proposal_message hspm_1; + eosio::hotstuff::hs_proposal_message hspm_2; + + hspm_1.block_id = eosio::chain::block_id_type("0b93846cf55a3ecbcd8f9bd86866b1aecc2e8bd981e40c92609ce3a68dbd0824"); //UX Network block #194217067 + hspm_1.final_on_qc = eosio::chain::block_id_type(); + hspm_1.phase_counter = 2; + + fc::sha256 b_exec = eosio::hotstuff::get_digest_to_sign(hspm_1.block_id, hspm_1.phase_counter, hspm_1.final_on_qc); + + hspm_2.block_id = eosio::chain::block_id_type("0b93846ba73bdfdc9b2383863b64f8f921c8a2379d6dde4e05bdd2e434e9392a"); //UX Network block #194217067 + hspm_2.final_on_qc = eosio::chain::block_id_type(); + hspm_2.phase_counter = 1; + + fc::sha256 b_leaf = eosio::hotstuff::get_digest_to_sign(hspm_2.block_id, hspm_2.phase_counter, hspm_2.final_on_qc); + + //mock quorum_certificate + eosio::hotstuff::quorum_certificate high_qc; + + high_qc.proposal_id = fc::sha256("0b93846cf55a3ecbcd8f9bd86866b1aecc2e8bd981e40c92609ce3a68dbd0824"); + high_qc.active_finalizers = 1245; + high_qc.active_agg_sig = fc::crypto::blslib::bls_signature("SIG_BLS_23PuSu1B72cPe6wxGkKjAaaZqA1Ph79zSoW7omsKKUrnprbA3cJCJVhT48QKUG6ofjYTTg4BA4TrVENWyrxjTomwLX6TGdVg2RYhKH7Kk9X23K5ohuhKQcWQ6AwJJGVSbSp4"); + + BOOST_CHECK(ls.high_qc == high_qc); + BOOST_CHECK(ls.b_exec == b_exec); + BOOST_CHECK(ls.b_leaf == b_leaf); + +} FC_LOG_AND_RETHROW();*/ + +BOOST_AUTO_TEST_SUITE_END() diff --git a/libraries/libfc/include/fc/io/cfile.hpp b/libraries/libfc/include/fc/io/cfile.hpp index 7e79b59942..f3cd05ded1 100644 --- a/libraries/libfc/include/fc/io/cfile.hpp +++ b/libraries/libfc/include/fc/io/cfile.hpp @@ -150,6 +150,14 @@ class cfile { } } + void truncate() { + const int fd = fileno(); + if( -1 == ftruncate(fd, 0) ) { + throw std::ios_base::failure( "cfile: " + _file_path.generic_string() + + " unable to truncate file, error: " + std::to_string( errno ) ); + } + } + void flush() { if( 0 != fflush( _file.get() ) ) { int err = ferror( _file.get() ); diff --git a/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp b/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp index 44b9f282c1..155bf00c65 100644 --- a/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp +++ b/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp @@ -851,7 +851,7 @@ class read_only : public api_base { justify = p.justify; phase_counter = p.phase_counter; block_height = p.block_num(); - view_number = p.get_height(); + view_number = p.get_key(); } hs_complete_proposal_message() = default; // cleos requires this }; @@ -865,7 +865,7 @@ class read_only : public api_base { fc::sha256 b_finality_violation; chain::block_id_type block_exec; chain::block_id_type pending_proposal_block; - uint32_t v_height = 0; + chain::view_number v_height; chain::quorum_certificate_message high_qc; chain::quorum_certificate_message current_qc; chain::extended_schedule schedule;