Merge pull request #1578 from AntelopeIO/persist_hs_safety_liveness_s…

…tate_WIP

IF: Persist hotstuff safety state

libraries/chain/include/eosio/chain/config.hpp

@@ -12,7 +12,7 @@ const static auto reversible_blocks_dir_name = "reversible";
 
 const static auto default_state_dir_name     = "state";
 const static auto forkdb_filename            = "fork_db.dat";
-const static auto qcdb_filename              = "qc_db.dat";
+const static auto safetydb_filename          = "safety_db.dat";
 const static auto default_state_size            = 1*1024*1024*1024ll;
 const static auto default_state_guard_size      =    128*1024*1024ll;
 

libraries/chain/include/eosio/chain/hotstuff.hpp

@@ -13,10 +13,33 @@ namespace eosio::chain {
    using hs_bitset = boost::dynamic_bitset<uint8_t>;
    using bls_key_map_t = std::map<fc::crypto::blslib::bls_public_key, fc::crypto::blslib::bls_private_key>;
 
+   inline digest_type get_digest_to_sign(const block_id_type& block_id, uint8_t phase_counter, const fc::sha256& final_on_qc) {
+      digest_type h1 = digest_type::hash( std::make_pair( std::cref(block_id), phase_counter ) );
+      digest_type h2 = digest_type::hash( std::make_pair( std::cref(h1), std::cref(final_on_qc) ) );
+      return h2;
+   }
+
    inline uint64_t compute_height(uint32_t block_height, uint32_t phase_counter) {
       return (uint64_t{block_height} << 32) | phase_counter;
    }
 
+   struct view_number {
+      view_number() : bheight(0), pcounter(0) {}
+      explicit view_number(uint32_t block_height, uint8_t phase_counter) : bheight(block_height), pcounter(phase_counter) {}
+      auto operator<=>(const view_number&) const = default;
+      friend std::ostream& operator<<(std::ostream& os, const view_number& vn) {
+         os << "view_number(" << vn.bheight << ", " << vn.pcounter << ")\n";
+      }
+
+      uint32_t block_height() const { return bheight; }
+      uint8_t phase_counter() const { return pcounter; }
+      uint64_t get_key() const { return compute_height(bheight, pcounter); }
+      std::string to_string() const { return std::to_string(bheight) + "::" + std::to_string(pcounter); }
+
+      uint32_t bheight;
+      uint8_t pcounter;
+   };
+
    struct extended_schedule {
       producer_authority_schedule                          producer_schedule;
       std::map<name, fc::crypto::blslib::bls_public_key>   bls_pub_keys;
@@ -42,8 +65,12 @@ namespace eosio::chain {
       quorum_certificate_message          justify; //justification
       uint8_t                             phase_counter = 0;
 
+      digest_type get_proposal_id() const { return get_digest_to_sign(block_id, phase_counter, final_on_qc); };
+
       uint32_t block_num() const { return block_header::num_from_id(block_id); }
-      uint64_t get_height() const { return compute_height(block_header::num_from_id(block_id), phase_counter); };
+      uint64_t get_key() const { return compute_height(block_header::num_from_id(block_id), phase_counter); };
+
+      view_number get_view_number() const { return view_number(block_header::num_from_id(block_id), phase_counter); };
    };
 
    struct hs_new_block_message {
@@ -72,7 +99,7 @@ namespace eosio::chain {
       fc::sha256 b_finality_violation;
       block_id_type block_exec;
       block_id_type pending_proposal_block;
-      uint32_t v_height = 0;
+      eosio::chain::view_number v_height;
       eosio::chain::quorum_certificate_message high_qc;
       eosio::chain::quorum_certificate_message current_qc;
       eosio::chain::extended_schedule schedule;
@@ -88,7 +115,8 @@ namespace eosio::chain {
 
 } //eosio::chain
 
-// // @ignore quorum_met
+
+FC_REFLECT(eosio::chain::view_number, (bheight)(pcounter));
 FC_REFLECT(eosio::chain::quorum_certificate_message, (proposal_id)(active_finalizers)(active_agg_sig));
 FC_REFLECT(eosio::chain::extended_schedule, (producer_schedule)(bls_pub_keys));
 FC_REFLECT(eosio::chain::hs_vote_message, (proposal_id)(finalizer_key)(sig));

libraries/hotstuff/chain_pacemaker.cpp

@@ -101,12 +101,13 @@ namespace eosio { namespace hotstuff {
 #endif
 //===============================================================================================
 
+#warning TODO get a data directory str passed into the chain_pacemaker ctor and use it to compose the absolute filepathname that is passed to qc_chain ctor
    chain_pacemaker::chain_pacemaker(controller* chain,
                                     std::set<account_name> my_producers,
                                     bls_key_map_t finalizer_keys,
                                     fc::logger& logger)
       : _chain(chain),
-        _qc_chain("default", this, std::move(my_producers), std::move(finalizer_keys), logger),
+        _qc_chain("default", this, std::move(my_producers), std::move(finalizer_keys), logger, eosio::chain::config::safetydb_filename),
         _logger(logger)
    {
       _accepted_block_connection = chain->accepted_block.connect( [this]( const block_state_ptr& blk ) {

libraries/hotstuff/include/eosio/hotstuff/qc_chain.hpp

@@ -4,6 +4,7 @@
 #include <eosio/chain/controller.hpp>
 #include <eosio/chain/block_state.hpp>
 #include <eosio/hotstuff/base_pacemaker.hpp>
+#include <eosio/hotstuff/state.hpp>
 
 #include <eosio/chain/finalizer_set.hpp>
 #include <eosio/chain/finalizer_authority.hpp>
@@ -21,6 +22,7 @@
 
 #include <boost/dynamic_bitset.hpp>
 
+#include <fc/io/cfile.hpp>
 
 #include <exception>
 #include <stdexcept>
@@ -30,6 +32,52 @@
 
 namespace eosio::hotstuff {
 
+   template<typename StateObjectType> class state_db_manager {
+   public:
+      static constexpr uint64_t magic = 0x0123456789abcdef;
+      static bool write(fc::cfile& pfile, const StateObjectType& sobj) {
+         if (!pfile.is_open())
+            return false;
+         pfile.seek(0);
+         pfile.truncate();
+         pfile.write((char*)(&magic), sizeof(magic));
+         auto data = fc::raw::pack(sobj);
+         pfile.write(data.data(), data.size());
+         pfile.flush();
+         return true;
+      }
+      static bool read(const std::string& file_path, StateObjectType& sobj) {
+         if (!std::filesystem::exists(file_path))
+            return false;
+         fc::cfile pfile;
+         pfile.set_file_path(file_path);
+         pfile.open("rb");
+         pfile.seek_end(0);
+         if (pfile.tellp() <= 0)
+            return false;
+         pfile.seek(0);
+         try {
+            uint64_t read_magic;
+            pfile.read((char*)(&read_magic), sizeof(read_magic));
+            if (read_magic != magic)
+               return false;
+            auto datastream = pfile.create_datastream();
+            StateObjectType read_sobj;
+            fc::raw::unpack(datastream, read_sobj);
+            sobj = std::move(read_sobj);
+            return true;
+         } catch (...) {
+            return false;
+         }
+      }
+      static bool write(const std::string& file_path, const StateObjectType& sobj) {
+         fc::cfile pfile;
+         pfile.set_file_path(file_path);
+         pfile.open(fc::cfile::truncate_rw_mode);
+         return write(pfile, sobj);
+      }
+   };
+
    using boost::multi_index_container;
    using namespace boost::multi_index;
    using namespace eosio::chain;
@@ -85,6 +133,7 @@ namespace eosio::hotstuff {
       bool is_quorum_met() const { return quorum_met; }
       void set_quorum_met() { quorum_met = true; }
 
+
    private:
       friend struct fc::reflector<quorum_certificate>;
       fc::sha256                          proposal_id;
@@ -109,7 +158,8 @@ namespace eosio::hotstuff {
       qc_chain(std::string id, base_pacemaker* pacemaker,
                std::set<name> my_producers,
                chain::bls_key_map_t finalizer_keys,
-               fc::logger& logger);
+               fc::logger& logger,
+               std::string safety_state_file);
 
       uint64_t get_state_version() const { return _state_version; } // no lock required
 
@@ -132,6 +182,8 @@ namespace eosio::hotstuff {
 
    private:
 
+      void write_safety_state_file();
+
       const hs_proposal_message* get_proposal(const fc::sha256& proposal_id); // returns nullptr if not found
 
       // returns false if proposal with that same ID already exists at the store of its height
@@ -141,9 +193,6 @@ namespace eosio::hotstuff {
 
       hs_bitset update_bitset(const hs_bitset& finalizer_set, const fc::crypto::blslib::bls_public_key& finalizer_key);
 
-      //get digest to sign from proposal data
-      digest_type get_digest_to_sign(const block_id_type& block_id, uint8_t phase_counter, const fc::sha256& final_on_qc);
-
       void reset_qc(const fc::sha256& proposal_id);
 
       bool evaluate_quorum(const hs_bitset& finalizers, const fc::crypto::blslib::bls_signature& agg_sig, const hs_proposal_message& proposal); //evaluate quorum for a proposal
@@ -164,7 +213,7 @@ namespace eosio::hotstuff {
       void process_new_view(const std::optional<uint32_t>& connection_id, const hs_new_view_message& msg);
       void process_new_block(const std::optional<uint32_t>& connection_id, const hs_new_block_message& msg);
 
-      hs_vote_message sign_proposal(const hs_proposal_message& proposal, const fc::crypto::blslib::bls_private_key& finalizer_priv_key);
+      hs_vote_message sign_proposal(const hs_proposal_message& proposal, const fc::crypto::blslib::bls_public_key& finalizer_pub_key, const fc::crypto::blslib::bls_private_key& finalizer_priv_key);
 
       //verify that a proposal descends from another
       bool extends(const fc::sha256& descendant, const fc::sha256& ancestor);
@@ -202,20 +251,23 @@ namespace eosio::hotstuff {
       };
 
       bool _chained_mode = false;
+
       block_id_type _block_exec;
       block_id_type _pending_proposal_block;
+      safety_state _safety_state;
       fc::sha256 _b_leaf;
-      fc::sha256 _b_lock;
       fc::sha256 _b_exec;
       fc::sha256 _b_finality_violation;
       quorum_certificate _high_qc;
       quorum_certificate _current_qc;
-      uint32_t _v_height = 0;
       base_pacemaker* _pacemaker = nullptr;
       std::set<name> _my_producers;
       chain::bls_key_map_t _my_finalizer_keys;
       std::string _id;
 
+      std::string _safety_state_file; // if empty, safety state persistence is turned off
+      fc::cfile _safety_state_file_handle;
+
       mutable std::atomic<uint64_t> _state_version = 1;
 
       fc::logger&            _logger;
@@ -239,11 +291,11 @@ namespace eosio::hotstuff {
          indexed_by<
             hashed_unique<
                tag<by_proposal_id>,
-               BOOST_MULTI_INDEX_MEMBER(hs_proposal_message,fc::sha256,proposal_id)
+               BOOST_MULTI_INDEX_MEMBER(hs_proposal_message, fc::sha256,proposal_id)
                >,
             ordered_non_unique<
                tag<by_proposal_height>,
-               BOOST_MULTI_INDEX_CONST_MEM_FUN(hs_proposal_message,uint64_t,get_height)
+               BOOST_MULTI_INDEX_CONST_MEM_FUN(hs_proposal_message, uint64_t, get_key)
                >
             >
          > proposal_store_type;

libraries/hotstuff/include/eosio/hotstuff/state.hpp

@@ -0,0 +1,61 @@
+#include <eosio/chain/block_header.hpp>
+
+#include <eosio/hotstuff/qc_chain.hpp>
+
+namespace eosio::hotstuff {
+
+    using namespace eosio::chain;
+
+    struct safety_state {
+
+      void set_v_height(const fc::crypto::blslib::bls_public_key& finalizer_key, const eosio::chain::view_number v_height) {
+         _states[finalizer_key].first = v_height;
+      }
+
+      void set_b_lock(const fc::crypto::blslib::bls_public_key& finalizer_key, const fc::sha256& b_lock) {
+         _states[finalizer_key].second = b_lock;
+      }
+
+      std::pair<eosio::chain::view_number, fc::sha256> get_safety_state(const fc::crypto::blslib::bls_public_key& finalizer_key) const {
+         auto s = _states.find(finalizer_key);
+         if (s != _states.end()) return s->second;
+         else return {};
+      }
+
+      eosio::chain::view_number get_v_height(const fc::crypto::blslib::bls_public_key& finalizer_key) const {
+         auto s = _states.find(finalizer_key);
+         if (s != _states.end()) return s->second.first;
+         else return {};
+      };
+
+      fc::sha256 get_b_lock(const fc::crypto::blslib::bls_public_key& finalizer_key) const {
+         auto s_itr = _states.find(finalizer_key);
+         if (s_itr != _states.end()) return s_itr->second.second;
+         else return {};
+      };
+
+      //todo : implement safety state default / sorting
+
+      std::pair<eosio::chain::view_number, fc::sha256> get_safety_state() const {
+         auto s = _states.begin();
+         if (s != _states.end()) return s->second;
+         else return {};
+      }
+
+      eosio::chain::view_number get_v_height() const {
+         auto s = _states.begin();
+         if (s != _states.end()) return s->second.first;
+         else return {};
+      };
+
+      fc::sha256 get_b_lock() const {
+         auto s_itr = _states.begin();
+         if (s_itr != _states.end()) return s_itr->second.second;
+         else return {};
+      };
+
+      std::map<fc::crypto::blslib::bls_public_key, std::pair<eosio::chain::view_number, fc::sha256>> _states;
+    };
+}
+
+FC_REFLECT(eosio::hotstuff::safety_state, (_states))