[1.0 -> main] Add testcase validating the new fsi design from issue

CMakeModules/EosioTester.cmake.in

@@ -99,6 +99,7 @@ target_link_libraries(EosioChain INTERFACE
    Boost::interprocess
    Boost::asio
    Boost::beast
+   Boost::crc
    Boost::signals2
    Boost::iostreams
    "-lz" # Needed by Boost iostreams

CMakeModules/EosioTesterBuild.cmake.in

@@ -96,6 +96,7 @@ target_link_libraries(EosioChain INTERFACE
    Boost::interprocess
    Boost::asio
    Boost::beast
+   Boost::crc
    Boost::signals2
    Boost::iostreams
    "-lz" # Needed by Boost iostreams

libraries/chain/controller.cpp

@@ -6001,6 +6001,10 @@ bool controller::is_node_finalizer_key(const bls_public_key& key) const {
    return my->my_finalizers.contains(key);
 }
 
+const my_finalizers_t& controller::get_node_finalizers() const {
+   return my->my_finalizers;
+}
+
 /// Protocol feature activation handlers:
 
 template<>

libraries/chain/include/eosio/chain/controller.hpp

@@ -9,6 +9,7 @@
 #include <eosio/chain/protocol_feature_manager.hpp>
 #include <eosio/chain/webassembly/eos-vm-oc/config.hpp>
 #include <eosio/chain/finality/vote_message.hpp>
+#include <eosio/chain/finality/finalizer.hpp>
 
 #include <chainbase/pinnable_mapped_file.hpp>
 
@@ -22,6 +23,10 @@ namespace boost::asio {
    class thread_pool;
 }
 
+namespace savanna_cluster {
+   class node_t;
+}
+
 namespace eosio::vm { class wasm_allocator; }
 
 namespace eosio::chain {
@@ -452,9 +457,13 @@ namespace eosio::chain {
       // is the bls key a registered finalizer key of this node, thread safe
       bool is_node_finalizer_key(const bls_public_key& key) const;
 
+
       private:
+         const my_finalizers_t& get_node_finalizers() const;  // used for tests (purpose is inspecting fsi).
+
          friend class apply_context;
          friend class transaction_context;
+         friend class savanna_cluster::node_t;
 
          chainbase::database& mutable_db()const;
 

libraries/chain/include/eosio/chain/finality/finalizer.hpp

@@ -181,7 +181,7 @@ namespace eosio::chain {
       fsi_map load_finalizer_safety_info();
 
       // for testing purposes only, not thread safe
-      const fsi_t& get_fsi(const bls_public_key& k) { return finalizers[k].fsi; }
+      const fsi_t& get_fsi(const bls_public_key& k) const { return finalizers.at(k).fsi; }
       void         set_fsi(const bls_public_key& k, const fsi_t& fsi) { finalizers[k].fsi = fsi; }
 
    private:

unittests/finalizer_tests.cpp

@@ -162,8 +162,7 @@ BOOST_AUTO_TEST_CASE( corrupt_finalizer_safety_file ) try {
                           finalizer_safety_exception);
 
       // make sure the safety info for our finalizer that we saved above is restored correctly
-      BOOST_CHECK_NE(fset.get_fsi(k.pubkey), fsi);
-      BOOST_CHECK_EQUAL(fset.get_fsi(k.pubkey), fsi_t());
+      BOOST_CHECK(!fset.contains(k.pubkey));
    }
 
 } FC_LOG_AND_RETHROW()

unittests/savanna_cluster.cpp

@@ -4,38 +4,50 @@ namespace savanna_cluster {
 
 node_t::node_t(size_t node_idx, cluster_t& cluster, setup_policy policy /* = setup_policy::none */)
    : tester(policy)
-   , node_idx(node_idx) {
+   , _node_idx(node_idx)
+   , _last_vote({}, false)
+{
 
    // since we are creating forks, finalizers may be locked on another fork and unable to vote.
    do_check_for_votes(false);
 
-   voted_block_cb = [&, node_idx](const eosio::chain::vote_signal_params& v) {
+   _voted_block_cb = [&, node_idx](const eosio::chain::vote_signal_params& v) {
       // no mutex needed because controller is set in tester (via `disable_async_voting(true)`)
       // to vote (and emit the `voted_block` signal) synchronously.
       // --------------------------------------------------------------------------------------
       vote_result_t status = std::get<1>(v);
 
       if (status == vote_result_t::success) {
          vote_message_ptr vote_msg = std::get<2>(v);
-         last_vote = vote_t(vote_msg);
-         if (propagate_votes)
-            cluster.dispatch_vote_to_peers(node_idx, skip_self_t::yes, std::get<2>(v));
+         _last_vote = vote_t(vote_msg->block_id, vote_msg->strong);
+
+         if (_propagate_votes) {
+            if (_vote_delay)
+               _delayed_votes.push_back(std::move(vote_msg));
+            while (_delayed_votes.size() > _vote_delay) {
+               vote_message_ptr vote = _delayed_votes.front();
+               _delayed_votes.erase(_delayed_votes.cbegin());
+               cluster.dispatch_vote_to_peers(node_idx, skip_self_t::yes, vote);
+            }
+            if (!_vote_delay)
+               cluster.dispatch_vote_to_peers(node_idx, skip_self_t::yes, vote_msg);
+         }
       }
    };
 
    // called on `commit_block`, for both blocks received from `push_block` and produced blocks
-   accepted_block_cb = [&, node_idx](const eosio::chain::block_signal_params& p) {
-      if (!pushing_a_block) {
+   _accepted_block_cb = [&, node_idx](const eosio::chain::block_signal_params& p) {
+      if (!_pushing_a_block) {
          // we want to propagate only blocks we produce, not the ones we receive from the network
          auto& b = std::get<0>(p);
          cluster.push_block_to_peers(node_idx, skip_self_t::yes, b);
       }
    };
 
    auto node_initialization_fn = [&, node_idx]() {
-      [[maybe_unused]] auto _a = control->voted_block().connect(voted_block_cb);
-      [[maybe_unused]] auto _b = control->accepted_block().connect(accepted_block_cb);
-      tester::set_node_finalizers(node_finalizers);
+      [[maybe_unused]] auto _a = control->voted_block().connect(_voted_block_cb);
+      [[maybe_unused]] auto _b = control->accepted_block().connect(_accepted_block_cb);
+      tester::set_node_finalizers(_node_finalizers);
       cluster.get_new_blocks_from_peers(node_idx);
    };
 

unittests/savanna_cluster.hpp

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <eosio/chain/finality/finalizer_authority.hpp>
+#include <eosio/chain/finality/finalizer.hpp>
 #include <fc/crypto/bls_private_key.hpp>
 
 #include <eosio/testing/tester.hpp>
@@ -22,6 +23,7 @@ namespace savanna_cluster {
    using block_header     = eosio::chain::block_header;
    using tester           = eosio::testing::tester;
    using setup_policy     = eosio::testing::setup_policy;
+   using fsi_t            = eosio::chain::finalizer_safety_information;
 
    class cluster_t;
 
@@ -51,34 +53,68 @@ namespace savanna_cluster {
       vector<bls_private_key> privkeys;
    };
 
+   // two classes for comparisons in BOOST_REQUIRE_EQUAL
+   // --------------------------------------------------
+   struct vote_t {
+      friend std::ostream& operator<<(std::ostream& s, const vote_t& v) {
+         s << "vote_t(" << v.id.str().substr(8, 16) << ", " << (v.strong ? "strong" : "weak") << ")";
+         return s;
+      }
+      bool operator==(const vote_t&) const = default;
+
+      block_id_type id;
+      bool          strong;
+   };
+
+   struct strong_vote : public vote_t {
+      explicit strong_vote(const signed_block_ptr& p) : vote_t(p->calculate_id(), true) {}
+   };
+   struct weak_vote : public vote_t {
+      explicit weak_vote(const signed_block_ptr& p) : vote_t(p->calculate_id(), false) {}
+   };
+
+
+   struct qc_s {
+      explicit qc_s(uint32_t block_num, bool strong) : block_num(block_num), strong(strong) {}
+      explicit qc_s(const std::optional<qc_t>& qc) : block_num(qc->block_num), strong(qc->is_strong()) {}
+
+      friend std::ostream& operator<<(std::ostream& s, const qc_s& v) {
+         s << "qc_s(" << v.block_num << ", " << (v.strong ? "strong" : "weak") << ")";
+         return s;
+      }
+      bool operator==(const qc_s&) const = default;
+
+      uint32_t block_num; // claimed block
+      bool     strong;
+   };
+
+   struct strong_qc : public qc_s {
+      explicit strong_qc(const signed_block_ptr& p) : qc_s(p->block_num(), true) {}
+   };
+   struct weak_qc : public qc_s {
+      explicit weak_qc(const signed_block_ptr& p) : qc_s(p->block_num(), false) {}
+   };
+
+   struct fsi_expect {
+      const signed_block_ptr& last_vote;
+      const signed_block_ptr& lock;
+      block_timestamp_type    other_branch_latest_time;
+   };
+
    // ----------------------------------------------------------------------------
    class node_t : public tester {
    private:
-      size_t node_idx;
-      bool   pushing_a_block{false};
-
-      std::function<void(const block_signal_params&)> accepted_block_cb;
-      std::function<void(const vote_signal_params&)>  voted_block_cb;
-
-   public:
-      struct vote_t {
-         vote_t() : strong(false) {}
-         explicit vote_t(const vote_message_ptr& p) : id(p->block_id), strong(p->strong) {}
-         explicit vote_t(const signed_block_ptr& p, bool strong) : id(p->calculate_id()), strong(strong) {}
-
-         friend std::ostream& operator<<(std::ostream& s, const vote_t& v) {
-            s << "vote_t(" << v.id.str().substr(8, 16) << ", " << (v.strong ? "strong" : "weak") << ")";
-            return s;
-         }
-         bool operator==(const vote_t&) const = default;
+      size_t                                          _node_idx;
+      bool                                            _pushing_a_block{false};
+      bool                                            _propagate_votes{true}; // if false, votes are dropped
+      vote_t                                          _last_vote;
+      std::vector<account_name>                       _node_finalizers;
 
-         block_id_type id;
-         bool          strong;
-      };
+      size_t                                          _vote_delay{0};         // delay vote propagation by this much
+      std::vector<vote_message_ptr>                   _delayed_votes;
 
-      bool   propagate_votes{true};
-      vote_t last_vote;
-      std::vector<account_name> node_finalizers;
+      std::function<void(const block_signal_params&)> _accepted_block_cb;
+      std::function<void(const vote_signal_params&)>  _voted_block_cb;
 
    public:
       node_t(size_t node_idx, cluster_t& cluster, setup_policy policy = setup_policy::none);
@@ -87,9 +123,18 @@ namespace savanna_cluster {
 
       node_t(node_t&&) = default;
 
+      bool& propagate_votes() { return _propagate_votes; }
+
+      size_t& vote_delay() { return _vote_delay; }
+
+      const vote_t& last_vote() const { return _last_vote; }
+
       void set_node_finalizers(std::span<const account_name> names) {
-         node_finalizers = std::vector<account_name>{ names.begin(), names.end() };
-         tester::set_node_finalizers(node_finalizers);
+         _node_finalizers = std::vector<account_name>{ names.begin(), names.end() };
+         if (control) {
+            // node is "open", se we can update the tester immediately
+            tester::set_node_finalizers(_node_finalizers);
+         }
       }
 
       void transition_to_savanna(std::span<const account_name> finalizer_policy_names) {
@@ -167,8 +212,8 @@ namespace savanna_cluster {
 
       void push_block(const signed_block_ptr& b) {
          if (is_open() && !fetch_block_by_id(b->calculate_id())) {
-            assert(!pushing_a_block);
-            fc::scoped_set_value set_pushing_a_block(pushing_a_block, true);
+            assert(!_pushing_a_block);
+            fc::scoped_set_value set_pushing_a_block(_pushing_a_block, true);
             tester::push_block(b);
          }
       }
@@ -189,19 +234,19 @@ namespace savanna_cluster {
       }
 
       std::string snapshot() const {
-         dlog("node ${i} - taking snapshot", ("i", node_idx));
+         dlog("node ${i} - taking snapshot", ("i", _node_idx));
          auto writer = buffered_snapshot_suite::get_writer();
          control->write_snapshot(writer);
          return buffered_snapshot_suite::finalize(writer);
       }
 
       void open_from_snapshot(const std::string& snapshot) {
-         dlog("node ${i} - restoring from snapshot", ("i", node_idx));
+         dlog("node ${i} - restoring from snapshot", ("i", _node_idx));
          open(buffered_snapshot_suite::get_reader(snapshot));
       }
 
       std::vector<uint8_t> save_fsi() const {
-         dlog("node ${i} - saving fsi", ("i", node_idx));
+         dlog("node ${i} - saving fsi", ("i", _node_idx));
          auto finalizer_path = get_fsi_path();
          std::ifstream file(finalizer_path.generic_string(), std::ios::binary | std::ios::ate);
          std::streamsize size = file.tellg();
@@ -214,21 +259,21 @@ namespace savanna_cluster {
       }
 
       void overwrite_fsi(const std::vector<uint8_t>& fsi) const {
-         dlog("node ${i} - overwriting fsi", ("i", node_idx));
+         dlog("node ${i} - overwriting fsi", ("i", _node_idx));
          auto finalizer_path = get_fsi_path();
          std::ofstream file(finalizer_path.generic_string(), std::ios::binary);
          assert(!fsi.empty());
          file.write((const char *)fsi.data(), fsi.size());
       }
 
       void remove_fsi() {
-         dlog("node ${i} - removing fsi", ("i", node_idx));
+         dlog("node ${i} - removing fsi", ("i", _node_idx));
          remove_all(get_fsi_path());
       }
 
       void remove_state() {
          auto state_path = cfg.state_dir;
-         dlog("node ${i} - removing state data from: ${state_path}", ("i", node_idx)("${state_path}", state_path));
+         dlog("node ${i} - removing state data from: ${state_path}", ("i", _node_idx)("${state_path}", state_path));
          remove_all(state_path);
          fs::create_directories(state_path);
       }
@@ -246,20 +291,34 @@ namespace savanna_cluster {
          for (auto const& dir_entry : std::filesystem::directory_iterator{path}) {
             auto path = dir_entry.path();
             if (path.filename().generic_string() != "reversible") {
-               dlog("node ${i} - removing : ${path}", ("i", node_idx)("${path}", path));
+               dlog("node ${i} - removing : ${path}", ("i", _node_idx)("${path}", path));
                remove_all(path);
             }
          }
       }
 
+      const fsi_t& get_fsi(size_t idx = 0) const {
+         assert(control);
+         assert(idx < _node_finalizers.size());
+         auto [privkey, pubkey, pop] = get_bls_key(_node_finalizers[idx]);
+         return control->get_node_finalizers().get_fsi(pubkey);
+      }
+
+      void check_fsi(const fsi_expect& expected) {
+         const fsi_t& fsi = get_fsi();
+         BOOST_REQUIRE_EQUAL(fsi.last_vote.block_id, expected.last_vote->calculate_id());
+         BOOST_REQUIRE_EQUAL(fsi.lock.block_id, expected.lock->calculate_id());
+         BOOST_REQUIRE_EQUAL(fsi.other_branch_latest_time, expected.other_branch_latest_time);
+      }
+
    private:
       // always removes reversible data (`blocks/reversible`)
       // optionally remove the blocks log as well by deleting the whole `blocks` directory
       // ---------------------------------------------------------------------------------
       void remove_blocks(bool rm_blocks_log) {
          auto reversible_path = cfg.blocks_dir / config::reversible_blocks_dir_name;
          auto& path = rm_blocks_log ? cfg.blocks_dir : reversible_path;
-         dlog("node ${i} - removing : ${path}", ("i", node_idx)("${path}", path));
+         dlog("node ${i} - removing : ${path}", ("i", _node_idx)("${path}", path));
          remove_all(path);
          fs::create_directories(reversible_path);
       }
@@ -471,22 +530,6 @@ namespace savanna_cluster {
 
       size_t num_nodes() const { return _num_nodes; }
 
-      // Class for comparisons in BOOST_REQUIRE_EQUAL
-      // --------------------------------------------
-      struct qc_s {
-         explicit qc_s(const signed_block_ptr& p, bool strong) : block_num(p->block_num()), strong(strong) {}
-         explicit qc_s(const std::optional<qc_t>& qc) : block_num(qc->block_num), strong(qc->is_strong()) {}
-
-         friend std::ostream& operator<<(std::ostream& s, const qc_s& v) {
-            s << "qc_s(" << v.block_num << ", " << (v.strong ? "strong" : "weak") << ")";
-            return s;
-         }
-         bool operator==(const qc_s&) const = default;
-
-         uint32_t block_num; // claimed block
-         bool     strong;
-      };
-
       static qc_claim_t qc_claim(const signed_block_ptr& b) {
          return b->extract_header_extension<finality_extension>().qc_claim;
       }