From e2804da31a28a87a6b445b389ce7657e5ee765da Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Tue, 9 Jan 2024 14:22:51 -0500 Subject: [PATCH 01/36] Update `start_block` for IF. wip. --- libraries/chain/controller.cpp | 71 +++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 31 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index d5b2857a14..df1b9fcbf1 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -260,11 +260,12 @@ struct block_data_t { template R apply_dpos(F& f) { if constexpr (std::is_same_v) - std::visit(overloaded{[&](block_data_legacy_t& bd) { bd.template apply(f); }, - [&](block_data_new_t& bd) {}}, v); + std::visit(overloaded{[&](block_data_legacy_t& bd) { bd.template apply(f); }, [&](block_data_new_t& bd) {}}, + v); else return std::visit(overloaded{[&](block_data_legacy_t& bd) -> R { return bd.template apply(f); }, - [&](block_data_new_t& bd) -> R { return {}; }}, v); + [&](block_data_new_t& bd) -> R { return {}; }}, + v); } }; @@ -411,7 +412,7 @@ struct assembled_block { return ab.pending_block_header_state.active_schedule; }, [](const assembled_block_if& ab) -> const producer_authority_schedule& { - static producer_authority_schedule pas; return pas; // [greg todo] + return ab.bhs.active_schedule_auth(); }}, v); } @@ -421,9 +422,9 @@ struct assembled_block { opt_pas& pending_producers() const { return std::visit( overloaded{[](const assembled_block_dpos& ab) -> opt_pas& { return ab.new_producer_authority_cache; }, - [](const assembled_block_if& ab) -> opt_pas& { + [](const assembled_block_if& ab) -> opt_pas& { // return ab.bhs.pending_schedule_auth(); // not an optional ref! static opt_pas empty; - return empty; // [greg todo] + return empty; // [areg] pending_producers() not needed in IF. proposed_proposers() sufficient. }}, v); } @@ -568,13 +569,13 @@ struct building_block { v(building_block_dpos(prev, when, num_prev_blocks_to_confirm, new_protocol_feature_activations)) {} - bool is_dpos() const { return std::holds_alternative(v); } - // if constructor - building_block(const block_header_state& prev, const building_block_input& bbi) : - v(building_block_if(prev, bbi)) + building_block(const block_header_state& prev, const building_block_input& input) : + v(building_block_if(prev, input)) {} + bool is_dpos() const { return std::holds_alternative(v); } + template R apply_dpos(F&& f) { if constexpr (std::is_same_v) @@ -778,21 +779,28 @@ struct building_block { using block_stage_type = std::variant; struct pending_state { - pending_state( maybe_session&& s, - const block_header_state_legacy& prev, - block_timestamp_type when, - uint16_t num_prev_blocks_to_confirm, - const vector& new_protocol_feature_activations ) - :_db_session( std::move(s) ) - ,_block_stage( building_block( prev, when, num_prev_blocks_to_confirm, new_protocol_feature_activations ) ) - {} - maybe_session _db_session; block_stage_type _block_stage; controller::block_status _block_status = controller::block_status::ephemeral; std::optional _producer_block_id; controller::block_report _block_report{}; + pending_state(maybe_session&& s, + const block_header_state_legacy& prev, + block_timestamp_type when, + uint16_t num_prev_blocks_to_confirm, + const vector& new_protocol_feature_activations) + :_db_session(std::move(s)) + ,_block_stage(building_block(prev, when, num_prev_blocks_to_confirm, new_protocol_feature_activations)) + {} + + pending_state(maybe_session&& s, + const block_header_state& prev, + const building_block_input& input) : + _db_session(std::move(s)), + _block_stage(building_block(prev, input)) + {} + deque extract_trx_metas() { return std::visit([](auto& stage) { return stage.extract_trx_metas(); }, _block_stage); } @@ -817,7 +825,7 @@ struct pending_state { _db_session.push(); } - bool is_dpos() const { return std::visit([](const auto& stage) { return stage.is_dpos(); }, _block_stage); } + //bool is_dpos() const { return std::visit([](const auto& stage) { return stage.is_dpos(); }, _block_stage); } const block_signing_authority& pending_block_signing_authority() const { return std::visit( @@ -2349,21 +2357,21 @@ struct controller_impl { pending.reset(); }); - //building_block_input bbi{ head->id(), when, head->get_scheduled_producer(when), std::move(new_protocol_feature_activations) }; - // [greg todo] build IF `building_block` below if not in dpos mode. - // we'll need a different `building_block` constructor for IF mode - auto update_pending = [&](auto& fork_db, auto& head) { - if (!self.skip_db_sessions(s)) { - EOS_ASSERT( db.revision() == head_block_num(), database_exception, "db revision is not on par with head block", + auto update_pending = [&](fork_db_t& fork_db, bsp& head) { + EOS_ASSERT( self.skip_db_sessions(s) || + db.revision() == head_block_num(), database_exception, "db revision is not on par with head block", ("db.revision()", db.revision())("controller_head_block", head_block_num())("fork_db_head_block", fork_db_head_block_num()) ); - - pending.emplace( maybe_session(db), *head, when, confirm_block_count, new_protocol_feature_activations ); - } else { - pending.emplace( maybe_session(), *head, when, confirm_block_count, new_protocol_feature_activations ); + maybe_session session = self.skip_db_sessions(s) ? maybe_session() : maybe_session(db); + if constexpr (std::is_same_v) + pending.emplace(std::move(session), *head, when, confirm_block_count, new_protocol_feature_activations); + else { + building_block_input bbi{ head->id(), when, head->get_scheduled_producer(when).producer_name, + new_protocol_feature_activations }; + pending.emplace(std::move(session), *head, bbi); } }; - block_data.apply_dpos(update_pending); + block_data.apply(update_pending); pending->_block_status = s; pending->_producer_block_id = producer_block_id; @@ -2440,6 +2448,7 @@ struct controller_impl { const auto& gpo = self.get_global_properties(); if (!hs_active) { +#warning todo: how do we update the producer_schedule after the switch to IF? bb.apply_dpos([&](building_block::building_block_dpos& bb_dpos) { pending_block_header_state_legacy& pbhs = bb_dpos.pending_block_header_state; From 610af7d0443f2654fec5b968d3bd664d976da09a Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Tue, 9 Jan 2024 14:34:12 -0500 Subject: [PATCH 02/36] Mostly whitespace cleanup. --- libraries/chain/controller.cpp | 73 ++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 35 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index df1b9fcbf1..2870857a9b 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -2352,7 +2352,7 @@ struct controller_impl { dm_logger->on_start_block(head_block_num() + 1); } - auto guard_pending = fc::make_scoped_exit([this, head_block_num=head_block_num()](){ + auto guard_pending = fc::make_scoped_exit([this, head_block_num=head_block_num()]() { protocol_features.popped_blocks_to( head_block_num ); pending.reset(); }); @@ -2380,8 +2380,7 @@ struct controller_impl { // block status is either ephemeral or incomplete. Modify state of speculative block only if we are building a // speculative incomplete block (otherwise we need clean state for head mode, ephemeral block) - if ( pending->_block_status != controller::block_status::ephemeral ) - { + if ( pending->_block_status != controller::block_status::ephemeral ) { const auto& pso = db.get(); auto num_preactivated_protocol_features = pso.preactivated_protocol_features.size(); @@ -2521,40 +2520,44 @@ struct controller_impl { EOS_ASSERT( std::holds_alternative(pending->_block_stage), block_validate_exception, "already called finalize_block"); try { - // [greg todo] move the merkle computation inside assemble_block. - auto& bb = std::get(pending->_block_stage); - - // Update resource limits: - resource_limits.process_account_limit_updates(); - const auto& chain_config = self.get_global_properties().configuration; - uint64_t CPU_TARGET = EOS_PERCENT(chain_config.max_block_cpu_usage, chain_config.target_block_cpu_usage_pct); - resource_limits.set_block_parameters( - { CPU_TARGET, chain_config.max_block_cpu_usage, config::block_cpu_usage_average_window_ms / config::block_interval_ms, config::maximum_elastic_resource_multiplier, {99, 100}, {1000, 999}}, - {EOS_PERCENT(chain_config.max_block_net_usage, chain_config.target_block_net_usage_pct), chain_config.max_block_net_usage, config::block_size_average_window_ms / config::block_interval_ms, config::maximum_elastic_resource_multiplier, {99, 100}, {1000, 999}} - ); - resource_limits.process_block_usage(bb.block_num()); - - auto assembled_block = - bb.assemble_block(thread_pool.get_executor(), protocol_features.get_protocol_feature_set(), block_data); - - // Update TaPoS table: - create_block_summary( assembled_block.id() ); + auto& bb = std::get(pending->_block_stage); - pending->_block_stage = std::move(assembled_block); + // Update resource limits: + resource_limits.process_account_limit_updates(); + const auto& chain_config = self.get_global_properties().configuration; + resource_limits.set_block_parameters( + { EOS_PERCENT(chain_config.max_block_cpu_usage, chain_config.target_block_cpu_usage_pct), + chain_config.max_block_cpu_usage, + config::block_cpu_usage_average_window_ms / config::block_interval_ms, + config::maximum_elastic_resource_multiplier, {99, 100}, {1000, 999}}, + { EOS_PERCENT(chain_config.max_block_net_usage, chain_config.target_block_net_usage_pct), + chain_config.max_block_net_usage, + config::block_size_average_window_ms / config::block_interval_ms, + config::maximum_elastic_resource_multiplier, {99, 100}, {1000, 999}} + ); + resource_limits.process_block_usage(bb.block_num()); + + auto assembled_block = + bb.assemble_block(thread_pool.get_executor(), protocol_features.get_protocol_feature_set(), block_data); + + // Update TaPoS table: + create_block_summary( assembled_block.id() ); + + pending->_block_stage = std::move(assembled_block); - /* - ilog( "finalized block ${n} (${id}) at ${t} by ${p} (${signing_key}); schedule_version: ${v} lib: ${lib} #dtrxs: ${ndtrxs} ${np}", - ("n",pbhs.block_num()) - ("id",id) - ("t",pbhs.timestamp) - ("p",pbhs.producer) - ("signing_key", pbhs.block_signing_key) - ("v",pbhs.active_schedule_version) - ("lib",pbhs.dpos_irreversible_blocknum) - ("ndtrxs",db.get_index().size()) - ("np",block_ptr->new_producers) - ); - */ + /* + ilog( "finalized block ${n} (${id}) at ${t} by ${p} (${signing_key}); schedule_version: ${v} lib: ${lib} #dtrxs: ${ndtrxs} ${np}", + ("n",pbhs.block_num()) + ("id",id) + ("t",pbhs.timestamp) + ("p",pbhs.producer) + ("signing_key", pbhs.block_signing_key) + ("v",pbhs.active_schedule_version) + ("lib",pbhs.dpos_irreversible_blocknum) + ("ndtrxs",db.get_index().size()) + ("np",block_ptr->new_producers) + ); + */ } FC_CAPTURE_AND_RETHROW() From 7d933af1d3794884343a3ed146d0eb4747ed4c0b Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Tue, 9 Jan 2024 14:46:59 -0500 Subject: [PATCH 03/36] Change `[greg todo]` to `#warning` for `producer_authority_schedule` related changes. --- libraries/chain/controller.cpp | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 2870857a9b..57697cd27c 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -689,8 +689,9 @@ struct building_block { return bb.pending_block_header_state.prev_pending_schedule.schedule; }, [](const building_block_if& bb) -> const producer_authority_schedule& { +#warning todo: support pending_producers correctly when in IF mode static producer_authority_schedule empty; - return empty; // [greg todo] + return empty; }}, v); } @@ -840,7 +841,7 @@ struct pending_state { } #if 0 - // [greg todo] maybe we don't need this and we can have the implementation in controller::pending_producers() +#warning todo: maybe we don't need this and we can have the implementation in controller::pending_producers() const producer_authority_schedule& pending_producers() const { return std::visit( overloaded{ @@ -2592,8 +2593,8 @@ struct controller_impl { emit( self.accepted_block, std::tie(bsp->block, bsp->id()) ); - if constexpr (std::is_same_v>) {\ - // [greg todo] support deep_mind_logger even when in IF mode + if constexpr (std::is_same_v>) { +#warning todo: support deep_mind_logger even when in IF mode // at block level, no transaction specific logging is possible if (auto* dm_logger = get_deep_mind_logger(false)) { dm_logger->on_accepted_block(bsp); @@ -4027,6 +4028,7 @@ const producer_authority_schedule& controller::active_producers()const { if( !(my->pending) ) return my->block_data.head_active_schedule_auth(); +#warning todo: support active/pending_producers correctly when in IF mode (see assembled_block and completed_block stages) return my->pending->active_producers(); } @@ -4035,9 +4037,9 @@ const producer_authority_schedule& controller::head_active_producers()const { } const producer_authority_schedule& controller::pending_producers()const { - if( !(my->pending) ) - return my->block_data.head_pending_schedule_auth(); // [greg todo] implement pending_producers correctly for IF mode - +#warning todo: support active/pending_producers correctly when in IF mode + if( !(my->pending) ) + return my->block_data.head_pending_schedule_auth(); if( std::holds_alternative(my->pending->_block_stage) ) return std::get(my->pending->_block_stage).pending_producers(); @@ -4422,7 +4424,8 @@ void controller::replace_producer_keys( const public_key_type& key ) { } }; - my->block_data.apply_dpos(replace_keys); // [greg todo]: make it work with `apply` instead of `apply_dpos` +#warning todo: support active/pending_producers correctly when in IF mode + my->block_data.apply_dpos(replace_keys); // make it work with `apply` instead of `apply_dpos` } void controller::replace_account_keys( name account, name permission, const public_key_type& key ) { From 1a3ded1e8ece8aedbb1274074bb21967e7b2c6e2 Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Thu, 11 Jan 2024 16:18:40 -0500 Subject: [PATCH 04/36] Resolve multiple `todo`s in controller and implement bew bhs::next() function. --- libraries/chain/block_header_state.cpp | 52 ++++++ libraries/chain/block_header_state_legacy.cpp | 9 +- libraries/chain/block_state.cpp | 90 +--------- libraries/chain/block_state_legacy.cpp | 27 +-- libraries/chain/controller.cpp | 107 ++++++------ libraries/chain/fork_database.cpp | 159 +++++++++--------- .../include/eosio/chain/block_header.hpp | 6 +- .../eosio/chain/block_header_state.hpp | 4 +- .../eosio/chain/block_header_state_legacy.hpp | 3 - .../eosio/chain/block_header_state_utils.hpp | 21 +++ .../chain/include/eosio/chain/block_state.hpp | 55 +++--- .../eosio/chain/block_state_legacy.hpp | 8 +- .../chain/include/eosio/chain/controller.hpp | 21 ++- .../include/eosio/chain/fork_database.hpp | 10 +- .../chain/unapplied_transaction_queue.hpp | 5 +- .../testing/include/eosio/testing/tester.hpp | 12 +- libraries/testing/tester.cpp | 16 +- plugins/producer_plugin/producer_plugin.cpp | 2 +- unittests/block_tests.cpp | 4 +- unittests/forked_tests.cpp | 2 +- unittests/protocol_feature_tests.cpp | 2 +- .../unapplied_transaction_queue_tests.cpp | 22 +-- 22 files changed, 323 insertions(+), 314 deletions(-) diff --git a/libraries/chain/block_header_state.cpp b/libraries/chain/block_header_state.cpp index 4a6161269c..af2276bf0f 100644 --- a/libraries/chain/block_header_state.cpp +++ b/libraries/chain/block_header_state.cpp @@ -75,6 +75,13 @@ block_header_state block_header_state::next(block_header_state_input& input) con else result.core = core; + if (!input.new_protocol_feature_activations.empty()) { + result.activated_protocol_features = std::make_shared( + *activated_protocol_features, input.new_protocol_feature_activations); + } else { + result.activated_protocol_features = activated_protocol_features; + } + // add block header extensions // --------------------------- if (input.new_finalizer_policy) @@ -93,5 +100,50 @@ block_header_state block_header_state::next(block_header_state_input& input) con return result; } +/** + * Transitions the current header state into the next header state given the supplied signed block header. + * + * Given a signed block header, generate the expected template based upon the header time, + * then validate that the provided header matches the template. + * + * If the header specifies new_producers then apply them accordingly. + */ +block_header_state block_header_state::next(const signed_block_header& h, const protocol_feature_set& pfs, + validator_t& validator) const { + auto producer = detail::get_scheduled_producer(proposer_policy->proposer_schedule.producers, h.timestamp).producer_name; + + EOS_ASSERT( h.previous == id, unlinkable_block_exception, "previous mismatch" ); + EOS_ASSERT( h.producer == producer, wrong_producer, "wrong producer specified" ); + + auto exts = h.validate_and_extract_header_extensions(); + + // handle protocol_feature_activation from incoming block + // ------------------------------------------------------ + vector new_protocol_feature_activations; + if( exts.count(protocol_feature_activation::extension_id() > 0) ) { + const auto& entry = exts.lower_bound(protocol_feature_activation::extension_id()); + new_protocol_feature_activations = std::move(std::get(entry->second).protocol_features); + } + + // retrieve instant_finality_extension data from block extension + // ------------------------------------------------------------- + EOS_ASSERT(exts.count(instant_finality_extension::extension_id() > 0), misc_exception, + "Instant Finality Extension is expected to be present in all block headers after switch to IF"); + const auto& if_entry = exts.lower_bound(instant_finality_extension::extension_id()); + const auto& if_ext = std::get(if_entry->second); + + building_block_input bb_input{ + .parent_id = id, + .timestamp = h.timestamp, + .producer = producer, + .new_protocol_feature_activations = std::move(new_protocol_feature_activations) + }; + + block_header_state_input bhs_input{ + bb_input, h.transaction_mroot, h.action_mroot, if_ext.new_proposer_policy, if_ext.new_finalizer_policy, + if_ext.qc_info}; + + return next(bhs_input); +} } // namespace eosio::chain \ No newline at end of file diff --git a/libraries/chain/block_header_state_legacy.cpp b/libraries/chain/block_header_state_legacy.cpp index 615df6b06d..8d695b1425 100644 --- a/libraries/chain/block_header_state_legacy.cpp +++ b/libraries/chain/block_header_state_legacy.cpp @@ -293,7 +293,7 @@ namespace eosio::chain { if( maybe_new_producer_schedule ) { result.pending_schedule.schedule = std::move(*maybe_new_producer_schedule); - result.pending_schedule.schedule_hash = std::move(*maybe_new_producer_schedule_hash); + result.pending_schedule.schedule_hash = *maybe_new_producer_schedule_hash; result.pending_schedule.schedule_lib_num = block_number; } else { if( was_pending_promoted ) { @@ -301,7 +301,7 @@ namespace eosio::chain { } else { result.pending_schedule.schedule = std::move( prev_pending_schedule.schedule ); } - result.pending_schedule.schedule_hash = std::move( prev_pending_schedule.schedule_hash ); + result.pending_schedule.schedule_hash = prev_pending_schedule.schedule_hash ; result.pending_schedule.schedule_lib_num = prev_pending_schedule.schedule_lib_num; } @@ -369,13 +369,12 @@ namespace eosio::chain { */ block_header_state_legacy block_header_state_legacy::next( const signed_block_header& h, - vector&& _additional_signatures, + vector&& additional_signatures, const protocol_feature_set& pfs, - bool hotstuff_activated, validator_t& validator, bool skip_validate_signee )const { - return next( h.timestamp, h.confirmed ).finish_next( h, std::move(_additional_signatures), pfs, validator, skip_validate_signee ); + return next( h.timestamp, h.confirmed ).finish_next( h, std::move(additional_signatures), pfs, validator, skip_validate_signee ); } digest_type block_header_state_legacy::sig_digest()const { diff --git a/libraries/chain/block_state.cpp b/libraries/chain/block_state.cpp index 4c43eadc12..4e5255a03e 100644 --- a/libraries/chain/block_state.cpp +++ b/libraries/chain/block_state.cpp @@ -1,94 +1,16 @@ #include +#include #include namespace eosio::chain { - namespace { - constexpr auto additional_sigs_eid = additional_block_signatures_extension::extension_id(); - - /** - * Given a complete signed block, extract the validated additional signatures if present; - * - * @param b complete signed block - * @param pfs protocol feature set for digest access - * @param pfa activated protocol feature set to determine if extensions are allowed - * @return the list of additional signatures - * @throws if additional signatures are present before being supported by protocol feature activations - */ - vector extract_additional_signatures( const signed_block_ptr& b, - const protocol_feature_set& pfs, - const protocol_feature_activation_set_ptr& pfa ) - { - auto exts = b->validate_and_extract_extensions(); - - if ( exts.count(additional_sigs_eid) > 0 ) { - auto& additional_sigs = std::get(exts.lower_bound(additional_sigs_eid)->second); - - return std::move(additional_sigs.signatures); - } - - return {}; - } - - /** - * Given a pending block header state, wrap the promotion to a block header state such that additional signatures - * can be allowed based on activations *prior* to the promoted block and properly injected into the signed block - * that is previously constructed and mutated by the promotion - * - * This cleans up lifetime issues involved with accessing activated protocol features and moving from the - * pending block header state - * - * @param cur the pending block header state to promote - * @param b the signed block that will receive signatures during this process - * @param pfs protocol feature set for digest access - * @param extras all the remaining parameters that pass through - * @return the block header state - * @throws if the block was signed with multiple signatures before the extension is allowed - */ - - template - block_header_state inject_additional_signatures(block_header_state&& cur, - signed_block& b, - const protocol_feature_set& pfs, - Extras&& ... extras) - { - - block_header_state result; -#if 0 - result = std::move(cur).finish_next(b, pfs, std::forward(extras)...); - auto pfa = cur.prev_activated_protocol_features; - - if (!result.additional_signatures.empty()) { - bool wtmsig_enabled = detail::is_builtin_activated(pfa, pfs, builtin_protocol_feature_t::wtmsig_block_signatures); - - EOS_ASSERT(wtmsig_enabled, block_validate_exception, - "Block has multiple signatures before activation of WTMsig Block Signatures"); - - // as an optimization we don't copy this out into the legitimate extension structure as it serializes - // the same way as the vector of signatures - static_assert(fc::reflector::total_member_count == 1); - static_assert(std::is_same_v>); - - emplace_extension(b.block_extensions, additional_sigs_eid, fc::raw::pack( result.additional_signatures )); - } -#endif - return result; - } - - } -#if 0 - - block_state::block_state(const block_header_state& prev, - signed_block_ptr b, - const protocol_feature_set& pfs, - bool hotstuff_activated, - const validator_t& validator, - bool skip_validate_signee - ) - :block_header_state( prev.next( *b, extract_additional_signatures(b, pfs, prev.activated_protocol_features), pfs, hotstuff_activated, validator, skip_validate_signee ) ) - ,block( std::move(b) ) + block_state::block_state(const block_header_state& prev, signed_block_ptr b, const protocol_feature_set& pfs, + const validator_t& validator, bool skip_validate_signee) + : block_header_state(prev.next(*b, pfs, validator)) + , block(std::move(b)) {} +#if 0 block_state::block_state(pending_block_header_state&& cur, signed_block_ptr&& b, deque&& trx_metas, diff --git a/libraries/chain/block_state_legacy.cpp b/libraries/chain/block_state_legacy.cpp index a3b11aadf7..100744e11c 100644 --- a/libraries/chain/block_state_legacy.cpp +++ b/libraries/chain/block_state_legacy.cpp @@ -7,30 +7,6 @@ namespace eosio { namespace chain { namespace { constexpr auto additional_sigs_eid = additional_block_signatures_extension::extension_id(); - /** - * Given a complete signed block, extract the validated additional signatures if present; - * - * @param b complete signed block - * @param pfs protocol feature set for digest access - * @param pfa activated protocol feature set to determine if extensions are allowed - * @return the list of additional signatures - * @throws if additional signatures are present before being supported by protocol feature activations - */ - vector extract_additional_signatures( const signed_block_ptr& b, - const protocol_feature_set& pfs, - const protocol_feature_activation_set_ptr& pfa ) - { - auto exts = b->validate_and_extract_extensions(); - - if ( exts.count(additional_sigs_eid) > 0 ) { - auto& additional_sigs = std::get(exts.lower_bound(additional_sigs_eid)->second); - - return std::move(additional_sigs.signatures); - } - - return {}; - } - /** * Given a pending block header state, wrap the promotion to a block header state such that additional signatures * can be allowed based on activations *prior* to the promoted block and properly injected into the signed block @@ -78,11 +54,10 @@ namespace eosio { namespace chain { block_state_legacy::block_state_legacy( const block_header_state_legacy& prev, signed_block_ptr b, const protocol_feature_set& pfs, - bool hotstuff_activated, const validator_t& validator, bool skip_validate_signee ) - :block_header_state_legacy( prev.next( *b, extract_additional_signatures(b, pfs, prev.activated_protocol_features), pfs, hotstuff_activated, validator, skip_validate_signee ) ) + :block_header_state_legacy( prev.next( *b, detail::extract_additional_signatures(b), pfs, validator, skip_validate_signee ) ) ,block( std::move(b) ) {} diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 57697cd27c..8f5020fe3a 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -121,7 +121,7 @@ struct block_data_gen_t { public: using bs = bsp::element_type; using bhs = bhsp::element_type; - using fork_db_t = fork_database; + using fork_db_t = fork_database; bsp head; fork_db_t fork_db; @@ -249,8 +249,8 @@ struct block_data_t { }, v); } - template - R apply(F &f) { + template + R apply(F& f) { if constexpr (std::is_same_v) std::visit([&](auto& bd) { bd.template apply(f); }, v); else @@ -446,7 +446,7 @@ struct assembled_block { std::move(ab.pending_block_header_state), std::move(ab.unsigned_block), std::move(ab.trx_metas), pfs, validator, signer); - return completed_block{block_state_legacy_ptr{std::move(bsp)}}; + return completed_block{std::move(bsp)}; }, [&](assembled_block_if& ab) { return completed_block{}; /* [greg todo] */ @@ -1156,12 +1156,13 @@ struct controller_impl { std::exception_ptr except_ptr; auto replay_blog = [&](auto& fork_db, auto& head) { + using BSP = std::decay_t; if( blog_head && start_block_num <= blog_head->block_num() ) { ilog( "existing block log, attempting to replay from ${s} to ${n} blocks", ("s", start_block_num)("n", blog_head->block_num()) ); try { while( auto next = blog.read_block_by_num( head->block_num() + 1 ) ) { - replay_push_block( next, controller::block_status::irreversible ); + replay_push_block( next, controller::block_status::irreversible ); if( check_shutdown() ) break; if( next->block_num() % 500 == 0 ) { ilog( "${n} of ${head}", ("n", next->block_num())("head", blog_head->block_num()) ); @@ -1208,7 +1209,7 @@ struct controller_impl { if( check_shutdown() ) break; if( (*i)->block_num() <= head_block_num ) continue; ++rev; - replay_push_block( (*i)->block, controller::block_status::validated ); + replay_push_block( (*i)->block, controller::block_status::validated ); } ilog( "${n} reversible blocks replayed", ("n",rev) ); } @@ -1410,7 +1411,7 @@ struct controller_impl { ) { wlog( "applying branch from fork database ending with block: ${id}", ("id", pending_head->id()) ); controller::block_report br; - maybe_switch_forks( br, pending_head, controller::block_status::complete, forked_branch_callback{}, trx_meta_cache_lookup{} ); + maybe_switch_forks( br, pending_head, controller::block_status::complete, {}, trx_meta_cache_lookup{} ); } } }; @@ -2820,7 +2821,7 @@ struct controller_impl { commit_block(s); br.total_time = fc::time_point::now() - start; }; - block_data.apply_dpos(do_the_work); + block_data.apply(do_the_work); } return; } catch ( const std::bad_alloc& ) { @@ -2856,7 +2857,6 @@ struct controller_impl { prev, b, protocol_features.get_protocol_feature_set(), - b->confirmed == hs_block_confirmed, // is hotstuff enabled for block [this]( block_timestamp_type timestamp, const flat_set& cur_features, const vector& new_features ) @@ -2908,9 +2908,10 @@ struct controller_impl { return block_data.apply_dpos(f); } + template void push_block( controller::block_report& br, - const block_state_legacy_ptr& bsp, - const forked_branch_callback& forked_branch_cb, + const BSP& bsp, + const forked_branch_callback_t& forked_branch_cb, const trx_meta_cache_lookup& trx_lookup ) { controller::block_status s = controller::block_status::complete; @@ -2930,7 +2931,8 @@ struct controller_impl { } auto do_push = [&](auto& fork_db, auto& head) { - fork_db.add( bsp ); + if constexpr (std::is_same_v>) + fork_db.add( bsp ); if (self.is_trusted_producer(b->producer)) { trusted_producer_light_validation = true; @@ -2939,18 +2941,19 @@ struct controller_impl { emit( self.accepted_block_header, std::tie(bsp->block, bsp->id()) ); if( read_mode != db_read_mode::IRREVERSIBLE ) { - maybe_switch_forks( br, fork_db.pending_head(), s, forked_branch_cb, trx_lookup ); + if constexpr (std::is_same_v>) + maybe_switch_forks( br, fork_db.pending_head(), s, forked_branch_cb, trx_lookup ); } else { log_irreversible(); } }; - block_data.apply_dpos(do_push); // [greg todo] make it work with apply() - `push_block` taking block_state_legacy_ptr - // and forked_branch_callback + block_data.apply(do_push); } FC_LOG_AND_RETHROW( ) } + template void replay_push_block( const signed_block_ptr& b, controller::block_status s ) { self.validate_db_available_size(); @@ -2968,53 +2971,49 @@ struct controller_impl { } const bool skip_validate_signee = !conf.force_all_checks; + validator_t validator = [this](block_timestamp_type timestamp, const flat_set& cur_features, + const vector& new_features) { + check_protocol_features(timestamp, cur_features, new_features); + }; auto do_push = [&](auto& fork_db, auto& head) { - auto bsp = std::make_shared( - *head, - b, - protocol_features.get_protocol_feature_set(), - b->confirmed == hs_block_confirmed, // is hotstuff enabled for block - [this]( block_timestamp_type timestamp, - const flat_set& cur_features, - const vector& new_features ) - { check_protocol_features( timestamp, cur_features, new_features ); }, - skip_validate_signee - ); + if constexpr (std::is_same_v>) { + auto bsp = std::make_shared( + *head, b, protocol_features.get_protocol_feature_set(), validator, skip_validate_signee); - if( s != controller::block_status::irreversible ) { - fork_db.add( bsp, true ); - } + if (s != controller::block_status::irreversible) { + fork_db.add(bsp, true); + } - emit( self.accepted_block_header, std::tie(bsp->block, bsp->id()) ); + emit(self.accepted_block_header, std::tie(bsp->block, bsp->id())); - controller::block_report br; - if( s == controller::block_status::irreversible ) { - apply_block( br, bsp, s, trx_meta_cache_lookup{} ); + controller::block_report br; + if (s == controller::block_status::irreversible) { + apply_block(br, bsp, s, trx_meta_cache_lookup{}); - // On replay, log_irreversible is not called and so no irreversible_block signal is emitted. - // So emit it explicitly here. - emit( self.irreversible_block, std::tie(bsp->block, bsp->id()) ); + // On replay, log_irreversible is not called and so no irreversible_block signal is emitted. + // So emit it explicitly here. + emit(self.irreversible_block, std::tie(bsp->block, bsp->id())); - if (!self.skip_db_sessions(s)) { - db.commit(bsp->block_num()); + if (!self.skip_db_sessions(s)) { + db.commit(bsp->block_num()); + } + } else { + EOS_ASSERT(read_mode != db_read_mode::IRREVERSIBLE, block_validate_exception, + "invariant failure: cannot replay reversible blocks while in irreversible mode"); + maybe_switch_forks(br, bsp, s, {}, trx_meta_cache_lookup{}); } - - } else { - EOS_ASSERT( read_mode != db_read_mode::IRREVERSIBLE, block_validate_exception, - "invariant failure: cannot replay reversible blocks while in irreversible mode" ); - maybe_switch_forks( br, bsp, s, forked_branch_callback{}, trx_meta_cache_lookup{} ); } }; - - block_data.apply_dpos(do_push); // [greg todo] make it work with apply() - need block_state constructor + + block_data.apply(do_push); } FC_LOG_AND_RETHROW( ) } template void maybe_switch_forks( controller::block_report& br, const BSP& new_head, controller::block_status s, - const forked_branch_callback& forked_branch_cb, const trx_meta_cache_lookup& trx_lookup ) + const forked_branch_callback_t& forked_branch_cb, const trx_meta_cache_lookup& trx_lookup ) { auto do_maybe_switch_forks = [&](auto& fork_db, auto& head) { bool head_changed = true; @@ -3043,7 +3042,9 @@ struct controller_impl { EOS_ASSERT( self.head_block_id() == branches.second.back()->header.previous, fork_database_exception, "loss of sync between fork_db and chainbase during fork switch" ); // _should_ never fail - if( forked_branch_cb ) forked_branch_cb( branches.second ); + if( forked_branch_cb ) + if constexpr (std::is_same_v>) + forked_branch_cb(branches.second); } for( auto ritr = branches.first.rbegin(); ritr != branches.first.rend(); ++ritr ) { @@ -3107,7 +3108,7 @@ struct controller_impl { log_irreversible(); }; - block_data.apply_dpos(do_maybe_switch_forks); // [greg todo] + block_data.apply(do_maybe_switch_forks); } /// push_block @@ -3693,6 +3694,15 @@ block_state_legacy_ptr controller::create_block_state( const block_id_type& id, void controller::push_block( controller::block_report& br, const block_state_legacy_ptr& bsp, + const forked_branch_callback_legacy& forked_branch_cb, + const trx_meta_cache_lookup& trx_lookup ) +{ + validate_db_available_size(); + my->push_block( br, bsp, forked_branch_cb, trx_lookup ); +} + +void controller::push_block( controller::block_report& br, + const block_state_ptr& bsp, const forked_branch_callback& forked_branch_cb, const trx_meta_cache_lookup& trx_lookup ) { @@ -3700,6 +3710,7 @@ void controller::push_block( controller::block_report& br, my->push_block( br, bsp, forked_branch_cb, trx_lookup ); } + transaction_trace_ptr controller::push_transaction( const transaction_metadata_ptr& trx, fc::time_point block_deadline, fc::microseconds max_transaction_time, uint32_t billed_cpu_time_us, bool explicit_billed_cpu_time, diff --git a/libraries/chain/fork_database.cpp b/libraries/chain/fork_database.cpp index 7c1c62a2d8..6431c7d5f1 100644 --- a/libraries/chain/fork_database.cpp +++ b/libraries/chain/fork_database.cpp @@ -15,13 +15,13 @@ namespace eosio::chain { using boost::multi_index_container; using namespace boost::multi_index; - template - const uint32_t fork_database::magic_number = 0x30510FDB; + template + const uint32_t fork_database::magic_number = 0x30510FDB; - template - const uint32_t fork_database::min_supported_version = 2; - template - const uint32_t fork_database::max_supported_version = 2; + template + const uint32_t fork_database::min_supported_version = 2; + template + const uint32_t fork_database::max_supported_version = 2; /** * History: @@ -42,12 +42,13 @@ namespace eosio::chain { return std::pair(lhs.irreversible_blocknum(), lhs.block_num()) > std::pair(rhs.irreversible_blocknum(), rhs.block_num()); } - template // either [block_state_legacy_ptr, block_state_ptr], same with block_header_state_ptr + template // either [block_state_legacy_ptr, block_state_ptr], same with block_header_state_ptr struct fork_database_impl { using bs = bsp::element_type; + using bhsp = bs::bhsp_t; using bhs = bhsp::element_type; - using fork_database_t = fork_database; + using fork_database_t = fork_database; using branch_type = fork_database_t::branch_type; using branch_type_pair = fork_database_t::branch_type_pair; @@ -89,20 +90,20 @@ namespace eosio::chain { }; - template - fork_database::fork_database( const std::filesystem::path& data_dir ) - :my( new fork_database_impl( data_dir ) ) + template + fork_database::fork_database( const std::filesystem::path& data_dir ) + :my( new fork_database_impl( data_dir ) ) {} - template - void fork_database::open( validator_t& validator ) { + template + void fork_database::open( validator_t& validator ) { std::lock_guard g( my->mtx ); my->open_impl( validator ); } - template - void fork_database_impl::open_impl( validator_t& validator ) { + template + void fork_database_impl::open_impl( validator_t& validator ) { if (!std::filesystem::is_directory(datadir)) std::filesystem::create_directories(datadir); @@ -177,14 +178,14 @@ namespace eosio::chain { } } - template - void fork_database::close() { + template + void fork_database::close() { std::lock_guard g( my->mtx ); my->close_impl(); } - template - void fork_database_impl::close_impl() { + template + void fork_database_impl::close_impl() { auto fork_db_dat = datadir / config::forkdb_filename; if( !root ) { @@ -251,19 +252,19 @@ namespace eosio::chain { index.clear(); } - template - fork_database::~fork_database() { + template + fork_database::~fork_database() { my->close_impl(); } - template - void fork_database::reset( const bhs& root_bhs ) { + template + void fork_database::reset( const bhs& root_bhs ) { std::lock_guard g( my->mtx ); my->reset_impl(root_bhs); } - template - void fork_database_impl::reset_impl( const bhs& root_bhs ) { + template + void fork_database_impl::reset_impl( const bhs& root_bhs ) { index.clear(); root = std::make_shared(); static_cast(*root) = root_bhs; @@ -271,14 +272,14 @@ namespace eosio::chain { head = root; } - template - void fork_database::rollback_head_to_root() { + template + void fork_database::rollback_head_to_root() { std::lock_guard g( my->mtx ); my->rollback_head_to_root_impl(); } - template - void fork_database_impl::rollback_head_to_root_impl() { + template + void fork_database_impl::rollback_head_to_root_impl() { auto& by_id_idx = index.template get(); auto itr = by_id_idx.begin(); while (itr != by_id_idx.end()) { @@ -290,14 +291,14 @@ namespace eosio::chain { head = root; } - template - void fork_database::advance_root( const block_id_type& id ) { + template + void fork_database::advance_root( const block_id_type& id ) { std::lock_guard g( my->mtx ); my->advance_root_impl( id ); } - template - void fork_database_impl::advance_root_impl( const block_id_type& id ) { + template + void fork_database_impl::advance_root_impl( const block_id_type& id ) { EOS_ASSERT( root, fork_database_exception, "root not yet set" ); auto new_root = get_block_impl( id ); @@ -330,14 +331,14 @@ namespace eosio::chain { root = new_root; } - template - bhsp fork_database::get_block_header( const block_id_type& id ) const { + template + fork_database::bhsp fork_database::get_block_header( const block_id_type& id ) const { std::shared_lock g( my->mtx ); return my->get_block_header_impl( id ); } - template - bhsp fork_database_impl::get_block_header_impl( const block_id_type& id ) const { + template + fork_database_impl::bhsp fork_database_impl::get_block_header_impl( const block_id_type& id ) const { if( root->id() == id ) { return root; } @@ -349,8 +350,8 @@ namespace eosio::chain { return bhsp(); } - template - void fork_database_impl::add_impl(const bsp& n, bool ignore_duplicate, bool validate, validator_t& validator) { + template + void fork_database_impl::add_impl(const bsp& n, bool ignore_duplicate, bool validate, validator_t& validator) { EOS_ASSERT( root, fork_database_exception, "root not yet set" ); EOS_ASSERT( n, fork_database_exception, "attempt to add null block state" ); @@ -382,8 +383,8 @@ namespace eosio::chain { } } - template - void fork_database::add( const bsp& n, bool ignore_duplicate ) { + template + void fork_database::add( const bsp& n, bool ignore_duplicate ) { std::lock_guard g( my->mtx ); my->add_impl( n, ignore_duplicate, false, []( block_timestamp_type timestamp, @@ -393,20 +394,20 @@ namespace eosio::chain { ); } - template - bsp fork_database::root() const { + template + bsp fork_database::root() const { std::shared_lock g( my->mtx ); return my->root; } - template - bsp fork_database::head() const { + template + bsp fork_database::head() const { std::shared_lock g( my->mtx ); return my->head; } - template - bsp fork_database::pending_head() const { + template + bsp fork_database::pending_head() const { std::shared_lock g( my->mtx ); const auto& indx = my->index.template get(); @@ -419,17 +420,17 @@ namespace eosio::chain { return my->head; } - template - fork_database::branch_type - fork_database::fetch_branch(const block_id_type& h, + template + fork_database::branch_type + fork_database::fetch_branch(const block_id_type& h, uint32_t trim_after_block_num) const { std::shared_lock g(my->mtx); return my->fetch_branch_impl(h, trim_after_block_num); } - template - fork_database::branch_type - fork_database_impl::fetch_branch_impl(const block_id_type& h, uint32_t trim_after_block_num) const { + template + fork_database::branch_type + fork_database_impl::fetch_branch_impl(const block_id_type& h, uint32_t trim_after_block_num) const { branch_type result; for (auto s = get_block_impl(h); s; s = get_block_impl(s->previous())) { if (s->block_num() <= trim_after_block_num) @@ -439,14 +440,14 @@ namespace eosio::chain { return result; } - template - bsp fork_database::search_on_branch( const block_id_type& h, uint32_t block_num ) const { + template + bsp fork_database::search_on_branch( const block_id_type& h, uint32_t block_num ) const { std::shared_lock g( my->mtx ); return my->search_on_branch_impl( h, block_num ); } - template - bsp fork_database_impl::search_on_branch_impl( const block_id_type& h, uint32_t block_num ) const { + template + bsp fork_database_impl::search_on_branch_impl( const block_id_type& h, uint32_t block_num ) const { for( auto s = get_block_impl(h); s; s = get_block_impl( s->previous() ) ) { if( s->block_num() == block_num ) return s; @@ -459,16 +460,16 @@ namespace eosio::chain { * Given two head blocks, return two branches of the fork graph that * end with a common ancestor (same prior block) */ - template - fork_database::branch_type_pair - fork_database::fetch_branch_from(const block_id_type& first, const block_id_type& second) const { + template + fork_database::branch_type_pair + fork_database::fetch_branch_from(const block_id_type& first, const block_id_type& second) const { std::shared_lock g(my->mtx); return my->fetch_branch_from_impl(first, second); } - template - fork_database::branch_type_pair - fork_database_impl::fetch_branch_from_impl(const block_id_type& first, const block_id_type& second) const { + template + fork_database::branch_type_pair + fork_database_impl::fetch_branch_from_impl(const block_id_type& first, const block_id_type& second) const { pair result; auto first_branch = (first == root->id()) ? root : get_block_impl(first); auto second_branch = (second == root->id()) ? root : get_block_impl(second); @@ -527,14 +528,14 @@ namespace eosio::chain { } /// fetch_branch_from_impl /// remove all of the invalid forks built off of this id including this id - template - void fork_database::remove( const block_id_type& id ) { + template + void fork_database::remove( const block_id_type& id ) { std::lock_guard g( my->mtx ); return my->remove_impl( id ); } - template - void fork_database_impl::remove_impl( const block_id_type& id ) { + template + void fork_database_impl::remove_impl( const block_id_type& id ) { deque remove_queue{id}; const auto& previdx = index.template get(); const auto& head_id = head->id(); @@ -555,14 +556,14 @@ namespace eosio::chain { } } - template - void fork_database::mark_valid( const bsp& h ) { + template + void fork_database::mark_valid( const bsp& h ) { std::lock_guard g( my->mtx ); my->mark_valid_impl( h ); } - template - void fork_database_impl::mark_valid_impl( const bsp& h ) { + template + void fork_database_impl::mark_valid_impl( const bsp& h ) { if( h->is_valid() ) return; auto& by_id_idx = index.template get(); @@ -582,14 +583,14 @@ namespace eosio::chain { } } - template - bsp fork_database::get_block(const block_id_type& id) const { + template + bsp fork_database::get_block(const block_id_type& id) const { std::shared_lock g( my->mtx ); return my->get_block_impl(id); } - template - bsp fork_database_impl::get_block_impl(const block_id_type& id) const { + template + bsp fork_database_impl::get_block_impl(const block_id_type& id) const { auto itr = index.find( id ); if( itr != index.end() ) return *itr; @@ -597,10 +598,10 @@ namespace eosio::chain { } // do class instantiations - template class fork_database; - template class fork_database; + template class fork_database; + template class fork_database; - template struct fork_database_impl; - template struct fork_database_impl; + template struct fork_database_impl; + template struct fork_database_impl; } /// eosio::chain diff --git a/libraries/chain/include/eosio/chain/block_header.hpp b/libraries/chain/include/eosio/chain/block_header.hpp index 1670ec6131..6f9d9e0393 100644 --- a/libraries/chain/include/eosio/chain/block_header.hpp +++ b/libraries/chain/include/eosio/chain/block_header.hpp @@ -7,7 +7,7 @@ #include #include -namespace eosio { namespace chain { +namespace eosio::chain { namespace detail { template @@ -26,6 +26,8 @@ namespace eosio { namespace chain { using block_header_extension = block_header_extension_types::block_header_extension_t; using header_extension_multimap = flat_multimap; + using validator_t = const std::function&, const vector&)>; + // totem for block_header.confirmed that indicates hotstuff consensus is active constexpr uint16_t hs_block_confirmed = std::numeric_limits::max(); @@ -84,7 +86,7 @@ namespace eosio { namespace chain { signature_type producer_signature; }; -} } /// namespace eosio::chain +} /// namespace eosio::chain FC_REFLECT(eosio::chain::block_header, (timestamp)(producer)(confirmed)(previous) diff --git a/libraries/chain/include/eosio/chain/block_header_state.hpp b/libraries/chain/include/eosio/chain/block_header_state.hpp index da9e525041..9a1b7c5f7b 100644 --- a/libraries/chain/include/eosio/chain/block_header_state.hpp +++ b/libraries/chain/include/eosio/chain/block_header_state.hpp @@ -70,7 +70,9 @@ struct block_header_state { const producer_authority_schedule& pending_schedule_auth() const { return proposer_policies.rbegin()->second->proposer_schedule; } // [greg todo] block_header_state next(block_header_state_input& data) const; - + + block_header_state next(const signed_block_header& h, const protocol_feature_set& pfs, validator_t& validator) const; + // block descending from this need the provided qc in the block extension bool is_needed(const quorum_certificate& qc) const { return !core.last_qc_block_num || qc.block_height > *core.last_qc_block_num; diff --git a/libraries/chain/include/eosio/chain/block_header_state_legacy.hpp b/libraries/chain/include/eosio/chain/block_header_state_legacy.hpp index d653bca2e5..6f7a3153d9 100644 --- a/libraries/chain/include/eosio/chain/block_header_state_legacy.hpp +++ b/libraries/chain/include/eosio/chain/block_header_state_legacy.hpp @@ -75,8 +75,6 @@ namespace detail { } -using validator_t = const std::function&, const vector&)>; - struct pending_block_header_state_legacy : public detail::block_header_state_legacy_common { protocol_feature_activation_set_ptr prev_activated_protocol_features; detail::schedule_info prev_pending_schedule; @@ -167,7 +165,6 @@ struct block_header_state_legacy : public detail::block_header_state_legacy_comm block_header_state_legacy next( const signed_block_header& h, vector&& additional_signatures, const protocol_feature_set& pfs, - bool hotstuff_activated, validator_t& validator, bool skip_validate_signee = false )const; diff --git a/libraries/chain/include/eosio/chain/block_header_state_utils.hpp b/libraries/chain/include/eosio/chain/block_header_state_utils.hpp index 91237d4709..09c2bb21fa 100644 --- a/libraries/chain/include/eosio/chain/block_header_state_utils.hpp +++ b/libraries/chain/include/eosio/chain/block_header_state_utils.hpp @@ -1,7 +1,9 @@ #pragma once #include "eosio/chain/protocol_feature_manager.hpp" +#include #include + namespace eosio::chain::detail { inline bool is_builtin_activated(const protocol_feature_activation_set_ptr& pfa, @@ -24,4 +26,23 @@ namespace eosio::chain::detail { return producers[index]; } + constexpr auto additional_sigs_eid = additional_block_signatures_extension::extension_id(); + + /** + * Given a complete signed block, extract the validated additional signatures if present; + * + * @param b complete signed block + * @return the list of additional signatures + */ + inline vector extract_additional_signatures(const signed_block_ptr& b) { + auto exts = b->validate_and_extract_extensions(); + + if (exts.count(additional_sigs_eid) > 0) { + auto& additional_sigs = std::get(exts.lower_bound(additional_sigs_eid)->second); + return std::move(additional_sigs.signatures); + } + + return {}; + } + } /// namespace eosio::chain diff --git a/libraries/chain/include/eosio/chain/block_state.hpp b/libraries/chain/include/eosio/chain/block_state.hpp index 6c98e1f2f1..a224095d0d 100644 --- a/libraries/chain/include/eosio/chain/block_state.hpp +++ b/libraries/chain/include/eosio/chain/block_state.hpp @@ -7,32 +7,41 @@ namespace eosio::chain { - struct block_state : public block_header_state { // block_header_state provides parent link - // ------ data members ------------------------------------------------------------- - signed_block_ptr block; - bool validated; // We have executed the block's trxs and verified that action merkle root (block id) matches. - digest_type finalizer_digest; - pending_quorum_certificate pending_qc; // where we accumulate votes we receive - std::optional valid_qc; // qc received from the network +struct block_state : public block_header_state { // block_header_state provides parent link + // ------ data members ------------------------------------------------------------- + signed_block_ptr block; + bool validated; // We have executed the block's trxs and verified that action merkle root (block id) matches. + digest_type strong_digest; // finalizer_digest (strong, cached so we can quickly validate votes) + digest_type weak_digest; // finalizer_digest (weak, cached so we can quickly validate votes) + pending_quorum_certificate pending_qc; // where we accumulate votes we receive + std::optional valid_qc; // best qc received from the network inside block extension - // ------ data members caching information available elsewhere ---------------------- - block_id_type cached_id; // cache of block_header_state::header.calculate_id() (indexed on this field) - header_extension_multimap header_exts; // redundant with the data stored in header - - // ------ functions ----------------------------------------------------------------- - const block_id_type& id() const { return cached_id; } - const block_id_type& previous() const { return block_header_state::previous(); } - uint32_t block_num() const { return block_header_state::block_num(); } - block_timestamp_type timestamp() const { return block_header_state::timestamp(); } - const extensions_type& header_extensions() const { return block_header_state::header.header_extensions; } - bool is_valid() const { return validated; } - void set_valid(bool b) { validated = b; } - uint32_t irreversible_blocknum() const { return 0; } // [greg todo] equivalent of dpos_irreversible_blocknum + // ------ data members caching information available elsewhere ---------------------- + block_id_type cached_id; // cache of block_header_state::header.calculate_id() (indexed on this field) + header_extension_multimap header_exts; // redundant with the data stored in header + + // ------ functions ----------------------------------------------------------------- + const block_id_type& id() const { return cached_id; } + const block_id_type& previous() const { return block_header_state::previous(); } + uint32_t block_num() const { return block_header_state::block_num(); } + block_timestamp_type timestamp() const { return block_header_state::timestamp(); } + const extensions_type& header_extensions() const { return block_header_state::header.header_extensions; } + bool is_valid() const { return validated; } + void set_valid(bool b) { validated = b; } + uint32_t irreversible_blocknum() const { return 0; } // [greg todo] equivalent of dpos_irreversible_blocknum - protocol_feature_activation_set_ptr get_activated_protocol_features() const { return block_header_state::activated_protocol_features; } - deque extract_trxs_metas() { return {}; }; // [greg todo] see impl in block_state_legacy.hpp - }; + protocol_feature_activation_set_ptr get_activated_protocol_features() const { return block_header_state::activated_protocol_features; } + deque extract_trxs_metas() { return {}; }; // [greg todo] see impl in block_state_legacy.hpp + + using bhs_t = block_header_state; + using bhsp_t = block_header_state_ptr; + + block_state() = default; + + block_state(const block_header_state& prev, signed_block_ptr b, const protocol_feature_set& pfs, + const validator_t& validator, bool skip_validate_signee); +}; using block_state_ptr = std::shared_ptr; diff --git a/libraries/chain/include/eosio/chain/block_state_legacy.hpp b/libraries/chain/include/eosio/chain/block_state_legacy.hpp index b8c1876f21..939beaf776 100644 --- a/libraries/chain/include/eosio/chain/block_state_legacy.hpp +++ b/libraries/chain/include/eosio/chain/block_state_legacy.hpp @@ -5,13 +5,15 @@ #include #include -namespace eosio { namespace chain { +namespace eosio::chain { struct block_state_legacy : public block_header_state_legacy { + using bhs_t = block_header_state_legacy; + using bhsp_t = block_header_state_legacy_ptr; + block_state_legacy( const block_header_state_legacy& prev, signed_block_ptr b, const protocol_feature_set& pfs, - bool hotstuff_activated, const validator_t& validator, bool skip_validate_signee ); @@ -74,6 +76,6 @@ namespace eosio { namespace chain { using block_state_legacy_ptr = std::shared_ptr; -} } /// namespace eosio::chain +} /// namespace eosio::chain FC_REFLECT_DERIVED( eosio::chain::block_state_legacy, (eosio::chain::block_header_state_legacy), (block)(validated) ) diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index b563e4a5ca..8798d3bf83 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -51,10 +51,18 @@ namespace eosio::chain { using resource_limits::resource_limits_manager; using apply_handler = std::function; - using fork_database_legacy = fork_database; - using branch_type = typename fork_database_legacy::branch_type; - - using forked_branch_callback = std::function; + template + using branch_type_t = fork_database::branch_type; + + using branch_type_legacy = branch_type_t; + using branch_type = branch_type_t; + + template + using forked_branch_callback_t = std::function&)>; + + using forked_branch_callback_legacy = forked_branch_callback_t; + using forked_branch_callback = forked_branch_callback_t; + // lookup transaction_metadata via supplied function to avoid re-creation using trx_meta_cache_lookup = std::function; @@ -192,6 +200,11 @@ namespace eosio::chain { */ void push_block( block_report& br, const block_state_legacy_ptr& bsp, + const forked_branch_callback_legacy& cb, + const trx_meta_cache_lookup& trx_lookup ); + + void push_block( block_report& br, + const block_state_ptr& bsp, const forked_branch_callback& cb, const trx_meta_cache_lookup& trx_lookup ); diff --git a/libraries/chain/include/eosio/chain/fork_database.hpp b/libraries/chain/include/eosio/chain/fork_database.hpp index b2967d9f67..cdd1d13fab 100644 --- a/libraries/chain/include/eosio/chain/fork_database.hpp +++ b/libraries/chain/include/eosio/chain/fork_database.hpp @@ -7,7 +7,7 @@ namespace eosio::chain { using boost::signals2::signal; - template + template struct fork_database_impl; /** @@ -21,11 +21,13 @@ namespace eosio::chain { * * An internal mutex is used to provide thread-safety. */ - template // either [block_state_legacy_ptr, block_state_ptr], same with block_header_state_ptr + template // either block_state_legacy_ptr or block_state_ptr class fork_database { public: using bs = bsp::element_type; + using bhsp = bs::bhsp_t; using bhs = bhsp::element_type; + using bsp_t = bsp; using branch_type = deque; using branch_type_pair = pair; @@ -97,9 +99,9 @@ namespace eosio::chain { static const uint32_t max_supported_version; private: - unique_ptr> my; + unique_ptr> my; }; - using fork_database_legacy = fork_database; + using fork_database_legacy = fork_database; } /// eosio::chain diff --git a/libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp b/libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp index 4480c06abc..4a28f9eb51 100644 --- a/libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp +++ b/libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp @@ -133,10 +133,11 @@ class unapplied_transaction_queue { } } - void add_forked( const branch_type& forked_branch ) { + template + void add_forked( const BRANCH_TYPE& forked_branch ) { // forked_branch is in reverse order for( auto ritr = forked_branch.rbegin(), rend = forked_branch.rend(); ritr != rend; ++ritr ) { - const block_state_legacy_ptr& bsptr = *ritr; + const auto& bsptr = *ritr; for( auto itr = bsptr->trxs_metas().begin(), end = bsptr->trxs_metas().end(); itr != end; ++itr ) { const auto& trx = *itr; auto insert_itr = queue.insert( { trx, trx_enum_type::forked } ); diff --git a/libraries/testing/include/eosio/testing/tester.hpp b/libraries/testing/include/eosio/testing/tester.hpp index e2297de848..eeabf71061 100644 --- a/libraries/testing/include/eosio/testing/tester.hpp +++ b/libraries/testing/include/eosio/testing/tester.hpp @@ -467,15 +467,15 @@ namespace eosio { namespace testing { } tester(controller::config config, const genesis_state& genesis) { - init(config, genesis); + init(std::move(config), genesis); } tester(controller::config config) { - init(config); + init(std::move(config)); } tester(controller::config config, protocol_feature_set&& pfs, const genesis_state& genesis) { - init(config, std::move(pfs), genesis); + init(std::move(config), std::move(pfs), genesis); } tester(const fc::temp_directory& tempdir, bool use_genesis) { @@ -608,7 +608,7 @@ namespace eosio { namespace testing { auto sb = _produce_block(skip_time, false); auto bsf = validating_node->create_block_state_future( sb->calculate_id(), sb ); controller::block_report br; - validating_node->push_block( br, bsf.get(), forked_branch_callback{}, trx_meta_cache_lookup{} ); + validating_node->push_block( br, bsf.get(), {}, trx_meta_cache_lookup{} ); return sb; } @@ -620,7 +620,7 @@ namespace eosio { namespace testing { void validate_push_block(const signed_block_ptr& sb) { auto bsf = validating_node->create_block_state_future( sb->calculate_id(), sb ); controller::block_report br; - validating_node->push_block( br, bsf.get(), forked_branch_callback{}, trx_meta_cache_lookup{} ); + validating_node->push_block( br, bsf.get(), {}, trx_meta_cache_lookup{} ); } signed_block_ptr produce_empty_block( fc::microseconds skip_time = fc::milliseconds(config::block_interval_ms) )override { @@ -628,7 +628,7 @@ namespace eosio { namespace testing { auto sb = _produce_block(skip_time, true); auto bsf = validating_node->create_block_state_future( sb->calculate_id(), sb ); controller::block_report br; - validating_node->push_block( br, bsf.get(), forked_branch_callback{}, trx_meta_cache_lookup{} ); + validating_node->push_block( br, bsf.get(), {}, trx_meta_cache_lookup{} ); return sb; } diff --git a/libraries/testing/tester.cpp b/libraries/testing/tester.cpp index e002c1507e..b5e35533cb 100644 --- a/libraries/testing/tester.cpp +++ b/libraries/testing/tester.cpp @@ -205,32 +205,32 @@ namespace eosio { namespace testing { } void base_tester::init(controller::config config, const snapshot_reader_ptr& snapshot) { - cfg = config; + cfg = std::move(config); open(snapshot); } void base_tester::init(controller::config config, const genesis_state& genesis) { - cfg = config; + cfg = std::move(config); open(genesis); } void base_tester::init(controller::config config) { - cfg = config; + cfg = std::move(config); open(default_genesis().compute_chain_id()); } void base_tester::init(controller::config config, protocol_feature_set&& pfs, const snapshot_reader_ptr& snapshot) { - cfg = config; + cfg = std::move(config); open(std::move(pfs), snapshot); } void base_tester::init(controller::config config, protocol_feature_set&& pfs, const genesis_state& genesis) { - cfg = config; + cfg = std::move(config); open(std::move(pfs), genesis); } void base_tester::init(controller::config config, protocol_feature_set&& pfs) { - cfg = config; + cfg = std::move(config); open(std::move(pfs), default_genesis().compute_chain_id()); } @@ -379,7 +379,7 @@ namespace eosio { namespace testing { auto bsf = control->create_block_state_future(b->calculate_id(), b); unapplied_transactions.add_aborted( control->abort_block() ); controller::block_report br; - control->push_block( br, bsf.get(), [this]( const branch_type& forked_branch ) { + control->push_block( br, bsf.get(), [this]( const branch_type_legacy& forked_branch ) { unapplied_transactions.add_forked( forked_branch ); }, [this]( const transaction_id_type& id ) { return unapplied_transactions.get_trx( id ); @@ -1118,7 +1118,7 @@ namespace eosio { namespace testing { auto bsf = b.control->create_block_state_future( block->calculate_id(), block ); b.control->abort_block(); controller::block_report br; - b.control->push_block(br, bsf.get(), forked_branch_callback{}, trx_meta_cache_lookup{}); //, eosio::chain::validation_steps::created_block); + b.control->push_block(br, bsf.get(), {}, trx_meta_cache_lookup{}); //, eosio::chain::validation_steps::created_block); } } }; diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index d64884a2a4..326086c465 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -715,7 +715,7 @@ class producer_plugin_impl : public std::enable_shared_from_thiscreate_block_state_future( copy_b->calculate_id(), copy_b ); validator.control->abort_block(); controller::block_report br; - BOOST_REQUIRE_EXCEPTION(validator.control->push_block( br, bsf.get(), forked_branch_callback{}, trx_meta_cache_lookup{} ), fc::exception , + BOOST_REQUIRE_EXCEPTION(validator.control->push_block( br, bsf.get(), {}, trx_meta_cache_lookup{} ), fc::exception , [] (const fc::exception &e)->bool { return e.code() == account_name_exists_exception::code_value ; }) ; @@ -86,7 +86,7 @@ BOOST_AUTO_TEST_CASE(block_with_invalid_tx_mroot_test) auto bsf = validator.control->create_block_state_future( copy_b->calculate_id(), copy_b ); validator.control->abort_block(); controller::block_report br; - BOOST_REQUIRE_EXCEPTION(validator.control->push_block( br, bsf.get(), forked_branch_callback{}, trx_meta_cache_lookup{} ), fc::exception, + BOOST_REQUIRE_EXCEPTION(validator.control->push_block( br, bsf.get(), {}, trx_meta_cache_lookup{} ), fc::exception, [] (const fc::exception &e)->bool { return e.code() == block_validate_exception::code_value && e.to_detail_string().find("invalid block transaction merkle root") != std::string::npos; diff --git a/unittests/forked_tests.cpp b/unittests/forked_tests.cpp index dc135a509c..2392663921 100644 --- a/unittests/forked_tests.cpp +++ b/unittests/forked_tests.cpp @@ -269,7 +269,7 @@ BOOST_AUTO_TEST_CASE( forking ) try { auto bad_block_bsf = c.control->create_block_state_future( bad_id, std::make_shared(std::move(bad_block)) ); c.control->abort_block(); controller::block_report br; - BOOST_REQUIRE_EXCEPTION(c.control->push_block( br, bad_block_bsf.get(), forked_branch_callback{}, trx_meta_cache_lookup{} ), fc::exception, + BOOST_REQUIRE_EXCEPTION(c.control->push_block( br, bad_block_bsf.get(), {}, trx_meta_cache_lookup{} ), fc::exception, [] (const fc::exception &ex)->bool { return ex.to_detail_string().find("block signed by unexpected key") != std::string::npos; }); diff --git a/unittests/protocol_feature_tests.cpp b/unittests/protocol_feature_tests.cpp index 09df3ebdcf..45c76ab339 100644 --- a/unittests/protocol_feature_tests.cpp +++ b/unittests/protocol_feature_tests.cpp @@ -2257,7 +2257,7 @@ BOOST_AUTO_TEST_CASE( block_validation_after_stage_1_test ) { try { controller::block_report br; // The block is invalidated - BOOST_REQUIRE_EXCEPTION(tester2.control->push_block( br, bsf.get(), forked_branch_callback{}, trx_meta_cache_lookup{} ), + BOOST_REQUIRE_EXCEPTION(tester2.control->push_block( br, bsf.get(), {}, trx_meta_cache_lookup{} ), fc::exception, fc_exception_message_starts_with("transaction cannot be delayed") ); diff --git a/unittests/unapplied_transaction_queue_tests.cpp b/unittests/unapplied_transaction_queue_tests.cpp index 5d91202698..d2f683dc32 100644 --- a/unittests/unapplied_transaction_queue_tests.cpp +++ b/unittests/unapplied_transaction_queue_tests.cpp @@ -136,7 +136,7 @@ BOOST_AUTO_TEST_CASE( unapplied_transaction_queue_test ) try { auto bs1 = create_test_block_state( { trx1, trx2 } ); auto bs2 = create_test_block_state( { trx3, trx4, trx5 } ); auto bs3 = create_test_block_state( { trx6 } ); - q.add_forked( { bs3, bs2, bs1, bs1 } ); // bs1 duplicate ignored + q.add_forked( branch_type_legacy{ bs3, bs2, bs1, bs1 } ); // bs1 duplicate ignored BOOST_CHECK( q.size() == 6u ); BOOST_REQUIRE( next( q ) == trx1 ); BOOST_CHECK( q.size() == 5u ); @@ -155,9 +155,9 @@ BOOST_AUTO_TEST_CASE( unapplied_transaction_queue_test ) try { // fifo forked auto bs4 = create_test_block_state( { trx7 } ); - q.add_forked( { bs1 } ); - q.add_forked( { bs3, bs2 } ); - q.add_forked( { bs4 } ); + q.add_forked( branch_type_legacy{ bs1 } ); + q.add_forked( branch_type_legacy{ bs3, bs2 } ); + q.add_forked( branch_type_legacy{ bs4 } ); BOOST_CHECK( q.size() == 7u ); BOOST_REQUIRE( next( q ) == trx1 ); BOOST_CHECK( q.size() == 6u ); @@ -189,10 +189,10 @@ BOOST_AUTO_TEST_CASE( unapplied_transaction_queue_test ) try { // fifo forked, multi forks auto bs5 = create_test_block_state( { trx11, trx12, trx13 } ); auto bs6 = create_test_block_state( { trx11, trx15 } ); - q.add_forked( { bs3, bs2, bs1 } ); - q.add_forked( { bs4 } ); - q.add_forked( { bs3, bs2 } ); // dups ignored - q.add_forked( { bs6, bs5 } ); + q.add_forked( branch_type_legacy{ bs3, bs2, bs1 } ); + q.add_forked( branch_type_legacy{ bs4 } ); + q.add_forked( branch_type_legacy{ bs3, bs2 } ); // dups ignored + q.add_forked( branch_type_legacy{ bs6, bs5 } ); BOOST_CHECK_EQUAL( q.size(), 11u ); BOOST_REQUIRE( next( q ) == trx1 ); BOOST_CHECK( q.size() == 10u ); @@ -220,10 +220,10 @@ BOOST_AUTO_TEST_CASE( unapplied_transaction_queue_test ) try { BOOST_CHECK( q.empty() ); // altogether, order fifo: forked, aborted - q.add_forked( { bs3, bs2, bs1 } ); + q.add_forked( branch_type_legacy{ bs3, bs2, bs1 } ); q.add_aborted( { trx9, trx14 } ); q.add_aborted( { trx18, trx19 } ); - q.add_forked( { bs6, bs5, bs4 } ); + q.add_forked( branch_type_legacy{ bs6, bs5, bs4 } ); // verify order verify_order( q, q.begin(), 15 ); // verify type order @@ -289,7 +289,7 @@ BOOST_AUTO_TEST_CASE( unapplied_transaction_queue_test ) try { BOOST_REQUIRE( next( q ) == trx22 ); BOOST_CHECK( q.empty() ); - q.add_forked( { bs3, bs2, bs1 } ); + q.add_forked( branch_type_legacy{ bs3, bs2, bs1 } ); q.add_aborted( { trx9, trx11 } ); q.clear(); BOOST_CHECK( q.empty() ); From d2af8636c116a66d530af34bc3565f73b7ed29eb Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 11 Jan 2024 15:53:22 -0600 Subject: [PATCH 05/36] GH-1980 Implement proposer policy change --- libraries/chain/block_header_state.cpp | 23 +- libraries/chain/controller.cpp | 239 ++++++++++++------ .../eosio/chain/block_header_state.hpp | 16 +- .../eosio/chain/block_header_state_utils.hpp | 6 +- .../eosio/chain/block_state_legacy.hpp | 2 +- .../chain/include/eosio/chain/controller.hpp | 9 +- .../hotstuff/instant_finality_extension.hpp | 8 +- plugins/chain_plugin/chain_plugin.cpp | 6 +- plugins/net_plugin/net_plugin.cpp | 4 +- unittests/block_header_tests.cpp | 8 +- unittests/chain_tests.cpp | 10 +- unittests/producer_schedule_tests.cpp | 115 +++++---- 12 files changed, 290 insertions(+), 156 deletions(-) diff --git a/libraries/chain/block_header_state.cpp b/libraries/chain/block_header_state.cpp index 4a6161269c..d52782411d 100644 --- a/libraries/chain/block_header_state.cpp +++ b/libraries/chain/block_header_state.cpp @@ -1,13 +1,14 @@ #include #include #include +#include #include #include namespace eosio::chain { producer_authority block_header_state::get_scheduled_producer(block_timestamp_type t) const { - return detail::get_scheduled_producer(proposer_policy->proposer_schedule.producers, t); + return detail::get_scheduled_producer(active_proposer_policy->proposer_schedule.producers, t); } #warning Add last_proposed_finalizer_policy_generation to snapshot_block_header_state_v3, see header file TODO @@ -61,13 +62,29 @@ block_header_state block_header_state::next(block_header_state_input& input) con result.header = block_header { .timestamp = input.timestamp, // [greg todo] do we have to do the slot++ stuff from the legacy version? .producer = input.producer, + .confirmed = hs_block_confirmed, // todo: consider 0 instead .previous = input.parent_id, .transaction_mroot = input.transaction_mroot, .action_mroot = input.action_mroot, - //.schedule_version = ?, [greg todo] - //.new_producers = ? [greg todo] + .schedule_version = header.schedule_version }; + if(!proposer_policies.empty()) { + auto it = proposer_policies.begin(); + if (it->first <= input.timestamp) { + result.active_proposer_policy = it->second; + result.header.schedule_version = header.schedule_version + 1; + result.active_proposer_policy->proposer_schedule.version = result.header.schedule_version; + result.proposer_policies = { ++it, proposer_policies.end() }; + } else { + result.proposer_policies = proposer_policies; + } + } + if (input.new_proposer_policy) { + // called when assembling the block + result.proposer_policies[result.header.timestamp] = input.new_proposer_policy; + } + // core // ---- if (input.qc_info) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index d5b2857a14..d52673bb2e 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -42,6 +43,7 @@ #include #include #include +#include namespace eosio::chain { @@ -184,9 +186,14 @@ struct block_data_t { const producer_authority_schedule& head_active_schedule_auth() { return std::visit([](const auto& bd) -> const producer_authority_schedule& { return bd.head->active_schedule_auth(); }, v); } - - const producer_authority_schedule& head_pending_schedule_auth() { - return std::visit([](const auto& bd) -> const producer_authority_schedule& { return bd.head->pending_schedule_auth(); }, v); + + const producer_authority_schedule* head_pending_schedule_auth_legacy() { + return std::visit(overloaded{ + [](const block_data_legacy_t& bd) -> const producer_authority_schedule* { + return bd.head->pending_schedule_auth(); + }, + [](const block_data_new_t&) -> const producer_authority_schedule* { return nullptr; } + }, v); } const block_id_type& head_block_id() const { @@ -201,6 +208,29 @@ struct block_data_t { return std::visit([](const auto& bd) -> const signed_block_ptr& { return bd.head->block; }, v); } + void replace_producer_keys( const public_key_type& key ) { + ilog("Replace producer keys with ${k}", ("k", key)); + + std::visit( + overloaded{ + [&](const block_data_legacy_t& bd) { + auto version = bd.head->pending_schedule.schedule.version; + bd.head->pending_schedule = {}; + bd.head->pending_schedule.schedule.version = version; + for (auto& prod: bd.head->active_schedule.producers ) { + ilog("${n}", ("n", prod.producer_name)); + std::visit([&](auto &auth) { + auth.threshold = 1; + auth.keys = {key_weight{key, 1}}; + }, prod.authority); + } + }, + [](const block_data_new_t&) { + // TODO IF: add instant-finality implementation, will need to replace finalizers as well + } + }, v); + } + // --------------- access fork_db head ---------------------------------------------------------------------- bool fork_db_has_head() const { return std::visit([&](const auto& bd) { return !!bd.fork_db_head(); }, v); @@ -300,8 +330,20 @@ struct completed_block { return std::visit([](const auto& bsp) -> const producer_authority_schedule& { return bsp->active_schedule_auth(); }, bsp); } - const producer_authority_schedule& pending_producers() const { - return std::visit([](const auto& bsp) -> const producer_authority_schedule& { return bsp->pending_schedule_auth();}, bsp); + const producer_authority_schedule* next_producers() const { + return std::visit(overloaded{ + [](const block_state_legacy_ptr& bsp) -> const producer_authority_schedule* { return bsp->pending_schedule_auth();}, + [](const block_state_ptr& bsp) -> const producer_authority_schedule* { + return bsp->proposer_policies.empty() ? nullptr : &bsp->proposer_policies.begin()->second->proposer_schedule; + } + }, bsp); + } + + const producer_authority_schedule* pending_producers_legacy() const { + return std::visit(overloaded{ + [](const block_state_legacy_ptr& bsp) -> const producer_authority_schedule* { return &bsp->pending_schedule.schedule; }, + [](const block_state_ptr&) -> const producer_authority_schedule* { return nullptr; } + }, bsp); } bool is_protocol_feature_activated(const digest_type& digest) const { @@ -416,16 +458,28 @@ struct assembled_block { v); } - using opt_pas = const std::optional; + const producer_authority_schedule* next_producers() const { + return std::visit(overloaded{ + [](const assembled_block_dpos& ab) -> const producer_authority_schedule* { + return ab.new_producer_authority_cache.has_value() ? &ab.new_producer_authority_cache.value() : nullptr; + }, + [](const assembled_block_if& ab) -> const producer_authority_schedule* { + return ab.bhs.proposer_policies.empty() ? nullptr : &ab.bhs.proposer_policies.begin()->second->proposer_schedule; + } + }, + v); + } - opt_pas& pending_producers() const { - return std::visit( - overloaded{[](const assembled_block_dpos& ab) -> opt_pas& { return ab.new_producer_authority_cache; }, - [](const assembled_block_if& ab) -> opt_pas& { - static opt_pas empty; - return empty; // [greg todo] - }}, - v); + const producer_authority_schedule* pending_producers_legacy() const { + return std::visit(overloaded{ + [](const assembled_block_dpos& ab) -> const producer_authority_schedule* { + return ab.new_producer_authority_cache.has_value() ? &ab.new_producer_authority_cache.value() : nullptr; + }, + [](const assembled_block_if&) -> const producer_authority_schedule* { + return nullptr; + } + }, + v); } const block_signing_authority& pending_block_signing_authority() const { @@ -532,7 +586,7 @@ struct building_block { const uint32_t block_num; // Cached: parent.block_num() + 1 // Members below (as well as non-const members of building_block_common) start from initial state and are mutated as the block is built. - std::optional new_proposer_policy; + std::shared_ptr new_proposer_policy; building_block_if(const block_header_state& parent, const building_block_input& input) : building_block_common(input.new_protocol_feature_activations) @@ -541,7 +595,7 @@ struct building_block { , timestamp(input.timestamp) , active_producer_authority{input.producer, [&]() -> block_signing_authority { - const auto& pas = parent.proposer_policy->proposer_schedule; + const auto& pas = parent.active_proposer_policy->proposer_schedule; for (const auto& pa : pas.producers) if (pa.producer_name == input.producer) return pa.authority; @@ -549,7 +603,7 @@ struct building_block { return {}; }()} , prev_activated_protocol_features(parent.activated_protocol_features) - , active_proposer_policy(parent.proposer_policy) + , active_proposer_policy(parent.active_proposer_policy) , block_num(parent.block_num() + 1) {} bool is_protocol_feature_activated(const digest_type& digest) const { @@ -558,6 +612,14 @@ struct building_block { uint32_t get_block_num() const { return block_num; } + uint32_t get_next_proposer_schedule_version() const { + if (!parent.proposer_policies.empty()) { + return (--parent.proposer_policies.end())->second->proposer_schedule.version + 1; + } + assert(active_proposer_policy); + return active_proposer_policy->proposer_schedule.version + 1; + } + }; std::variant v; @@ -599,6 +661,19 @@ struct building_block { std::visit([&](auto& bb) { bb.new_finalizer_policy = fin_pol; }, v); } + int64_t set_proposed_producers( std::vector producers ) { + return std::visit( + overloaded{[](building_block_dpos&) -> int64_t { return -1; }, + [&](building_block_if& bb) -> int64_t { + bb.new_proposer_policy = std::make_shared(); + bb.new_proposer_policy->active_time = detail::get_next_next_round_block_time(bb.timestamp); + bb.new_proposer_policy->proposer_schedule.producers = std::move(producers); + bb.new_proposer_policy->proposer_schedule.version = bb.get_next_proposer_schedule_version(); + return bb.new_proposer_policy->proposer_schedule.version; + }}, + v); + } + deque extract_trx_metas() { return std::visit([](auto& bb) { return std::move(bb.pending_trx_metas); }, v); } @@ -681,15 +756,30 @@ struct building_block { v); } - const producer_authority_schedule& pending_producers() const { - return std::visit(overloaded{[](const building_block_dpos& bb) -> const producer_authority_schedule& { + const producer_authority_schedule* next_producers() const { + return std::visit(overloaded{[](const building_block_dpos& bb) -> const producer_authority_schedule* { if (bb.new_pending_producer_schedule) - return *bb.new_pending_producer_schedule; - return bb.pending_block_header_state.prev_pending_schedule.schedule; + return &bb.new_pending_producer_schedule.value(); + return &bb.pending_block_header_state.prev_pending_schedule.schedule; }, - [](const building_block_if& bb) -> const producer_authority_schedule& { - static producer_authority_schedule empty; - return empty; // [greg todo] + [](const building_block_if& bb) -> const producer_authority_schedule* { + if (!bb.parent.proposer_policies.empty()) + return &bb.parent.proposer_policies.begin()->second->proposer_schedule; + if (bb.new_proposer_policy) + return &bb.new_proposer_policy->proposer_schedule; + return nullptr; + }}, + v); + } + + const producer_authority_schedule* pending_producers_legacy() const { + return std::visit(overloaded{[](const building_block_dpos& bb) -> const producer_authority_schedule* { + if (bb.new_pending_producer_schedule) + return &bb.new_pending_producer_schedule.value(); + return &bb.pending_block_header_state.prev_pending_schedule.schedule; + }, + [](const building_block_if&) -> const producer_authority_schedule* { + return nullptr; }}, v); } @@ -761,7 +851,7 @@ struct building_block { }; block_header_state_input bhs_input{ - bb_input, transaction_mroot, action_mroot, bb.new_proposer_policy, bb.new_finalizer_policy, + bb_input, transaction_mroot, action_mroot, std::move(bb.new_proposer_policy), std::move(bb.new_finalizer_policy), qc_data ? qc_data->qc_info : std::optional{} }; assembled_block::assembled_block_if ab{bb.active_producer_authority, bb.parent.next(bhs_input), @@ -830,18 +920,19 @@ struct pending_state { [](const auto& stage) -> const producer_authority_schedule& { return stage.active_producers(); }, _block_stage); } - -#if 0 - // [greg todo] maybe we don't need this and we can have the implementation in controller::pending_producers() - const producer_authority_schedule& pending_producers() const { + + const producer_authority_schedule* pending_producers_legacy() const { return std::visit( - overloaded{ - [](const building_block& bb) -> const producer_authority_schedule& { return bb.pending_producers(); }, - [](const assembled_block& ab) -> const producer_authority_schedule& { return ab.pending_producers(); }, - [](const completed_block& cb) -> const producer_authority_schedule& { return cb.pending_producers(); }}, + [](const auto& stage) -> const producer_authority_schedule* { return stage.pending_producers_legacy(); }, _block_stage); } -#endif + + const producer_authority_schedule* next_producers()const { + return std::visit( + [](const auto& stage) -> const producer_authority_schedule* { return stage.next_producers(); }, + _block_stage); + } + }; struct controller_impl { @@ -896,6 +987,8 @@ struct controller_impl { map< account_name, map > apply_handlers; unordered_map< builtin_protocol_feature_t, std::function, enum_hash > protocol_feature_activation_handlers; + int64_t set_proposed_producers( vector producers ); + int64_t set_proposed_producers_legacy( vector producers ); void pop_block() { uint32_t prev_block_num = block_data.pop_block(); @@ -3932,10 +4025,28 @@ void controller::write_snapshot( const snapshot_writer_ptr& snapshot ) { } int64_t controller::set_proposed_producers( vector producers ) { - const auto& gpo = get_global_properties(); + assert(my->pending); + if (my->pending->is_dpos()) { + return my->set_proposed_producers_legacy(std::move(producers)); + } else { + return my->set_proposed_producers(std::move(producers)); + } +} + +int64_t controller_impl::set_proposed_producers( vector producers ) { + // TODO: zero out gpo.proposed_schedule_block_num and gpo.proposed_schedule on instant finality enabled + if (producers.empty()) + return -1; + + auto& bb = std::get(pending->_block_stage); + return bb.set_proposed_producers(std::move(producers)); +} + +int64_t controller_impl::set_proposed_producers_legacy( vector producers ) { + const auto& gpo = self.get_global_properties(); auto cur_block_num = head_block_num() + 1; - if( producers.size() == 0 && is_builtin_activated( builtin_protocol_feature_t::disallow_empty_producer_schedule ) ) { + if( producers.size() == 0 && self.is_builtin_activated( builtin_protocol_feature_t::disallow_empty_producer_schedule ) ) { return -1; } @@ -3953,17 +4064,18 @@ int64_t controller::set_proposed_producers( vector producers decltype(sch.producers.cend()) end; decltype(end) begin; - const auto& pending_sch = pending_producers(); + const auto* pending_sch = self.pending_producers_legacy(); + assert(pending_sch); // can't be null during dpos - if( pending_sch.producers.size() == 0 ) { - const auto& active_sch = active_producers(); + if( pending_sch->producers.size() == 0 ) { + const auto& active_sch = self.active_producers(); begin = active_sch.producers.begin(); end = active_sch.producers.end(); sch.version = active_sch.version + 1; } else { - begin = pending_sch.producers.begin(); - end = pending_sch.producers.end(); - sch.version = pending_sch.version + 1; + begin = pending_sch->producers.begin(); + end = pending_sch->producers.end(); + sch.version = pending_sch->version + 1; } if( std::equal( producers.begin(), producers.end(), begin, end ) ) @@ -3975,7 +4087,7 @@ int64_t controller::set_proposed_producers( vector producers ilog( "proposed producer schedule with version ${v}", ("v", version) ); - my->db.modify( gpo, [&]( auto& gp ) { + db.modify( gpo, [&]( auto& gp ) { gp.proposed_schedule_block_num = cur_block_num; gp.proposed_schedule = sch; }); @@ -4022,25 +4134,14 @@ const producer_authority_schedule& controller::head_active_producers()const { return my->block_data.head_active_schedule_auth(); } -const producer_authority_schedule& controller::pending_producers()const { - if( !(my->pending) ) - return my->block_data.head_pending_schedule_auth(); // [greg todo] implement pending_producers correctly for IF mode - - if( std::holds_alternative(my->pending->_block_stage) ) - return std::get(my->pending->_block_stage).pending_producers(); - - if( std::holds_alternative(my->pending->_block_stage) ) { - const auto& pp = std::get(my->pending->_block_stage).pending_producers(); - if( pp ) { - return *pp; - } - } +const producer_authority_schedule* controller::pending_producers_legacy()const { + if( !(my->pending) ) + return my->block_data.head_pending_schedule_auth_legacy(); - const auto& bb = std::get(my->pending->_block_stage); - return bb.pending_producers(); + return my->pending->pending_producers_legacy(); } -std::optional controller::proposed_producers()const { +std::optional controller::proposed_producers_legacy()const { const auto& gpo = get_global_properties(); if( !gpo.proposed_schedule_block_num ) return std::optional(); @@ -4048,6 +4149,10 @@ std::optional controller::proposed_producers()const return producer_authority_schedule::from_shared(gpo.proposed_schedule); } +const producer_authority_schedule* controller::next_producers()const { + return my->pending->next_producers(); +} + bool controller::light_validation_allowed() const { if (!my->pending || my->in_trx_requiring_checks) { return false; @@ -4391,26 +4496,14 @@ std::optional controller::extract_chain_id_from_db( const path& s void controller::replace_producer_keys( const public_key_type& key ) { ilog("Replace producer keys with ${k}", ("k", key)); + // can be done even after instant-finality, will be no-op then mutable_db().modify( db().get(), [&]( auto& gp ) { gp.proposed_schedule_block_num = {}; gp.proposed_schedule.version = 0; gp.proposed_schedule.producers.clear(); }); - - auto replace_keys = [&key](auto& fork_db, auto& head) { - auto version = head->pending_schedule.schedule.version; - head->pending_schedule = {}; - head->pending_schedule.schedule.version = version; - for (auto& prod: head->active_schedule.producers ) { - ilog("${n}", ("n", prod.producer_name)); - std::visit([&](auto &auth) { - auth.threshold = 1; - auth.keys = {key_weight{key, 1}}; - }, prod.authority); - } - }; - my->block_data.apply_dpos(replace_keys); // [greg todo]: make it work with `apply` instead of `apply_dpos` + my->block_data.replace_producer_keys(key); } void controller::replace_account_keys( name account, name permission, const public_key_type& key ) { diff --git a/libraries/chain/include/eosio/chain/block_header_state.hpp b/libraries/chain/include/eosio/chain/block_header_state.hpp index da9e525041..c565cbf273 100644 --- a/libraries/chain/include/eosio/chain/block_header_state.hpp +++ b/libraries/chain/include/eosio/chain/block_header_state.hpp @@ -29,7 +29,7 @@ struct qc_data_t { struct block_header_state_input : public building_block_input { digest_type transaction_mroot; // Comes from std::get(building_block::trx_mroot_or_receipt_digests) digest_type action_mroot; // Compute root from building_block::action_receipt_digests - std::optional new_proposer_policy; // Comes from building_block::new_proposer_policy + std::shared_ptr new_proposer_policy; // Comes from building_block::new_proposer_policy std::optional new_finalizer_policy; // Comes from building_block::new_finalizer_policy std::optional qc_info; // Comes from traversing branch from parent and calling get_best_qc() // assert(qc->block_num <= num_from_id(previous)); @@ -54,10 +54,11 @@ struct block_header_state { incremental_merkle_tree proposal_mtree; incremental_merkle_tree finality_mtree; - finalizer_policy_ptr finalizer_policy; // finalizer set + threshold + generation, supports `digest()` - proposer_policy_ptr proposer_policy; // producer authority schedule, supports `digest()` + finalizer_policy_ptr active_finalizer_policy; // finalizer set + threshold + generation, supports `digest()` + proposer_policy_ptr active_proposer_policy; // producer authority schedule, supports `digest()` - flat_map proposer_policies; + // block time when proposer_policy will become active + flat_map proposer_policies; flat_map finalizer_policies; // ------ functions ----------------------------------------------------------------- @@ -66,9 +67,8 @@ struct block_header_state { account_name producer() const { return header.producer; } const block_id_type& previous() const { return header.previous; } uint32_t block_num() const { return block_header::num_from_id(previous()) + 1; } - const producer_authority_schedule& active_schedule_auth() const { return proposer_policy->proposer_schedule; } - const producer_authority_schedule& pending_schedule_auth() const { return proposer_policies.rbegin()->second->proposer_schedule; } // [greg todo] - + const producer_authority_schedule& active_schedule_auth() const { return active_proposer_policy->proposer_schedule; } + block_header_state next(block_header_state_input& data) const; // block descending from this need the provided qc in the block extension @@ -77,10 +77,8 @@ struct block_header_state { } flat_set get_activated_protocol_features() const { return activated_protocol_features->protocol_features; } - detail::schedule_info prev_pending_schedule() const; producer_authority get_scheduled_producer(block_timestamp_type t) const; uint32_t active_schedule_version() const; - std::optional& new_pending_producer_schedule() { static std::optional x; return x; } // [greg todo] signed_block_header make_block_header(const checksum256_type& transaction_mroot, const checksum256_type& action_mroot, const std::optional& new_producers, diff --git a/libraries/chain/include/eosio/chain/block_header_state_utils.hpp b/libraries/chain/include/eosio/chain/block_header_state_utils.hpp index 91237d4709..b7dabf9a35 100644 --- a/libraries/chain/include/eosio/chain/block_header_state_utils.hpp +++ b/libraries/chain/include/eosio/chain/block_header_state_utils.hpp @@ -12,10 +12,10 @@ namespace eosio::chain::detail { return digest && protocol_features.find(*digest) != protocol_features.end(); } - inline uint32_t get_next_next_round_block_num(block_timestamp_type t, uint32_t block_num) { + inline block_timestamp_type get_next_next_round_block_time( block_timestamp_type t) { auto index = t.slot % config::producer_repetitions; // current index in current round - // (increment to the end of this round ) + next round - return block_num + (config::producer_repetitions - index) + config::producer_repetitions; + // (increment to the end of this round ) + next round + return block_timestamp_type{t.slot + (config::producer_repetitions - index) + config::producer_repetitions}; } inline producer_authority get_scheduled_producer(const vector& producers, block_timestamp_type t) { diff --git a/libraries/chain/include/eosio/chain/block_state_legacy.hpp b/libraries/chain/include/eosio/chain/block_state_legacy.hpp index b8c1876f21..f6c0ab35d5 100644 --- a/libraries/chain/include/eosio/chain/block_state_legacy.hpp +++ b/libraries/chain/include/eosio/chain/block_state_legacy.hpp @@ -42,7 +42,7 @@ namespace eosio { namespace chain { protocol_feature_activation_set_ptr get_activated_protocol_features() const { return activated_protocol_features; } const producer_authority_schedule& active_schedule_auth() const { return block_header_state_legacy_common::active_schedule; } - const producer_authority_schedule& pending_schedule_auth() const { return block_header_state_legacy::pending_schedule.schedule; } + const producer_authority_schedule* pending_schedule_auth() const { return &block_header_state_legacy::pending_schedule.schedule; } const deque& trxs_metas() const { return _cached_trxs; } diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index b563e4a5ca..5fae42731a 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -250,8 +250,13 @@ namespace eosio::chain { const producer_authority_schedule& active_producers()const; const producer_authority_schedule& head_active_producers()const; - const producer_authority_schedule& pending_producers()const; - std::optional proposed_producers()const; + // pending for pre-instant-finality, next proposed that will take affect, null if none are pending/proposed + const producer_authority_schedule* next_producers()const; + // post-instant-finality this always returns empty std::optional + std::optional proposed_producers_legacy()const; + // pre-instant-finality this always returns a valid producer_authority_schedule + // post-instant-finality this always returns nullptr + const producer_authority_schedule* pending_producers_legacy()const; // Called by qc_chain to indicate the current irreversible block num // After hotstuff is activated, this should be called on startup by qc_chain diff --git a/libraries/chain/include/eosio/chain/hotstuff/instant_finality_extension.hpp b/libraries/chain/include/eosio/chain/hotstuff/instant_finality_extension.hpp index a663e16f9d..a535414285 100644 --- a/libraries/chain/include/eosio/chain/hotstuff/instant_finality_extension.hpp +++ b/libraries/chain/include/eosio/chain/hotstuff/instant_finality_extension.hpp @@ -17,7 +17,7 @@ struct instant_finality_extension : fc::reflect_init { instant_finality_extension() = default; instant_finality_extension(std::optional qc_info, std::optional new_finalizer_policy, - std::optional new_proposer_policy) : + std::shared_ptr new_proposer_policy) : qc_info(qc_info), new_finalizer_policy(std::move(new_finalizer_policy)), new_proposer_policy(std::move(new_proposer_policy)) @@ -25,9 +25,9 @@ struct instant_finality_extension : fc::reflect_init { void reflector_init(); - std::optional qc_info; - std::optional new_finalizer_policy; - std::optional new_proposer_policy; + std::optional qc_info; + std::optional new_finalizer_policy; + std::shared_ptr new_proposer_policy; }; } /// eosio::chain diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp index ee3a564499..56761924a6 100644 --- a/plugins/chain_plugin/chain_plugin.cpp +++ b/plugins/chain_plugin/chain_plugin.cpp @@ -1800,9 +1800,9 @@ read_only::get_producers( const read_only::get_producers_params& params, const f read_only::get_producer_schedule_result read_only::get_producer_schedule( const read_only::get_producer_schedule_params& p, const fc::time_point& ) const { read_only::get_producer_schedule_result result; to_variant(db.active_producers(), result.active); - if(!db.pending_producers().producers.empty()) - to_variant(db.pending_producers(), result.pending); - auto proposed = db.proposed_producers(); + if (const auto* pending = db.next_producers()) // not applicable for instant-finality + to_variant(*pending, result.pending); + auto proposed = db.proposed_producers_legacy(); // empty for instant-finality if(proposed && !proposed->producers.empty()) to_variant(*proposed, result.proposed); return result; diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index bf8f7d0a07..967686ef4f 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -3929,7 +3929,9 @@ namespace eosio { } void net_plugin_impl::on_accepted_block() { - on_pending_schedule(chain_plug->chain().pending_producers()); + if (const auto* next_producers = chain_plug->chain().next_producers()) { + on_pending_schedule(*next_producers); + } on_active_schedule(chain_plug->chain().active_producers()); } diff --git a/unittests/block_header_tests.cpp b/unittests/block_header_tests.cpp index 48e7e39abb..ae7d31421c 100644 --- a/unittests/block_header_tests.cpp +++ b/unittests/block_header_tests.cpp @@ -24,7 +24,7 @@ BOOST_AUTO_TEST_CASE(instant_finality_extension_with_empty_values_test) emplace_extension( header.header_extensions, instant_finality_extension::extension_id(), - fc::raw::pack( instant_finality_extension{qc_info_t{last_qc_block_num, is_last_qc_strong}, std::optional{}, std::optional{}} ) + fc::raw::pack( instant_finality_extension{qc_info_t{last_qc_block_num, is_last_qc_strong}, std::optional{}, std::shared_ptr{}} ) ); std::optional ext = header.extract_header_extension(instant_finality_extension::extension_id()); @@ -45,7 +45,7 @@ BOOST_AUTO_TEST_CASE(instant_finality_extension_uniqueness_test) emplace_extension( header.header_extensions, instant_finality_extension::extension_id(), - fc::raw::pack( instant_finality_extension{qc_info_t{0, false}, {std::nullopt}, {std::nullopt}} ) + fc::raw::pack( instant_finality_extension{qc_info_t{0, false}, {std::nullopt}, std::shared_ptr{}} ) ); std::vector finalizers { {"test description", 50, fc::crypto::blslib::bls_public_key{"PUB_BLS_MPPeebAPxt/ibL2XPuZVGpADjGn+YEVPPoYmTZeBD6Ok2E19M8SnmDGSdZBf2qwSuJim+8H83EsTpEn3OiStWBiFeJYfVRLlEsZuSF0SYYwtVteY48n+KeE1IWzlSAkSyBqiGA==" }} }; @@ -54,7 +54,7 @@ BOOST_AUTO_TEST_CASE(instant_finality_extension_uniqueness_test) new_finalizer_policy.threshold = 100; new_finalizer_policy.finalizers = finalizers; - proposer_policy new_proposer_policy {1, block_timestamp_type{200}, {} }; + proposer_policy_ptr new_proposer_policy = std::make_shared(1, block_timestamp_type{200}, producer_authority_schedule{} ); emplace_extension( header.header_extensions, @@ -78,7 +78,7 @@ BOOST_AUTO_TEST_CASE(instant_finality_extension_with_values_test) new_finalizer_policy.threshold = 100; new_finalizer_policy.finalizers = finalizers; - proposer_policy new_proposer_policy {1, block_timestamp_type{200}, {} }; + proposer_policy_ptr new_proposer_policy = std::make_shared(1, block_timestamp_type{200}, producer_authority_schedule{} ); emplace_extension( header.header_extensions, diff --git a/unittests/chain_tests.cpp b/unittests/chain_tests.cpp index 40e49e5a04..45ad505115 100644 --- a/unittests/chain_tests.cpp +++ b/unittests/chain_tests.cpp @@ -27,11 +27,14 @@ BOOST_AUTO_TEST_CASE( replace_producer_keys ) try { } } - const auto old_pending_version = tester.control->pending_producers().version; + // TODO: Add test with instant-finality enabled + BOOST_REQUIRE(tester.control->pending_producers_legacy()); + const auto old_pending_version = tester.control->pending_producers_legacy()->version; const auto old_version = tester.control->active_producers().version; BOOST_REQUIRE_NO_THROW(tester.control->replace_producer_keys(new_key)); const auto new_version = tester.control->active_producers().version; - const auto pending_version = tester.control->pending_producers().version; + BOOST_REQUIRE(tester.control->pending_producers_legacy()); + const auto pending_version = tester.control->pending_producers_legacy()->version; // make sure version not been changed BOOST_REQUIRE(old_version == new_version); BOOST_REQUIRE(old_version == pending_version); @@ -44,7 +47,8 @@ BOOST_AUTO_TEST_CASE( replace_producer_keys ) try { const uint32_t expected_threshold = 1; const weight_type expected_key_weight = 1; - for(const auto& prod : tester.control->pending_producers().producers) { + BOOST_REQUIRE(tester.control->pending_producers_legacy()); + for(const auto& prod : tester.control->pending_producers_legacy()->producers) { BOOST_REQUIRE_EQUAL(std::get(prod.authority).threshold, expected_threshold); for(const auto& key : std::get(prod.authority).keys){ BOOST_REQUIRE_EQUAL(key.key, new_key); diff --git a/unittests/producer_schedule_tests.cpp b/unittests/producer_schedule_tests.cpp index dbfd9531ce..ad894303b1 100644 --- a/unittests/producer_schedule_tests.cpp +++ b/unittests/producer_schedule_tests.cpp @@ -137,12 +137,13 @@ BOOST_FIXTURE_TEST_CASE( producer_schedule_promotion_test, validating_tester ) t }; //wdump((fc::json::to_pretty_string(res))); wlog("set producer schedule to [alice,bob]"); - BOOST_REQUIRE_EQUAL( true, control->proposed_producers().has_value() ); - BOOST_CHECK_EQUAL( true, compare_schedules( sch1, *control->proposed_producers() ) ); - BOOST_CHECK_EQUAL( control->pending_producers().version, 0u ); + BOOST_REQUIRE_EQUAL( true, control->proposed_producers_legacy().has_value() ); + BOOST_CHECK_EQUAL( true, compare_schedules( sch1, *control->proposed_producers_legacy() ) ); + BOOST_REQUIRE(control->pending_producers_legacy()); + BOOST_CHECK_EQUAL( control->pending_producers_legacy()->version, 0u ); produce_block(); // Starts new block which promotes the proposed schedule to pending - BOOST_CHECK_EQUAL( control->pending_producers().version, 1u ); - BOOST_CHECK_EQUAL( true, compare_schedules( sch1, control->pending_producers() ) ); + BOOST_CHECK_EQUAL( control->pending_producers_legacy()->version, 1u ); + BOOST_CHECK_EQUAL( true, compare_schedules( sch1, *control->pending_producers_legacy() ) ); BOOST_CHECK_EQUAL( control->active_producers().version, 0u ); produce_block(); produce_block(); // Starts new block which promotes the pending schedule to active @@ -157,15 +158,16 @@ BOOST_FIXTURE_TEST_CASE( producer_schedule_promotion_test, validating_tester ) t producer_authority{"carol"_n, block_signing_authority_v0{1, {{get_public_key("carol"_n, "active"),1}}}} }; wlog("set producer schedule to [alice,bob,carol]"); - BOOST_REQUIRE_EQUAL( true, control->proposed_producers().has_value() ); - BOOST_CHECK_EQUAL( true, compare_schedules( sch2, *control->proposed_producers() ) ); + BOOST_REQUIRE_EQUAL( true, control->proposed_producers_legacy().has_value() ); + BOOST_CHECK_EQUAL( true, compare_schedules( sch2, *control->proposed_producers_legacy() ) ); produce_block(); produce_blocks(23); // Alice produces the last block of her first round. // Bob's first block (which advances LIB to Alice's last block) is started but not finalized. BOOST_REQUIRE_EQUAL( control->head_block_producer(), "alice"_n ); BOOST_REQUIRE_EQUAL( control->pending_block_producer(), "bob"_n ); - BOOST_CHECK_EQUAL( control->pending_producers().version, 2u ); + BOOST_REQUIRE(control->pending_producers_legacy()); + BOOST_CHECK_EQUAL( control->pending_producers_legacy()->version, 2u ); produce_blocks(12); // Bob produces his first 11 blocks BOOST_CHECK_EQUAL( control->active_producers().version, 1u ); @@ -202,12 +204,13 @@ BOOST_FIXTURE_TEST_CASE( producer_schedule_reduction, tester ) try { producer_authority{"carol"_n, block_signing_authority_v0{ 1, {{get_public_key("carol"_n, "active"),1}}}} }; wlog("set producer schedule to [alice,bob,carol]"); - BOOST_REQUIRE_EQUAL( true, control->proposed_producers().has_value() ); - BOOST_CHECK_EQUAL( true, compare_schedules( sch1, *control->proposed_producers() ) ); - BOOST_CHECK_EQUAL( control->pending_producers().version, 0u ); + BOOST_REQUIRE_EQUAL( true, control->proposed_producers_legacy().has_value() ); + BOOST_CHECK_EQUAL( true, compare_schedules( sch1, *control->proposed_producers_legacy() ) ); + BOOST_REQUIRE(control->pending_producers_legacy()); + BOOST_CHECK_EQUAL( control->pending_producers_legacy()->version, 0u ); produce_block(); // Starts new block which promotes the proposed schedule to pending - BOOST_CHECK_EQUAL( control->pending_producers().version, 1u ); - BOOST_CHECK_EQUAL( true, compare_schedules( sch1, control->pending_producers() ) ); + BOOST_CHECK_EQUAL( control->pending_producers_legacy()->version, 1u ); + BOOST_CHECK_EQUAL( true, compare_schedules( sch1, *control->pending_producers_legacy() ) ); BOOST_CHECK_EQUAL( control->active_producers().version, 0u ); produce_block(); produce_block(); // Starts new block which promotes the pending schedule to active @@ -221,13 +224,14 @@ BOOST_FIXTURE_TEST_CASE( producer_schedule_reduction, tester ) try { producer_authority{"bob"_n, block_signing_authority_v0{ 1, {{ get_public_key("bob"_n, "active"),1}}}} }; wlog("set producer schedule to [alice,bob]"); - BOOST_REQUIRE_EQUAL( true, control->proposed_producers().has_value() ); - BOOST_CHECK_EQUAL( true, compare_schedules( sch2, *control->proposed_producers() ) ); + BOOST_REQUIRE_EQUAL( true, control->proposed_producers_legacy().has_value() ); + BOOST_CHECK_EQUAL( true, compare_schedules( sch2, *control->proposed_producers_legacy() ) ); produce_blocks(48); BOOST_REQUIRE_EQUAL( control->head_block_producer(), "bob"_n ); BOOST_REQUIRE_EQUAL( control->pending_block_producer(), "carol"_n ); - BOOST_CHECK_EQUAL( control->pending_producers().version, 2u ); + BOOST_REQUIRE(control->pending_producers_legacy()); + BOOST_CHECK_EQUAL( control->pending_producers_legacy()->version, 2u ); produce_blocks(47); BOOST_CHECK_EQUAL( control->active_producers().version, 1u ); @@ -264,14 +268,16 @@ BOOST_AUTO_TEST_CASE( empty_producer_schedule_has_no_effect ) try { producer_authority{"bob"_n, block_signing_authority_v0{ 1, {{ get_public_key("bob"_n, "active"),1}}}} }; wlog("set producer schedule to [alice,bob]"); - BOOST_REQUIRE_EQUAL( true, c.control->proposed_producers().has_value() ); - BOOST_CHECK_EQUAL( true, compare_schedules( sch1, *c.control->proposed_producers() ) ); - BOOST_CHECK_EQUAL( c.control->pending_producers().producers.size(), 0u ); + BOOST_REQUIRE_EQUAL( true, c.control->proposed_producers_legacy().has_value() ); + BOOST_CHECK_EQUAL( true, compare_schedules( sch1, *c.control->proposed_producers_legacy() ) ); + BOOST_REQUIRE(c.control->pending_producers_legacy()); + BOOST_CHECK_EQUAL( c.control->pending_producers_legacy()->producers.size(), 0u ); // Start a new block which promotes the proposed schedule to pending c.produce_block(); - BOOST_CHECK_EQUAL( c.control->pending_producers().version, 1u ); - BOOST_CHECK_EQUAL( true, compare_schedules( sch1, c.control->pending_producers() ) ); + BOOST_REQUIRE(c.control->pending_producers_legacy()); + BOOST_CHECK_EQUAL( c.control->pending_producers_legacy()->version, 1u ); + BOOST_CHECK_EQUAL( true, compare_schedules( sch1, *c.control->pending_producers_legacy() ) ); BOOST_CHECK_EQUAL( c.control->active_producers().version, 0u ); // Start a new block which promotes the pending schedule to active @@ -282,22 +288,25 @@ BOOST_AUTO_TEST_CASE( empty_producer_schedule_has_no_effect ) try { res = c.set_producers_legacy( {} ); wlog("set producer schedule to []"); - BOOST_REQUIRE_EQUAL( true, c.control->proposed_producers().has_value() ); - BOOST_CHECK_EQUAL( c.control->proposed_producers()->producers.size(), 0u ); - BOOST_CHECK_EQUAL( c.control->proposed_producers()->version, 2u ); + BOOST_REQUIRE_EQUAL( true, c.control->proposed_producers_legacy().has_value() ); + BOOST_CHECK_EQUAL( c.control->proposed_producers_legacy()->producers.size(), 0u ); + BOOST_CHECK_EQUAL( c.control->proposed_producers_legacy()->version, 2u ); c.produce_blocks(12); - BOOST_CHECK_EQUAL( c.control->pending_producers().version, 1u ); + BOOST_REQUIRE(c.control->pending_producers_legacy()); + BOOST_CHECK_EQUAL( c.control->pending_producers_legacy()->version, 1u ); // Empty producer schedule does get promoted from proposed to pending c.produce_block(); - BOOST_CHECK_EQUAL( c.control->pending_producers().version, 2u ); - BOOST_CHECK_EQUAL( false, c.control->proposed_producers().has_value() ); + BOOST_REQUIRE(c.control->pending_producers_legacy()); + BOOST_CHECK_EQUAL( c.control->pending_producers_legacy()->version, 2u ); + BOOST_CHECK_EQUAL( false, c.control->proposed_producers_legacy().has_value() ); // However it should not get promoted from pending to active c.produce_blocks(24); BOOST_CHECK_EQUAL( c.control->active_producers().version, 1u ); - BOOST_CHECK_EQUAL( c.control->pending_producers().version, 2u ); + BOOST_REQUIRE(c.control->pending_producers_legacy()); + BOOST_CHECK_EQUAL( c.control->pending_producers_legacy()->version, 2u ); // Setting a new producer schedule should still use version 2 res = c.set_producers_legacy( {"alice"_n,"bob"_n,"carol"_n} ); @@ -307,15 +316,16 @@ BOOST_AUTO_TEST_CASE( empty_producer_schedule_has_no_effect ) try { producer_authority{"carol"_n, block_signing_authority_v0{ 1, {{get_public_key("carol"_n, "active"),1}}}} }; wlog("set producer schedule to [alice,bob,carol]"); - BOOST_REQUIRE_EQUAL( true, c.control->proposed_producers().has_value() ); - BOOST_CHECK_EQUAL( true, compare_schedules( sch2, *c.control->proposed_producers() ) ); - BOOST_CHECK_EQUAL( c.control->proposed_producers()->version, 2u ); + BOOST_REQUIRE_EQUAL( true, c.control->proposed_producers_legacy().has_value() ); + BOOST_CHECK_EQUAL( true, compare_schedules( sch2, *c.control->proposed_producers_legacy() ) ); + BOOST_CHECK_EQUAL( c.control->proposed_producers_legacy()->version, 2u ); // Produce enough blocks to promote the proposed schedule to pending, which it can do because the existing pending has zero producers c.produce_blocks(24); BOOST_CHECK_EQUAL( c.control->active_producers().version, 1u ); - BOOST_CHECK_EQUAL( c.control->pending_producers().version, 2u ); - BOOST_CHECK_EQUAL( true, compare_schedules( sch2, c.control->pending_producers() ) ); + BOOST_REQUIRE(c.control->pending_producers_legacy()); + BOOST_CHECK_EQUAL( c.control->pending_producers_legacy()->version, 2u ); + BOOST_CHECK_EQUAL( true, compare_schedules( sch2, *c.control->pending_producers_legacy() ) ); // Produce enough blocks to promote the pending schedule to active c.produce_blocks(24); @@ -342,12 +352,14 @@ BOOST_AUTO_TEST_CASE( producer_watermark_test ) try { producer_authority{"carol"_n, block_signing_authority_v0{ 1, {{c.get_public_key("carol"_n, "active"),1}}}} }; wlog("set producer schedule to [alice,bob,carol]"); - BOOST_REQUIRE_EQUAL( true, c.control->proposed_producers().has_value() ); - BOOST_CHECK_EQUAL( true, compare_schedules( sch1, *c.control->proposed_producers() ) ); - BOOST_CHECK_EQUAL( c.control->pending_producers().version, 0u ); + BOOST_REQUIRE_EQUAL( true, c.control->proposed_producers_legacy().has_value() ); + BOOST_CHECK_EQUAL( true, compare_schedules( sch1, *c.control->proposed_producers_legacy() ) ); + BOOST_REQUIRE(c.control->pending_producers_legacy()); + BOOST_CHECK_EQUAL( c.control->pending_producers_legacy()->version, 0u ); c.produce_block(); // Starts new block which promotes the proposed schedule to pending - BOOST_CHECK_EQUAL( c.control->pending_producers().version, 1u ); - BOOST_CHECK_EQUAL( true, compare_schedules( sch1, c.control->pending_producers() ) ); + BOOST_REQUIRE(c.control->pending_producers_legacy()); + BOOST_CHECK_EQUAL( c.control->pending_producers_legacy()->version, 1u ); + BOOST_CHECK_EQUAL( true, compare_schedules( sch1, *c.control->pending_producers_legacy() ) ); BOOST_CHECK_EQUAL( c.control->active_producers().version, 0u ); c.produce_block(); c.produce_block(); // Starts new block which promotes the pending schedule to active @@ -364,16 +376,18 @@ BOOST_AUTO_TEST_CASE( producer_watermark_test ) try { producer_authority{"bob"_n, block_signing_authority_v0{ 1, {{c.get_public_key("bob"_n, "active"),1}}}} }; wlog("set producer schedule to [alice,bob]"); - BOOST_REQUIRE_EQUAL( true, c.control->proposed_producers().has_value() ); - BOOST_CHECK_EQUAL( true, compare_schedules( sch2, *c.control->proposed_producers() ) ); + BOOST_REQUIRE_EQUAL( true, c.control->proposed_producers_legacy().has_value() ); + BOOST_CHECK_EQUAL( true, compare_schedules( sch2, *c.control->proposed_producers_legacy() ) ); produce_until_transition( c, "bob"_n, "carol"_n ); produce_until_transition( c, "alice"_n, "bob"_n ); - BOOST_CHECK_EQUAL( c.control->pending_producers().version, 2u ); + BOOST_REQUIRE(c.control->pending_producers_legacy()); + BOOST_CHECK_EQUAL( c.control->pending_producers_legacy()->version, 2u ); BOOST_CHECK_EQUAL( c.control->active_producers().version, 1u ); produce_until_transition( c, "carol"_n, "alice"_n ); - BOOST_CHECK_EQUAL( c.control->pending_producers().version, 2u ); + BOOST_REQUIRE(c.control->pending_producers_legacy()); + BOOST_CHECK_EQUAL( c.control->pending_producers_legacy()->version, 2u ); BOOST_CHECK_EQUAL( c.control->active_producers().version, 1u ); produce_until_transition( c, "bob"_n, "carol"_n ); @@ -388,8 +402,8 @@ BOOST_AUTO_TEST_CASE( producer_watermark_test ) try { res = c.set_producers( {"alice"_n,"bob"_n,"carol"_n} ); wlog("set producer schedule to [alice,bob,carol]"); - BOOST_REQUIRE_EQUAL( true, c.control->proposed_producers().has_value() ); - BOOST_CHECK_EQUAL( true, compare_schedules( sch1, *c.control->proposed_producers() ) ); + BOOST_REQUIRE_EQUAL( true, c.control->proposed_producers_legacy().has_value() ); + BOOST_CHECK_EQUAL( true, compare_schedules( sch1, *c.control->proposed_producers_legacy() ) ); produce_until_transition( c, "bob"_n, "alice"_n ); @@ -415,7 +429,8 @@ BOOST_AUTO_TEST_CASE( producer_watermark_test ) try { BOOST_CHECK_EQUAL( carol_itr->second, carol_last_produced_block_num ); } - BOOST_CHECK_EQUAL( c.control->pending_producers().version, 3u ); + BOOST_REQUIRE(c.control->pending_producers_legacy()); + BOOST_CHECK_EQUAL( c.control->pending_producers_legacy()->version, 3u ); BOOST_REQUIRE_EQUAL( c.control->active_producers().version, 2u ); produce_until_transition( c, "bob"_n, "alice"_n ); @@ -495,7 +510,7 @@ BOOST_FIXTURE_TEST_CASE( satisfiable_msig_test, validating_tester ) try { fc_exception_message_is( "producer schedule includes an unsatisfiable authority for alice" ) ); - BOOST_REQUIRE_EQUAL( false, control->proposed_producers().has_value() ); + BOOST_REQUIRE_EQUAL( false, control->proposed_producers_legacy().has_value() ); } FC_LOG_AND_RETHROW() @@ -514,7 +529,7 @@ BOOST_FIXTURE_TEST_CASE( duplicate_producers_test, validating_tester ) try { fc_exception_message_is( "duplicate producer name in producer schedule" ) ); - BOOST_REQUIRE_EQUAL( false, control->proposed_producers().has_value() ); + BOOST_REQUIRE_EQUAL( false, control->proposed_producers_legacy().has_value() ); } FC_LOG_AND_RETHROW() @@ -532,7 +547,7 @@ BOOST_FIXTURE_TEST_CASE( duplicate_keys_test, validating_tester ) try { fc_exception_message_is( "producer schedule includes a duplicated key for alice" ) ); - BOOST_REQUIRE_EQUAL( false, control->proposed_producers().has_value() ); + BOOST_REQUIRE_EQUAL( false, control->proposed_producers_legacy().has_value() ); // ensure that multiple producers are allowed to share keys vector sch2 = { @@ -541,7 +556,7 @@ BOOST_FIXTURE_TEST_CASE( duplicate_keys_test, validating_tester ) try { }; set_producer_schedule( sch2 ); - BOOST_REQUIRE_EQUAL( true, control->proposed_producers().has_value() ); + BOOST_REQUIRE_EQUAL( true, control->proposed_producers_legacy().has_value() ); } FC_LOG_AND_RETHROW() BOOST_AUTO_TEST_CASE( large_authority_overflow_test ) try { @@ -603,7 +618,7 @@ BOOST_AUTO_TEST_CASE( extra_signatures_test ) try { }; main.set_producer_schedule( sch1 ); - BOOST_REQUIRE_EQUAL( true, main.control->proposed_producers().has_value() ); + BOOST_REQUIRE_EQUAL( true, main.control->proposed_producers_legacy().has_value() ); main.block_signing_private_keys.emplace(get_public_key("alice"_n, "bs1"), get_private_key("alice"_n, "bs1")); main.block_signing_private_keys.emplace(get_public_key("alice"_n, "bs2"), get_private_key("alice"_n, "bs2")); From 57ccf142be52ed5ff7997adaca41d04e39ee4bd6 Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Thu, 11 Jan 2024 16:56:59 -0500 Subject: [PATCH 06/36] Update some todo comments. --- libraries/chain/controller.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 8f5020fe3a..720e1a919d 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -939,7 +939,7 @@ struct controller_impl { cfg.state_size, false, cfg.db_map_mode ), blog( cfg.blocks_dir, cfg.blog ), block_data(block_data_t::block_data_variant{ - std::in_place_type, // [greg todo] create correct type depending on whether IF activated + std::in_place_type, // initial state is always dpos std::filesystem::path{cfg.blocks_dir / config::reversible_blocks_dir_name}}), resource_limits( db, [&s](bool is_trx_transient) { return s.get_deep_mind_logger(is_trx_transient); }), authorization( s, db ), @@ -1508,7 +1508,7 @@ struct controller_impl { section.add_row(chain_snapshot_header(), db); }); - // [greg todo] add snapshot support for new (IF) block_state section +#warning todo: add snapshot support for new (IF) block_state section auto write_block_state_section = [&](auto& fork_db, auto& head) { snapshot->write_section("eosio::chain::block_state", [&]( auto §ion ) { section.template add_row(*head, db); @@ -1562,7 +1562,7 @@ struct controller_impl { header.validate(); }); - // [greg todo] add snapshot support for new (IF) block_state section +#warning todo: add snapshot support for new (IF) block_state section auto read_block_state_section = [&](auto& fork_db, auto& head) { /// load and upgrade the block header state block_header_state_legacy head_header_state; using v2 = legacy::snapshot_block_header_state_v2; @@ -2690,9 +2690,9 @@ struct controller_impl { void report_block_header_diff( const block_header& b, const block_header& ab ) { #define EOS_REPORT(DESC,A,B) \ - if( A != B ) { \ - elog("${desc}: ${bv} != ${abv}", ("desc", DESC)("bv", A)("abv", B)); \ - } + if( A != B ) { \ + elog("${desc}: ${bv} != ${abv}", ("desc", DESC)("bv", A)("abv", B)); \ + } EOS_REPORT( "timestamp", b.timestamp, ab.timestamp ) EOS_REPORT( "producer", b.producer, ab.producer ) From e89610589fc59ff59d8cee4d0e1a83790a5c142c Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 12 Jan 2024 08:37:45 -0600 Subject: [PATCH 07/36] GH-1980 Fix merge issue --- libraries/chain/block_header_state.cpp | 2 +- libraries/chain/controller.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/chain/block_header_state.cpp b/libraries/chain/block_header_state.cpp index 3199350383..cf612c42f6 100644 --- a/libraries/chain/block_header_state.cpp +++ b/libraries/chain/block_header_state.cpp @@ -127,7 +127,7 @@ block_header_state block_header_state::next(block_header_state_input& input) con */ block_header_state block_header_state::next(const signed_block_header& h, const protocol_feature_set& pfs, validator_t& validator) const { - auto producer = detail::get_scheduled_producer(proposer_policy->proposer_schedule.producers, h.timestamp).producer_name; + auto producer = detail::get_scheduled_producer(active_proposer_policy->proposer_schedule.producers, h.timestamp).producer_name; EOS_ASSERT( h.previous == id, unlinkable_block_exception, "previous mismatch" ); EOS_ASSERT( h.producer == producer, wrong_producer, "wrong producer specified" ); diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index b48f1c1040..50253241ab 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -915,7 +915,7 @@ struct pending_state { _db_session.push(); } - //bool is_dpos() const { return std::visit([](const auto& stage) { return stage.is_dpos(); }, _block_stage); } + bool is_dpos() const { return std::visit([](const auto& stage) { return stage.is_dpos(); }, _block_stage); } const block_signing_authority& pending_block_signing_authority() const { return std::visit( From 0e72f5acac6e675af29e1fb8a36e889b84196360 Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Fri, 12 Jan 2024 11:05:40 -0500 Subject: [PATCH 08/36] Move `header_exts` to block_header_state --- libraries/chain/include/eosio/chain/block_header_state.hpp | 5 +++++ libraries/chain/include/eosio/chain/block_state.hpp | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/libraries/chain/include/eosio/chain/block_header_state.hpp b/libraries/chain/include/eosio/chain/block_header_state.hpp index 9a1b7c5f7b..5f96139544 100644 --- a/libraries/chain/include/eosio/chain/block_header_state.hpp +++ b/libraries/chain/include/eosio/chain/block_header_state.hpp @@ -60,6 +60,11 @@ struct block_header_state { flat_map proposer_policies; flat_map finalizer_policies; + + // ------ data members caching information available elsewhere ---------------------- + header_extension_multimap header_exts; // redundant with the data stored in header + + // ------ functions ----------------------------------------------------------------- digest_type compute_finalizer_digest() const; block_timestamp_type timestamp() const { return header.timestamp; } diff --git a/libraries/chain/include/eosio/chain/block_state.hpp b/libraries/chain/include/eosio/chain/block_state.hpp index a224095d0d..ee1c1e8e3d 100644 --- a/libraries/chain/include/eosio/chain/block_state.hpp +++ b/libraries/chain/include/eosio/chain/block_state.hpp @@ -19,7 +19,6 @@ struct block_state : public block_header_state { // block_header_state provi // ------ data members caching information available elsewhere ---------------------- block_id_type cached_id; // cache of block_header_state::header.calculate_id() (indexed on this field) - header_extension_multimap header_exts; // redundant with the data stored in header // ------ functions ----------------------------------------------------------------- const block_id_type& id() const { return cached_id; } From 8e3dca22e689c70f658f6ca629a2b0d5cbd0e031 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 12 Jan 2024 10:26:37 -0600 Subject: [PATCH 09/36] GH-1980 Copy over active policies --- libraries/chain/block_header_state.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libraries/chain/block_header_state.cpp b/libraries/chain/block_header_state.cpp index cf612c42f6..209a2bca5d 100644 --- a/libraries/chain/block_header_state.cpp +++ b/libraries/chain/block_header_state.cpp @@ -69,6 +69,9 @@ block_header_state block_header_state::next(block_header_state_input& input) con .schedule_version = header.schedule_version }; + result.active_finalizer_policy = active_finalizer_policy; + result.active_proposer_policy = active_proposer_policy; + if(!proposer_policies.empty()) { auto it = proposer_policies.begin(); if (it->first <= input.timestamp) { From 3b15160bbb4d6b5117e09725f05a4d3099cd4502 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 12 Jan 2024 13:32:41 -0600 Subject: [PATCH 10/36] GH-2045 Rename finalize_block to finish_block to avoid confusion with finalizing a block in instant finality --- docs/block_production/lifecycle.md | 2 +- libraries/chain/controller.cpp | 8 ++++---- libraries/chain/hotstuff/chain_pacemaker.cpp | 2 +- .../include/eosio/chain/block_header_state_legacy.hpp | 2 +- libraries/chain/include/eosio/chain/controller.hpp | 2 +- libraries/chain/resource_limits.cpp | 6 +++--- libraries/testing/tester.cpp | 2 +- plugins/producer_plugin/producer_plugin.cpp | 2 +- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/block_production/lifecycle.md b/docs/block_production/lifecycle.md index cba10965f0..19478ffb2a 100644 --- a/docs/block_production/lifecycle.md +++ b/docs/block_production/lifecycle.md @@ -12,7 +12,7 @@ flowchart TD direction TB start -- "stage = Ø" --> sb sb("start_block()"):::fun -- "stage = building_block" --> et - et["execute transactions" ] -- "stage = building_block" --> fb("finalize_block()"):::fun + et["execute transactions" ] -- "stage = building_block" --> fb("finish_block()"):::fun fb -- "stage = assembled block" --> cb["add transaction metadata and create completed block"] cb -- "stage = completed block" --> commit("commit_block() (where we [maybe] add to fork_db and mark valid)"):::fun diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 50253241ab..004e6a0122 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -2608,7 +2608,7 @@ struct controller_impl { guard_pending.cancel(); } /// start_block - void finalize_block() + void finish_block() { EOS_ASSERT( pending, block_validate_exception, "it is not valid to finalize when there is no pending block"); EOS_ASSERT( std::holds_alternative(pending->_block_stage), block_validate_exception, "already called finalize_block"); @@ -2888,7 +2888,7 @@ struct controller_impl { ("lhs", r)("rhs", static_cast(receipt)) ); } - finalize_block(); + finish_block(); auto& ab = std::get(pending->_block_stage); @@ -3749,10 +3749,10 @@ void controller::start_block( block_timestamp_type when, bs, std::optional(), deadline ); } -void controller::finalize_block( block_report& br, const signer_callback_type& signer_callback ) { +void controller::finish_block( block_report& br, const signer_callback_type& signer_callback ) { validate_db_available_size(); - my->finalize_block(); + my->finish_block(); auto& ab = std::get(my->pending->_block_stage); my->pending->_block_stage = ab.make_completed_block( diff --git a/libraries/chain/hotstuff/chain_pacemaker.cpp b/libraries/chain/hotstuff/chain_pacemaker.cpp index 2d64185a53..2f3a84c41c 100644 --- a/libraries/chain/hotstuff/chain_pacemaker.cpp +++ b/libraries/chain/hotstuff/chain_pacemaker.cpp @@ -175,7 +175,7 @@ namespace eosio::chain { if (_active_finalizer_policy.generation == 0) { ilog("Switching to instant finality at block ${b}", ("b", block->block_num())); // switching from dpos to hotstuff, all nodes will switch at same block height - // block header extension is set in finalize_block to value set by host function set_finalizers + // block header extension is set in finish_block to value set by host function set_finalizers _chain->set_hs_irreversible_block_num(block->block_num()); // can be any value <= dpos lib } auto if_extension = std::get(*ext); diff --git a/libraries/chain/include/eosio/chain/block_header_state_legacy.hpp b/libraries/chain/include/eosio/chain/block_header_state_legacy.hpp index 6f7a3153d9..95a3e71f69 100644 --- a/libraries/chain/include/eosio/chain/block_header_state_legacy.hpp +++ b/libraries/chain/include/eosio/chain/block_header_state_legacy.hpp @@ -120,7 +120,7 @@ struct pending_block_header_state_legacy : public detail::block_header_state_leg * start_block -> (global_property_object.proposed_schedule_block_num == dpos_lib) * building_block._new_pending_producer_schedule = producers * - * finalize_block -> + * finish_block -> * block_header.extensions.wtmsig_block_signatures = producers * block_header.new_producers = producers * diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index 2941b55280..c7ccd89a9c 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -183,7 +183,7 @@ namespace eosio::chain { fc::microseconds total_time{}; }; - void finalize_block( block_report& br, const signer_callback_type& signer_callback ); + void finish_block( block_report& br, const signer_callback_type& signer_callback ); void sign_block( const signer_callback_type& signer_callback ); void commit_block(); diff --git a/libraries/chain/resource_limits.cpp b/libraries/chain/resource_limits.cpp index b11eea1dba..4c5e4b6cb3 100644 --- a/libraries/chain/resource_limits.cpp +++ b/libraries/chain/resource_limits.cpp @@ -117,7 +117,7 @@ void resource_limits_manager::set_block_parameters(const elastic_limit_parameter c.cpu_limit_parameters = cpu_limit_parameters; c.net_limit_parameters = net_limit_parameters; - // set_block_parameters is called by controller::finalize_block, + // set_block_parameters is called by controller::finish_block, // where transaction specific logging is not possible if (auto dm_logger = _get_deep_mind_logger(false)) { dm_logger->on_update_resource_limits_config(c); @@ -359,7 +359,7 @@ void resource_limits_manager::process_account_limit_updates() { multi_index.remove(*itr); } - // process_account_limit_updates is called by controller::finalize_block, + // process_account_limit_updates is called by controller::finish_block, // where transaction specific logging is not possible if (auto dm_logger = _get_deep_mind_logger(false)) { dm_logger->on_update_resource_limits_state(state); @@ -381,7 +381,7 @@ void resource_limits_manager::process_block_usage(uint32_t block_num) { state.update_virtual_net_limit(config); state.pending_net_usage = 0; - // process_block_usage is called by controller::finalize_block, + // process_block_usage is called by controller::finish, // where transaction specific logging is not possible if (auto dm_logger = _get_deep_mind_logger(false)) { dm_logger->on_update_resource_limits_state(state); diff --git a/libraries/testing/tester.cpp b/libraries/testing/tester.cpp index b5e35533cb..80e098e689 100644 --- a/libraries/testing/tester.cpp +++ b/libraries/testing/tester.cpp @@ -487,7 +487,7 @@ namespace eosio { namespace testing { }); controller::block_report br; - control->finalize_block( br, [&]( digest_type d ) { + control->finish_block( br, [&]( digest_type d ) { std::vector result; result.reserve(signing_keys.size()); for (const auto& k: signing_keys) diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index 182534b0f1..b8db00f4c0 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -2646,7 +2646,7 @@ void producer_plugin_impl::produce_block() { // idump( (fc::time_point::now() - chain.pending_block_time()) ); controller::block_report br; - chain.finalize_block(br, [&](const digest_type& d) { + chain.finish_block(br, [&](const digest_type& d) { auto debug_logger = maybe_make_debug_time_logger(); vector sigs; sigs.reserve(relevant_providers.size()); From cd3190d6d567e124295dca57b436710bb3166488 Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Fri, 12 Jan 2024 15:17:08 -0500 Subject: [PATCH 11/36] cleanup `next(block_header_state_input& data)` function and update `header_exts` --- libraries/chain/block_header_state.cpp | 77 ++++++++++++++++++-------- 1 file changed, 54 insertions(+), 23 deletions(-) diff --git a/libraries/chain/block_header_state.cpp b/libraries/chain/block_header_state.cpp index cf612c42f6..9c63f1191c 100644 --- a/libraries/chain/block_header_state.cpp +++ b/libraries/chain/block_header_state.cpp @@ -69,6 +69,30 @@ block_header_state block_header_state::next(block_header_state_input& input) con .schedule_version = header.schedule_version }; + // activated protocol features + // --------------------------- + if (!input.new_protocol_feature_activations.empty()) { + result.activated_protocol_features = std::make_shared( + *activated_protocol_features, input.new_protocol_feature_activations); + } else { + result.activated_protocol_features = activated_protocol_features; + } + + // core + // ---- + if (input.qc_info) + result.core = core.next(input.qc_info->last_qc_block_num, input.qc_info->is_last_qc_strong); + else + result.core = core; + + + // proposal_mtree and finality_mtree + // [greg todo] + + // proposer policy + // --------------- + result.active_proposer_policy = active_proposer_policy; + if(!proposer_policies.empty()) { auto it = proposer_policies.begin(); if (it->first <= input.timestamp) { @@ -80,40 +104,47 @@ block_header_state block_header_state::next(block_header_state_input& input) con result.proposer_policies = proposer_policies; } } + if (input.new_proposer_policy) { // called when assembling the block result.proposer_policies[result.header.timestamp] = input.new_proposer_policy; } - // core - // ---- - if (input.qc_info) - result.core = core.next(input.qc_info->last_qc_block_num, input.qc_info->is_last_qc_strong); - else - result.core = core; + // finalizer policy + // ---------------- + result.active_finalizer_policy = active_finalizer_policy; - if (!input.new_protocol_feature_activations.empty()) { - result.activated_protocol_features = std::make_shared( - *activated_protocol_features, input.new_protocol_feature_activations); - } else { - result.activated_protocol_features = activated_protocol_features; - } + // [greg todo] correct support for new finalizer_policy activation using finalizer_policies map - // add block header extensions - // --------------------------- if (input.new_finalizer_policy) ++input.new_finalizer_policy->generation; - std::optional qc_info = input.qc_info; - if (!qc_info) { - // [greg todo]: copy the one from the previous block (look in header.header_extensions) + + // add IF block header extension + // ----------------------------- + uint16_t if_ext_id = instant_finality_extension::extension_id(); + auto if_entry = header_exts.lower_bound(if_ext_id); + auto& if_ext = std::get(if_entry->second); + + instant_finality_extension new_if_ext {if_ext.qc_info, + std::move(input.new_finalizer_policy), + std::move(input.new_proposer_policy)}; + if (input.qc_info) + new_if_ext.qc_info = *input.qc_info; + + emplace_extension(result.header.header_extensions, if_ext_id, fc::raw::pack(new_if_ext)); + result.header_exts.emplace(if_ext_id, std::move(new_if_ext)); + + // add protocol_feature_activation extension + // ----------------------------------------- + if (!input.new_protocol_feature_activations.empty()) { + uint16_t ext_id = protocol_feature_activation::extension_id(); + protocol_feature_activation pfa_ext{.protocol_features = std::move(input.new_protocol_feature_activations)}; + + emplace_extension(result.header.header_extensions, ext_id, fc::raw::pack(pfa_ext)); + result.header_exts.emplace(ext_id, std::move(pfa_ext)); } - - emplace_extension(result.header.header_extensions, instant_finality_extension::extension_id(), - fc::raw::pack(instant_finality_extension{qc_info, - std::move(input.new_finalizer_policy), - std::move(input.new_proposer_policy)})); - + return result; } From d01debb4364ccf9226e428cc6aa30bfb048cbe03 Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Fri, 12 Jan 2024 15:26:44 -0500 Subject: [PATCH 12/36] Cleanup `next(const signed_block_header& h, ...)`. --- libraries/chain/block_header_state.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/libraries/chain/block_header_state.cpp b/libraries/chain/block_header_state.cpp index 9c63f1191c..7db909d1f4 100644 --- a/libraries/chain/block_header_state.cpp +++ b/libraries/chain/block_header_state.cpp @@ -169,16 +169,17 @@ block_header_state block_header_state::next(const signed_block_header& h, const // ------------------------------------------------------ vector new_protocol_feature_activations; if( exts.count(protocol_feature_activation::extension_id() > 0) ) { - const auto& entry = exts.lower_bound(protocol_feature_activation::extension_id()); - new_protocol_feature_activations = std::move(std::get(entry->second).protocol_features); + auto pfa_entry = exts.lower_bound(protocol_feature_activation::extension_id()); + auto& pfa_ext = std::get(pfa_entry->second); + new_protocol_feature_activations = std::move(pfa_ext.protocol_features); } // retrieve instant_finality_extension data from block extension // ------------------------------------------------------------- EOS_ASSERT(exts.count(instant_finality_extension::extension_id() > 0), misc_exception, "Instant Finality Extension is expected to be present in all block headers after switch to IF"); - const auto& if_entry = exts.lower_bound(instant_finality_extension::extension_id()); - const auto& if_ext = std::get(if_entry->second); + auto if_entry = exts.lower_bound(instant_finality_extension::extension_id()); + auto& if_ext = std::get(if_entry->second); building_block_input bb_input{ .parent_id = id, From 94996301971298fdd847934d93cec032e8ea6af6 Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Fri, 12 Jan 2024 15:32:39 -0500 Subject: [PATCH 13/36] Add missing block id computation to `next()` function. --- libraries/chain/block_header_state.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/libraries/chain/block_header_state.cpp b/libraries/chain/block_header_state.cpp index 7db909d1f4..4ae6e02fcf 100644 --- a/libraries/chain/block_header_state.cpp +++ b/libraries/chain/block_header_state.cpp @@ -145,6 +145,10 @@ block_header_state block_header_state::next(block_header_state_input& input) con result.header_exts.emplace(ext_id, std::move(pfa_ext)); } + // Finally update block id from header + // ----------------------------------- + result.id = result.header.calculate_id(); + return result; } @@ -165,8 +169,8 @@ block_header_state block_header_state::next(const signed_block_header& h, const auto exts = h.validate_and_extract_header_extensions(); - // handle protocol_feature_activation from incoming block - // ------------------------------------------------------ + // retrieve protocol_feature_activation from incoming block header extension + // ------------------------------------------------------------------------- vector new_protocol_feature_activations; if( exts.count(protocol_feature_activation::extension_id() > 0) ) { auto pfa_entry = exts.lower_bound(protocol_feature_activation::extension_id()); @@ -174,8 +178,8 @@ block_header_state block_header_state::next(const signed_block_header& h, const new_protocol_feature_activations = std::move(pfa_ext.protocol_features); } - // retrieve instant_finality_extension data from block extension - // ------------------------------------------------------------- + // retrieve instant_finality_extension data from block header extension + // -------------------------------------------------------------------- EOS_ASSERT(exts.count(instant_finality_extension::extension_id() > 0), misc_exception, "Instant Finality Extension is expected to be present in all block headers after switch to IF"); auto if_entry = exts.lower_bound(instant_finality_extension::extension_id()); From 7253499595afb74620c24798a6afde3b1e6de6f2 Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Fri, 12 Jan 2024 15:40:23 -0500 Subject: [PATCH 14/36] Simplify `block_header_state_core::next()` API. wip --- libraries/chain/block_header_state.cpp | 19 ++++++++----------- .../eosio/chain/block_header_state.hpp | 2 +- unittests/block_header_state_tests.cpp | 14 +++++++------- 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/libraries/chain/block_header_state.cpp b/libraries/chain/block_header_state.cpp index 4ae6e02fcf..8c43e3af27 100644 --- a/libraries/chain/block_header_state.cpp +++ b/libraries/chain/block_header_state.cpp @@ -13,9 +13,9 @@ producer_authority block_header_state::get_scheduled_producer(block_timestamp_ty #warning Add last_proposed_finalizer_policy_generation to snapshot_block_header_state_v3, see header file TODO -block_header_state_core block_header_state_core::next(const uint32_t last_qc_block_num, bool is_last_qc_strong) const { +block_header_state_core block_header_state_core::next(qc_info_t incoming) const { // no state change if last_qc_block_num is the same - if (last_qc_block_num == this->last_qc_block_num) { + if (incoming.last_qc_block_num == this->last_qc_block_num) { return {*this}; } @@ -27,7 +27,7 @@ block_header_state_core block_header_state_core::next(const uint32_t last_qc_blo block_header_state_core result{*this}; - if (is_last_qc_strong) { + if (incoming.is_last_qc_strong) { // last QC is strong. We can progress forward. // block with old final_on_strong_qc_block_num becomes irreversible @@ -78,16 +78,13 @@ block_header_state block_header_state::next(block_header_state_input& input) con result.activated_protocol_features = activated_protocol_features; } - // core - // ---- - if (input.qc_info) - result.core = core.next(input.qc_info->last_qc_block_num, input.qc_info->is_last_qc_strong); - else - result.core = core; - + // block_header_state_core + // ----------------------- + result.core = input.qc_info ? core.next(*input.qc_info) : core; // proposal_mtree and finality_mtree - // [greg todo] + // --------------------------------- + // [greg todo] ?? // proposer policy // --------------- diff --git a/libraries/chain/include/eosio/chain/block_header_state.hpp b/libraries/chain/include/eosio/chain/block_header_state.hpp index 6c3ac93dc9..036a25ee45 100644 --- a/libraries/chain/include/eosio/chain/block_header_state.hpp +++ b/libraries/chain/include/eosio/chain/block_header_state.hpp @@ -41,7 +41,7 @@ struct block_header_state_core { std::optional last_qc_block_num; // uint32_t finalizer_policy_generation; // - block_header_state_core next(uint32_t last_qc_block_num, bool is_last_qc_strong) const; + block_header_state_core next(qc_info_t incoming) const; }; struct block_header_state { diff --git a/unittests/block_header_state_tests.cpp b/unittests/block_header_state_tests.cpp index 9c78b133c2..eddddf8f62 100644 --- a/unittests/block_header_state_tests.cpp +++ b/unittests/block_header_state_tests.cpp @@ -34,7 +34,7 @@ BOOST_AUTO_TEST_CASE(block_header_state_core_state_transition_test) // verifies the state is kept the same when old last_final_block_num // and new last_final_block_num are the same for (bool is_last_qc_strong: { true, false }) { - auto new_bhs_core = old_bhs_core.next(old_last_qc_block_num, is_last_qc_strong); + auto new_bhs_core = old_bhs_core.next({old_last_qc_block_num, is_last_qc_strong}); BOOST_REQUIRE_EQUAL(new_bhs_core.last_final_block_num, old_bhs_core.last_final_block_num); BOOST_REQUIRE_EQUAL(*new_bhs_core.final_on_strong_qc_block_num, *old_bhs_core.final_on_strong_qc_block_num); BOOST_REQUIRE_EQUAL(*new_bhs_core.last_qc_block_num, *old_bhs_core.last_qc_block_num); @@ -42,12 +42,12 @@ BOOST_AUTO_TEST_CASE(block_header_state_core_state_transition_test) // verifies state cannot be transitioned to a smaller last_qc_block_num for (bool is_last_qc_strong: { true, false }) { - BOOST_REQUIRE_THROW(old_bhs_core.next(old_last_qc_block_num - 1, is_last_qc_strong), block_validate_exception); + BOOST_REQUIRE_THROW(old_bhs_core.next({old_last_qc_block_num - 1, is_last_qc_strong}), block_validate_exception); } // verifies state transition works when is_last_qc_strong is true constexpr auto input_last_qc_block_num = 4u; - auto new_bhs_core = old_bhs_core.next(input_last_qc_block_num, true); + auto new_bhs_core = old_bhs_core.next({input_last_qc_block_num, true}); // old final_on_strong_qc block became final BOOST_REQUIRE_EQUAL(new_bhs_core.last_final_block_num, old_final_on_strong_qc_block_num); // old last_qc block became final_on_strong_qc block @@ -56,7 +56,7 @@ BOOST_AUTO_TEST_CASE(block_header_state_core_state_transition_test) BOOST_REQUIRE_EQUAL(*new_bhs_core.last_qc_block_num, input_last_qc_block_num); // verifies state transition works when is_last_qc_strong is false - new_bhs_core = old_bhs_core.next(input_last_qc_block_num, false); + new_bhs_core = old_bhs_core.next({input_last_qc_block_num, false}); // last_final_block_num should not change BOOST_REQUIRE_EQUAL(new_bhs_core.last_final_block_num, old_last_final_block_num); // new final_on_strong_qc_block_num should not be present @@ -75,7 +75,7 @@ BOOST_AUTO_TEST_CASE(block_header_state_core_3_chain_transition_test) // block2 --> block3 constexpr auto block3_input_last_qc_block_num = 2u; - auto block3_bhs_core = block2_bhs_core.next(block3_input_last_qc_block_num, true); + auto block3_bhs_core = block2_bhs_core.next({block3_input_last_qc_block_num, true}); // last_final_block_num should be the same as old one BOOST_REQUIRE_EQUAL(block3_bhs_core.last_final_block_num, block2_last_final_block_num); // final_on_strong_qc_block_num should be same as old one @@ -86,7 +86,7 @@ BOOST_AUTO_TEST_CASE(block_header_state_core_3_chain_transition_test) // block3 --> block4 constexpr auto block4_input_last_qc_block_num = 3u; - auto block4_bhs_core = block3_bhs_core.next(block4_input_last_qc_block_num, true); + auto block4_bhs_core = block3_bhs_core.next({block4_input_last_qc_block_num, true}); // last_final_block_num should not change BOOST_REQUIRE_EQUAL(block4_bhs_core.last_final_block_num, block2_last_final_block_num); // final_on_strong_qc_block_num should be block3's last_qc_block_num @@ -98,7 +98,7 @@ BOOST_AUTO_TEST_CASE(block_header_state_core_3_chain_transition_test) // block4 --> block5 constexpr auto block5_input_last_qc_block_num = 4u; - auto block5_bhs_core = block4_bhs_core.next(block5_input_last_qc_block_num, true); + auto block5_bhs_core = block4_bhs_core.next({block5_input_last_qc_block_num, true}); // last_final_block_num should have a new value BOOST_REQUIRE_EQUAL(block5_bhs_core.last_final_block_num, block4_final_on_strong_qc_block_num); // final_on_strong_qc_block_num should be block4's last_qc_block_num From fa2a60a1858a9019c5bf0a03613045502d026c69 Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Fri, 12 Jan 2024 15:49:36 -0500 Subject: [PATCH 15/36] Remove unneeded template parameter for `block_data_gen_t` --- libraries/chain/controller.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 50253241ab..1f491b0ceb 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -103,7 +103,7 @@ class maybe_session { _session->push(); } - maybe_session& operator = ( maybe_session&& mv ) { + maybe_session& operator=( maybe_session&& mv ) noexcept { if (mv._session) { _session.emplace(std::move(*mv._session)); mv._session.reset(); @@ -118,11 +118,10 @@ class maybe_session { std::optional _session; }; -template +template struct block_data_gen_t { public: using bs = bsp::element_type; - using bhs = bhsp::element_type; using fork_db_t = fork_database; bsp head; @@ -167,8 +166,8 @@ struct block_data_gen_t { } }; -using block_data_legacy_t = block_data_gen_t; -using block_data_new_t = block_data_gen_t; +using block_data_legacy_t = block_data_gen_t; +using block_data_new_t = block_data_gen_t; struct block_data_t { using block_data_variant = std::variant; From 1cc9fe31dad01a3a213f5e590eee2d3ba69a59b1 Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Fri, 12 Jan 2024 15:54:46 -0500 Subject: [PATCH 16/36] Fix issue I introduced in commit 7253499 --- libraries/chain/block_header_state.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/chain/block_header_state.cpp b/libraries/chain/block_header_state.cpp index 8c43e3af27..c4f7075548 100644 --- a/libraries/chain/block_header_state.cpp +++ b/libraries/chain/block_header_state.cpp @@ -19,7 +19,7 @@ block_header_state_core block_header_state_core::next(qc_info_t incoming) const return {*this}; } - EOS_ASSERT(last_qc_block_num > this->last_qc_block_num, block_validate_exception, + EOS_ASSERT(incoming.last_qc_block_num > this->last_qc_block_num, block_validate_exception, "new last_qc_block_num must be greater than old last_qc_block_num"); auto old_last_qc_block_num = this->last_qc_block_num; @@ -48,7 +48,7 @@ block_header_state_core block_header_state_core::next(qc_info_t incoming) const } // new last_qc_block_num is always the input last_qc_block_num. - result.last_qc_block_num = last_qc_block_num; + result.last_qc_block_num = incoming.last_qc_block_num; return result; } From 6473f311dcbac7c3c9950e33f6f6a1c624dcdd1d Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Fri, 12 Jan 2024 18:15:28 -0500 Subject: [PATCH 17/36] Update `apply_block()` so that it works for the IF path. --- libraries/chain/block_header_state.cpp | 4 + libraries/chain/block_header_state_legacy.cpp | 13 +-- libraries/chain/block_state.cpp | 41 +++++---- libraries/chain/controller.cpp | 84 ++++++++++--------- .../eosio/chain/block_header_state.hpp | 1 + .../eosio/chain/block_header_state_utils.hpp | 12 +++ .../chain/include/eosio/chain/block_state.hpp | 7 +- 7 files changed, 97 insertions(+), 65 deletions(-) diff --git a/libraries/chain/block_header_state.cpp b/libraries/chain/block_header_state.cpp index c4f7075548..b50c95dfab 100644 --- a/libraries/chain/block_header_state.cpp +++ b/libraries/chain/block_header_state.cpp @@ -11,6 +11,10 @@ producer_authority block_header_state::get_scheduled_producer(block_timestamp_ty return detail::get_scheduled_producer(active_proposer_policy->proposer_schedule.producers, t); } +const vector& block_header_state::get_new_protocol_feature_activations()const { + return detail::get_new_protocol_feature_activations(header_exts); +} + #warning Add last_proposed_finalizer_policy_generation to snapshot_block_header_state_v3, see header file TODO block_header_state_core block_header_state_core::next(qc_info_t incoming) const { diff --git a/libraries/chain/block_header_state_legacy.cpp b/libraries/chain/block_header_state_legacy.cpp index 8d695b1425..7595d2d5f2 100644 --- a/libraries/chain/block_header_state_legacy.cpp +++ b/libraries/chain/block_header_state_legacy.cpp @@ -429,15 +429,10 @@ namespace eosio::chain { } /** - * Reference cannot outlive *this. Assumes header_exts is not mutated after instatiation. + * Reference cannot outlive *this. Assumes header_exts is not mutated after instantiation. */ const vector& block_header_state_legacy::get_new_protocol_feature_activations()const { - static const vector no_activations{}; - - if( header_exts.count(protocol_feature_activation::extension_id()) == 0 ) - return no_activations; - - return std::get(header_exts.lower_bound(protocol_feature_activation::extension_id())->second).protocol_features; + return detail::get_new_protocol_feature_activations(header_exts); } block_header_state_legacy::block_header_state_legacy( legacy::snapshot_block_header_state_v2&& snapshot ) @@ -451,10 +446,10 @@ namespace eosio::chain { producer_to_last_implied_irb = std::move(snapshot.producer_to_last_implied_irb); valid_block_signing_authority = block_signing_authority_v0{ 1, {{std::move(snapshot.block_signing_key), 1}} }; confirm_count = std::move(snapshot.confirm_count); - id = std::move(snapshot.id); + id = snapshot.id; header = std::move(snapshot.header); pending_schedule.schedule_lib_num = snapshot.pending_schedule.schedule_lib_num; - pending_schedule.schedule_hash = std::move(snapshot.pending_schedule.schedule_hash); + pending_schedule.schedule_hash = snapshot.pending_schedule.schedule_hash; pending_schedule.schedule = producer_authority_schedule( snapshot.pending_schedule.schedule ); activated_protocol_features = std::move(snapshot.activated_protocol_features); } diff --git a/libraries/chain/block_state.cpp b/libraries/chain/block_state.cpp index 4e5255a03e..6c5738bd7f 100644 --- a/libraries/chain/block_state.cpp +++ b/libraries/chain/block_state.cpp @@ -4,25 +4,38 @@ namespace eosio::chain { - block_state::block_state(const block_header_state& prev, signed_block_ptr b, const protocol_feature_set& pfs, - const validator_t& validator, bool skip_validate_signee) - : block_header_state(prev.next(*b, pfs, validator)) - , block(std::move(b)) - {} +block_state::block_state(const block_header_state& prev, signed_block_ptr b, const protocol_feature_set& pfs, + const validator_t& validator, bool skip_validate_signee) + : block_header_state(prev.next(*b, pfs, validator)) + , block(std::move(b)) +{} #if 0 - block_state::block_state(pending_block_header_state&& cur, - signed_block_ptr&& b, - deque&& trx_metas, - const protocol_feature_set& pfs, - const validator_t& validator, - const signer_callback_type& signer - ) +block_state::block_state(pending_block_header_state&& cur, + signed_block_ptr&& b, + deque&& trx_metas, + const protocol_feature_set& pfs, + const validator_t& validator, + const signer_callback_type& signer + ) :block_header_state( inject_additional_signatures( std::move(cur), *b, pfs, validator, signer ) ) ,block( std::move(b) ) ,_pub_keys_recovered( true ) // called by produce_block so signature recovery of trxs must have been done - ,_cached_trxs( std::move(trx_metas) ) - {} + ,cached_trxs( std::move(trx_metas) ) +{} #endif + +deque block_state::extract_trxs_metas() { + pub_keys_recovered = false; + auto result = std::move(cached_trxs); + cached_trxs.clear(); + return result; +} + +void block_state::set_trxs_metas( deque&& trxs_metas, bool keys_recovered ) { + pub_keys_recovered = keys_recovered; + cached_trxs = std::move( trxs_metas ); +} + } /// eosio::chain diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 1f491b0ceb..2fbb531992 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -2800,11 +2800,10 @@ struct controller_impl { template void apply_block( controller::block_report& br, const BSP& bsp, controller::block_status s, - const trx_meta_cache_lookup& trx_lookup ) - { try { + const trx_meta_cache_lookup& trx_lookup ) { try { - // [greg todo] remove `if`, `lambda` and `apply_dpos`, and make code work for both versions of BSP - if constexpr (std::is_same_v) { + try { + // [greg todo] remove `if`, `lambda` and `apply_dpos`, and make code work for both versions of BSP auto do_the_work = [&](auto& fork_db, auto& head) { auto start = fc::time_point::now(); const signed_block_ptr& b = bsp->block; @@ -2840,7 +2839,8 @@ struct controller_impl { } else { packed_transaction_ptr ptrx( b, &pt ); // alias signed_block_ptr auto fut = transaction_metadata::start_recover_keys( - std::move( ptrx ), thread_pool.get_executor(), chain_id, fc::microseconds::maximum(), transaction_metadata::trx_type::input ); + std::move( ptrx ), thread_pool.get_executor(), chain_id, fc::microseconds::maximum(), + transaction_metadata::trx_type::input ); trx_metas.emplace_back( transaction_metadata_ptr{}, std::move( fut ) ); } } @@ -2854,37 +2854,39 @@ struct controller_impl { for( const auto& receipt : b->transactions ) { auto num_pending_receipts = trx_receipts.size(); if( std::holds_alternative(receipt.trx) ) { - const auto& trx_meta = ( use_bsp_cached ? bsp->trxs_metas().at( packed_idx ) - : ( !!std::get<0>( trx_metas.at( packed_idx ) ) ? - std::get<0>( trx_metas.at( packed_idx ) ) - : std::get<1>( trx_metas.at( packed_idx ) ).get() ) ); - trace = push_transaction( trx_meta, fc::time_point::maximum(), fc::microseconds::maximum(), receipt.cpu_usage_us, true, 0 ); + const auto& trx_meta = (use_bsp_cached ? bsp->trxs_metas().at(packed_idx) + : (!!std::get<0>(trx_metas.at(packed_idx)) + ? std::get<0>(trx_metas.at(packed_idx)) + : std::get<1>(trx_metas.at(packed_idx)).get())); + trace = push_transaction(trx_meta, fc::time_point::maximum(), fc::microseconds::maximum(), + receipt.cpu_usage_us, true, 0); ++packed_idx; } else if( std::holds_alternative(receipt.trx) ) { - trace = push_scheduled_transaction( std::get(receipt.trx), fc::time_point::maximum(), fc::microseconds::maximum(), receipt.cpu_usage_us, true ); + trace = push_scheduled_transaction(std::get(receipt.trx), fc::time_point::maximum(), + fc::microseconds::maximum(), receipt.cpu_usage_us, true); } else { EOS_ASSERT( false, block_validate_exception, "encountered unexpected receipt type" ); } - bool transaction_failed = trace && trace->except; - bool transaction_can_fail = receipt.status == transaction_receipt_header::hard_fail && std::holds_alternative(receipt.trx); + bool transaction_failed = trace && trace->except; + bool transaction_can_fail = receipt.status == transaction_receipt_header::hard_fail && + std::holds_alternative(receipt.trx); + if( transaction_failed && !transaction_can_fail) { edump((*trace)); throw *trace->except; } - EOS_ASSERT( trx_receipts.size() > 0, - block_validate_exception, "expected a receipt, block_num ${bn}, block_id ${id}, receipt ${e}", - ("bn", b->block_num())("id", producer_block_id)("e", receipt) - ); - EOS_ASSERT( trx_receipts.size() == num_pending_receipts + 1, - block_validate_exception, "expected receipt was not added, block_num ${bn}, block_id ${id}, receipt ${e}", - ("bn", b->block_num())("id", producer_block_id)("e", receipt) - ); + EOS_ASSERT(trx_receipts.size() > 0, block_validate_exception, + "expected a receipt, block_num ${bn}, block_id ${id}, receipt ${e}", + ("bn", b->block_num())("id", producer_block_id)("e", receipt)); + EOS_ASSERT(trx_receipts.size() == num_pending_receipts + 1, block_validate_exception, + "expected receipt was not added, block_num ${bn}, block_id ${id}, receipt ${e}", + ("bn", b->block_num())("id", producer_block_id)("e", receipt)); const transaction_receipt_header& r = trx_receipts.back(); - EOS_ASSERT( r == static_cast(receipt), - block_validate_exception, "receipt does not match, ${lhs} != ${rhs}", - ("lhs", r)("rhs", static_cast(receipt)) ); + EOS_ASSERT(r == static_cast(receipt), block_validate_exception, + "receipt does not match, ${lhs} != ${rhs}", + ("lhs", r)("rhs", static_cast(receipt))); } finalize_block(); @@ -2898,8 +2900,8 @@ struct controller_impl { ab.apply_dpos([&](assembled_block::assembled_block_dpos& ab) { report_block_header_diff( *b, *ab.unsigned_block ); }); // this implicitly asserts that all header fields (less the signature) are identical - EOS_ASSERT( producer_block_id == ab.id(), block_validate_exception, "Block ID does not match", - ("producer_block_id", producer_block_id)("validator_block_id", ab.id()) ); + EOS_ASSERT(producer_block_id == ab.id(), block_validate_exception, "Block ID does not match", + ("producer_block_id", producer_block_id)("validator_block_id", ab.id())); } if( !use_bsp_cached ) { @@ -2913,22 +2915,22 @@ struct controller_impl { br.total_time = fc::time_point::now() - start; }; block_data.apply(do_the_work); + return; + } catch ( const std::bad_alloc& ) { + throw; + } catch ( const boost::interprocess::bad_alloc& ) { + throw; + } catch ( const fc::exception& e ) { + edump((e.to_detail_string())); + abort_block(); + throw; + } catch ( const std::exception& e ) { + edump((e.what())); + abort_block(); + throw; } - return; - } catch ( const std::bad_alloc& ) { - throw; - } catch ( const boost::interprocess::bad_alloc& ) { - throw; - } catch ( const fc::exception& e ) { - edump((e.to_detail_string())); - abort_block(); - throw; - } catch ( const std::exception& e ) { - edump((e.what())); - abort_block(); - throw; - } - } FC_CAPTURE_AND_RETHROW() } /// apply_block + } FC_CAPTURE_AND_RETHROW(); + } /// apply_block // thread safe, expected to be called from thread other than the main thread diff --git a/libraries/chain/include/eosio/chain/block_header_state.hpp b/libraries/chain/include/eosio/chain/block_header_state.hpp index 036a25ee45..2780d1b709 100644 --- a/libraries/chain/include/eosio/chain/block_header_state.hpp +++ b/libraries/chain/include/eosio/chain/block_header_state.hpp @@ -84,6 +84,7 @@ struct block_header_state { } flat_set get_activated_protocol_features() const { return activated_protocol_features->protocol_features; } + const vector& get_new_protocol_feature_activations() const; producer_authority get_scheduled_producer(block_timestamp_type t) const; uint32_t active_schedule_version() const; signed_block_header make_block_header(const checksum256_type& transaction_mroot, diff --git a/libraries/chain/include/eosio/chain/block_header_state_utils.hpp b/libraries/chain/include/eosio/chain/block_header_state_utils.hpp index 136392f780..72c700bca9 100644 --- a/libraries/chain/include/eosio/chain/block_header_state_utils.hpp +++ b/libraries/chain/include/eosio/chain/block_header_state_utils.hpp @@ -45,4 +45,16 @@ namespace eosio::chain::detail { return {}; } + /** + * Reference cannot outlive header_exts. Assumes header_exts is not mutated after instantiation. + */ + inline const vector& get_new_protocol_feature_activations(const header_extension_multimap& header_exts) { + static const vector no_activations{}; + + if( header_exts.count(protocol_feature_activation::extension_id()) == 0 ) + return no_activations; + + return std::get(header_exts.lower_bound(protocol_feature_activation::extension_id())->second).protocol_features; + } + } /// namespace eosio::chain diff --git a/libraries/chain/include/eosio/chain/block_state.hpp b/libraries/chain/include/eosio/chain/block_state.hpp index ee1c1e8e3d..7114365b5a 100644 --- a/libraries/chain/include/eosio/chain/block_state.hpp +++ b/libraries/chain/include/eosio/chain/block_state.hpp @@ -19,6 +19,8 @@ struct block_state : public block_header_state { // block_header_state provi // ------ data members caching information available elsewhere ---------------------- block_id_type cached_id; // cache of block_header_state::header.calculate_id() (indexed on this field) + bool pub_keys_recovered = false; + deque cached_trxs; // ------ functions ----------------------------------------------------------------- const block_id_type& id() const { return cached_id; } @@ -31,7 +33,10 @@ struct block_state : public block_header_state { // block_header_state provi uint32_t irreversible_blocknum() const { return 0; } // [greg todo] equivalent of dpos_irreversible_blocknum protocol_feature_activation_set_ptr get_activated_protocol_features() const { return block_header_state::activated_protocol_features; } - deque extract_trxs_metas() { return {}; }; // [greg todo] see impl in block_state_legacy.hpp + bool is_pub_keys_recovered() const { return pub_keys_recovered; } + deque extract_trxs_metas(); + void set_trxs_metas(deque&& trxs_metas, bool keys_recovered); + const deque& trxs_metas() const { return cached_trxs; } using bhs_t = block_header_state; using bhsp_t = block_header_state_ptr; From 854e9dbfc6fdae99560ebe040352b96fa0ced0cd Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Fri, 12 Jan 2024 18:27:07 -0500 Subject: [PATCH 18/36] Fix todo for `report_block_header_diff`. wip --- libraries/chain/controller.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 2fbb531992..0a7cbf1e5e 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -448,6 +448,13 @@ struct assembled_block { v); } + const block_header& header() const { + return std::visit( + overloaded{[](const assembled_block_dpos& ab) -> const block_header& { return *ab.unsigned_block; }, + [](const assembled_block_if& ab) -> const block_header& { return ab.bhs.header; }}, + v); + } + const producer_authority_schedule& active_producers() const { return std::visit(overloaded{[](const assembled_block_dpos& ab) -> const producer_authority_schedule& { return ab.pending_block_header_state.active_schedule; @@ -2803,7 +2810,6 @@ struct controller_impl { const trx_meta_cache_lookup& trx_lookup ) { try { try { - // [greg todo] remove `if`, `lambda` and `apply_dpos`, and make code work for both versions of BSP auto do_the_work = [&](auto& fork_db, auto& head) { auto start = fc::time_point::now(); const signed_block_ptr& b = bsp->block; @@ -2896,8 +2902,7 @@ struct controller_impl { if( producer_block_id != ab.id() ) { elog( "Validation block id does not match producer block id" ); - // [greg todo] also call `report_block_header_diff in IF mode once we have a signed_block - ab.apply_dpos([&](assembled_block::assembled_block_dpos& ab) { report_block_header_diff( *b, *ab.unsigned_block ); }); + report_block_header_diff(*b, ab.header()); // this implicitly asserts that all header fields (less the signature) are identical EOS_ASSERT(producer_block_id == ab.id(), block_validate_exception, "Block ID does not match", From 71cf33c55212156ff5a8eec82c91bbe388d0e984 Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Sat, 13 Jan 2024 10:13:01 -0500 Subject: [PATCH 19/36] remove `block_state::cached_id` and update `id()` member function. --- libraries/chain/include/eosio/chain/block_state.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libraries/chain/include/eosio/chain/block_state.hpp b/libraries/chain/include/eosio/chain/block_state.hpp index 7114365b5a..041ce447a0 100644 --- a/libraries/chain/include/eosio/chain/block_state.hpp +++ b/libraries/chain/include/eosio/chain/block_state.hpp @@ -18,12 +18,11 @@ struct block_state : public block_header_state { // block_header_state provi // ------ data members caching information available elsewhere ---------------------- - block_id_type cached_id; // cache of block_header_state::header.calculate_id() (indexed on this field) bool pub_keys_recovered = false; deque cached_trxs; // ------ functions ----------------------------------------------------------------- - const block_id_type& id() const { return cached_id; } + const block_id_type& id() const { return block_header_state::id; } const block_id_type& previous() const { return block_header_state::previous(); } uint32_t block_num() const { return block_header_state::block_num(); } block_timestamp_type timestamp() const { return block_header_state::timestamp(); } From 8934a42663836b98c5b862ecdd5d02dbaf80cf6c Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Sat, 13 Jan 2024 18:25:51 -0500 Subject: [PATCH 20/36] Add `completed_block` construction for IF mode. --- libraries/chain/block_state.cpp | 19 ++++++------------- libraries/chain/controller.cpp | 4 +++- .../chain/include/eosio/chain/block_state.hpp | 3 +++ 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/libraries/chain/block_state.cpp b/libraries/chain/block_state.cpp index 6c5738bd7f..67a40ebcc3 100644 --- a/libraries/chain/block_state.cpp +++ b/libraries/chain/block_state.cpp @@ -10,20 +10,13 @@ block_state::block_state(const block_header_state& prev, signed_block_ptr b, con , block(std::move(b)) {} -#if 0 -block_state::block_state(pending_block_header_state&& cur, - signed_block_ptr&& b, - deque&& trx_metas, - const protocol_feature_set& pfs, - const validator_t& validator, - const signer_callback_type& signer - ) - :block_header_state( inject_additional_signatures( std::move(cur), *b, pfs, validator, signer ) ) - ,block( std::move(b) ) - ,_pub_keys_recovered( true ) // called by produce_block so signature recovery of trxs must have been done - ,cached_trxs( std::move(trx_metas) ) +block_state::block_state(const block_header_state& bhs, deque&& trx_metas, + deque&& trx_receipts) + : block_header_state(bhs) + , block(std::make_shared(signed_block_header{bhs.header})) // [greg todo] do we need signatures? + , pub_keys_recovered(true) // probably not needed + , cached_trxs(std::move(trx_metas)) {} -#endif deque block_state::extract_trxs_metas() { pub_keys_recovered = false; diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 0a7cbf1e5e..eacb250315 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -509,7 +509,9 @@ struct assembled_block { return completed_block{std::move(bsp)}; }, [&](assembled_block_if& ab) { - return completed_block{}; /* [greg todo] */ + auto bsp = std::make_shared(ab.bhs, std::move(ab.trx_metas), + std::move(ab.trx_receipts)); + return completed_block{std::move(bsp)}; }}, v); } diff --git a/libraries/chain/include/eosio/chain/block_state.hpp b/libraries/chain/include/eosio/chain/block_state.hpp index 041ce447a0..3e853db596 100644 --- a/libraries/chain/include/eosio/chain/block_state.hpp +++ b/libraries/chain/include/eosio/chain/block_state.hpp @@ -44,6 +44,9 @@ struct block_state : public block_header_state { // block_header_state provi block_state(const block_header_state& prev, signed_block_ptr b, const protocol_feature_set& pfs, const validator_t& validator, bool skip_validate_signee); + + block_state(const block_header_state& bhs, deque&& trx_metas, + deque&& trx_receipts); }; using block_state_ptr = std::shared_ptr; From a482982b7830ef8d709b944a34e37eb576d0127d Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Sat, 13 Jan 2024 19:15:07 -0600 Subject: [PATCH 21/36] GH-2045 Add get_data_dir() --- libraries/chain/fork_database.cpp | 7 ++++++- libraries/chain/include/eosio/chain/fork_database.hpp | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/libraries/chain/fork_database.cpp b/libraries/chain/fork_database.cpp index 6431c7d5f1..c98a48c3d1 100644 --- a/libraries/chain/fork_database.cpp +++ b/libraries/chain/fork_database.cpp @@ -69,7 +69,7 @@ namespace eosio::chain { fork_multi_index_type index; bsp root; // Only uses the block_header_state_legacy portion bsp head; - std::filesystem::path datadir; + const std::filesystem::path datadir; explicit fork_database_impl( const std::filesystem::path& data_dir ) : datadir(data_dir) {} @@ -102,6 +102,11 @@ namespace eosio::chain { my->open_impl( validator ); } + template + std::filesystem::path fork_database::get_data_dir() const { + return my->datadir; + } + template void fork_database_impl::open_impl( validator_t& validator ) { if (!std::filesystem::is_directory(datadir)) diff --git a/libraries/chain/include/eosio/chain/fork_database.hpp b/libraries/chain/include/eosio/chain/fork_database.hpp index cdd1d13fab..d43031cf98 100644 --- a/libraries/chain/include/eosio/chain/fork_database.hpp +++ b/libraries/chain/include/eosio/chain/fork_database.hpp @@ -34,6 +34,8 @@ namespace eosio::chain { explicit fork_database( const std::filesystem::path& data_dir ); ~fork_database(); + std::filesystem::path get_data_dir() const; + void open( validator_t& validator ); void close(); From 4e631aee9d06d31edc6879484b8beb59cbe00906 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Sat, 13 Jan 2024 19:18:09 -0600 Subject: [PATCH 22/36] GH-2045 Add constructor for transition from block_state_legacy to block_state. Add missing set of transactions in block. --- libraries/chain/block_state.cpp | 25 ++++++++++++++++++- .../chain/include/eosio/chain/block_state.hpp | 6 ++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/libraries/chain/block_state.cpp b/libraries/chain/block_state.cpp index 67a40ebcc3..8630a4bcca 100644 --- a/libraries/chain/block_state.cpp +++ b/libraries/chain/block_state.cpp @@ -1,5 +1,6 @@ #include #include +#include #include namespace eosio::chain { @@ -16,7 +17,29 @@ block_state::block_state(const block_header_state& bhs, deque(signed_block_header{bhs.header})) // [greg todo] do we need signatures? , pub_keys_recovered(true) // probably not needed , cached_trxs(std::move(trx_metas)) -{} +{ + block->transactions = std::move(trx_receipts); +} + +// Used for transition from dbpos to instant-finality +block_state::block_state(const block_state_legacy& bsp) { + block_header_state::id = bsp.id(); + header = bsp.header; + activated_protocol_features = bsp.activated_protocol_features; + std::optional ext = bsp.block->extract_header_extension(instant_finality_extension::extension_id()); + assert(ext); // required by current transistion mechanism + const auto& if_extension = std::get(*ext); + assert(if_extension.new_finalizer_policy); // required by current transistion mechanism + active_finalizer_policy = std::make_shared(*if_extension.new_finalizer_policy); + active_proposer_policy = std::make_shared(); + active_proposer_policy->active_time = bsp.timestamp(); + active_proposer_policy->proposer_schedule = bsp.active_schedule; + header_exts = bsp.header_exts; // not needed, but copy over just in case + block = bsp.block; + validated = bsp.is_valid(); + pub_keys_recovered = bsp._pub_keys_recovered; + cached_trxs = bsp._cached_trxs; +} deque block_state::extract_trxs_metas() { pub_keys_recovered = false; diff --git a/libraries/chain/include/eosio/chain/block_state.hpp b/libraries/chain/include/eosio/chain/block_state.hpp index 3e853db596..39cb9a49ab 100644 --- a/libraries/chain/include/eosio/chain/block_state.hpp +++ b/libraries/chain/include/eosio/chain/block_state.hpp @@ -7,10 +7,12 @@ namespace eosio::chain { +struct block_state_legacy; + struct block_state : public block_header_state { // block_header_state provides parent link // ------ data members ------------------------------------------------------------- signed_block_ptr block; - bool validated; // We have executed the block's trxs and verified that action merkle root (block id) matches. + bool validated = false; // We have executed the block's trxs and verified that action merkle root (block id) matches. digest_type strong_digest; // finalizer_digest (strong, cached so we can quickly validate votes) digest_type weak_digest; // finalizer_digest (weak, cached so we can quickly validate votes) pending_quorum_certificate pending_qc; // where we accumulate votes we receive @@ -47,6 +49,8 @@ struct block_state : public block_header_state { // block_header_state provi block_state(const block_header_state& bhs, deque&& trx_metas, deque&& trx_receipts); + + explicit block_state(const block_state_legacy& bsp); }; using block_state_ptr = std::shared_ptr; From abdfa68dcd32356579343761a467974a93b6eef8 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Sat, 13 Jan 2024 19:19:01 -0600 Subject: [PATCH 23/36] GH-2045 Fix when active_producer_policy updated, done at end of block not at beginning. --- libraries/chain/block_header_state.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/chain/block_header_state.cpp b/libraries/chain/block_header_state.cpp index b50c95dfab..23499533a2 100644 --- a/libraries/chain/block_header_state.cpp +++ b/libraries/chain/block_header_state.cpp @@ -96,7 +96,8 @@ block_header_state block_header_state::next(block_header_state_input& input) con if(!proposer_policies.empty()) { auto it = proposer_policies.begin(); - if (it->first <= input.timestamp) { + // -1 since this is called after the block is built, this will be the active schedule for the next block + if (it->first.slot <= input.timestamp.slot - 1) { result.active_proposer_policy = it->second; result.header.schedule_version = header.schedule_version + 1; result.active_proposer_policy->proposer_schedule.version = result.header.schedule_version; From 61a595f943bce065ee2e82b11e174b8eca3bc135 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Sat, 13 Jan 2024 19:21:09 -0600 Subject: [PATCH 24/36] GH-2045 Add transition from dpos fork database to instant finality fork database. Remove fetch_block_state_* methods. --- libraries/chain/controller.cpp | 168 ++++++++++-------- .../eosio/chain/block_state_legacy.hpp | 1 + .../chain/include/eosio/chain/controller.hpp | 4 - 3 files changed, 91 insertions(+), 82 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 81fb46b681..1e8d5065bd 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -127,7 +127,7 @@ struct block_data_gen_t { bsp head; fork_db_t fork_db; - block_data_gen_t(const std::filesystem::path& path) : fork_db(path) {} + explicit block_data_gen_t(const std::filesystem::path& path) : fork_db(path) {} bsp fork_db_head(bool irreversible_mode) const { if (irreversible_mode) { @@ -177,6 +177,19 @@ struct block_data_t { uint32_t head_block_num() const { return std::visit([](const auto& bd) { return bd.head->block_num(); }, v); } block_timestamp_type head_block_time() const { return std::visit([](const auto& bd) { return bd.head->timestamp(); }, v); } account_name head_block_producer() const { return std::visit([](const auto& bd) { return bd.head->producer(); }, v); } + + void transistion_fork_db_to_if(const auto& vbsp) { + std::visit([](auto& bd) { bd.fork_db.close(); }, v); + auto bsp = std::make_shared(*std::get(vbsp)); + v.emplace(std::visit([](const auto& bd) { return bd.fork_db.get_data_dir(); }, v)); + std::visit(overloaded{ + [](block_data_legacy_t&) {}, + [&](block_data_new_t& bd) { + bd.head = bsp; + bd.fork_db.reset(*bd.head); + } + }, v); + } protocol_feature_activation_set_ptr head_activated_protocol_features() const { return std::visit([](const auto& bd) { return bd.head->get_activated_protocol_features(); }, v); @@ -863,8 +876,8 @@ struct building_block { bb_input, transaction_mroot, action_mroot, std::move(bb.new_proposer_policy), std::move(bb.new_finalizer_policy), qc_data ? qc_data->qc_info : std::optional{} }; - assembled_block::assembled_block_if ab{bb.active_producer_authority, bb.parent.next(bhs_input), - bb.pending_trx_metas, bb.pending_trx_receipts, + assembled_block::assembled_block_if ab{std::move(bb.active_producer_authority), bb.parent.next(bhs_input), + std::move(bb.pending_trx_metas), std::move(bb.pending_trx_receipts), qc_data ? std::move(qc_data->qc) : std::optional{}}; return assembled_block{.v = std::move(ab)}; @@ -2444,7 +2457,6 @@ struct controller_impl { // can change during start_block, so use same value throughout uint32_t hs_lib = hs_irreversible_block_num.load(); - const bool hs_active = hs_lib > 0; // the transition from 0 to >0 cannot happen during start_block emit( self.block_start, head_block_num() + 1 ); @@ -2548,37 +2560,35 @@ struct controller_impl { const auto& gpo = self.get_global_properties(); - if (!hs_active) { -#warning todo: how do we update the producer_schedule after the switch to IF? - bb.apply_dpos([&](building_block::building_block_dpos& bb_dpos) { - pending_block_header_state_legacy& pbhs = bb_dpos.pending_block_header_state; - - if( gpo.proposed_schedule_block_num && // if there is a proposed schedule that was proposed in a block ... - ( hs_active || *gpo.proposed_schedule_block_num <= pbhs.dpos_irreversible_blocknum ) && // ... that has now become irreversible or hotstuff activated... - pbhs.prev_pending_schedule.schedule.producers.size() == 0 // ... and there was room for a new pending schedule prior to any possible promotion - ) - { - // Promote proposed schedule to pending schedule; happens in next block after hotstuff activated - EOS_ASSERT( gpo.proposed_schedule.version == pbhs.active_schedule_version + 1, - producer_schedule_exception, "wrong producer schedule version specified" ); - - bb_dpos.new_pending_producer_schedule = producer_authority_schedule::from_shared(gpo.proposed_schedule); - - if( !replaying ) { - ilog( "promoting proposed schedule (set in block ${proposed_num}) to pending; current block: ${n} lib: ${lib} schedule: ${schedule} ", - ("proposed_num", *gpo.proposed_schedule_block_num)("n", pbhs.block_num) - ("lib", hs_active ? hs_lib : pbhs.dpos_irreversible_blocknum) - ("schedule", bb_dpos.new_pending_producer_schedule ) ); - } - - db.modify( gpo, [&]( auto& gp ) { - gp.proposed_schedule_block_num = std::optional(); - gp.proposed_schedule.version=0; - gp.proposed_schedule.producers.clear(); - }); + // instant finality uses alternative method for chaning producer schedule + bb.apply_dpos([&](building_block::building_block_dpos& bb_dpos) { + pending_block_header_state_legacy& pbhs = bb_dpos.pending_block_header_state; + + if( gpo.proposed_schedule_block_num && // if there is a proposed schedule that was proposed in a block ... + ( *gpo.proposed_schedule_block_num <= pbhs.dpos_irreversible_blocknum ) && // ... that has now become irreversible ... + pbhs.prev_pending_schedule.schedule.producers.size() == 0 // ... and there was room for a new pending schedule prior to any possible promotion + ) + { + // Promote proposed schedule to pending schedule; happens in next block after hotstuff activated + EOS_ASSERT( gpo.proposed_schedule.version == pbhs.active_schedule_version + 1, + producer_schedule_exception, "wrong producer schedule version specified" ); + + bb_dpos.new_pending_producer_schedule = producer_authority_schedule::from_shared(gpo.proposed_schedule); + + if( !replaying ) { + ilog( "promoting proposed schedule (set in block ${proposed_num}) to pending; current block: ${n} lib: ${lib} schedule: ${schedule} ", + ("proposed_num", *gpo.proposed_schedule_block_num)("n", pbhs.block_num) + ("lib", pbhs.dpos_irreversible_blocknum) + ("schedule", bb_dpos.new_pending_producer_schedule ) ); } - }); - } + + db.modify( gpo, [&]( auto& gp ) { + gp.proposed_schedule_block_num = std::optional(); + gp.proposed_schedule.version=0; + gp.proposed_schedule.producers.clear(); + }); + } + }); try { transaction_metadata_ptr onbtrx = @@ -2619,7 +2629,7 @@ struct controller_impl { void finish_block() { EOS_ASSERT( pending, block_validate_exception, "it is not valid to finalize when there is no pending block"); - EOS_ASSERT( std::holds_alternative(pending->_block_stage), block_validate_exception, "already called finalize_block"); + EOS_ASSERT( std::holds_alternative(pending->_block_stage), block_validate_exception, "already called finish_block"); try { auto& bb = std::get(pending->_block_stage); @@ -2646,24 +2656,9 @@ struct controller_impl { create_block_summary( assembled_block.id() ); pending->_block_stage = std::move(assembled_block); - - /* - ilog( "finalized block ${n} (${id}) at ${t} by ${p} (${signing_key}); schedule_version: ${v} lib: ${lib} #dtrxs: ${ndtrxs} ${np}", - ("n",pbhs.block_num()) - ("id",id) - ("t",pbhs.timestamp) - ("p",pbhs.producer) - ("signing_key", pbhs.block_signing_key) - ("v",pbhs.active_schedule_version) - ("lib",pbhs.dpos_irreversible_blocknum) - ("ndtrxs",db.get_index().size()) - ("np",block_ptr->new_producers) - ); - */ - } FC_CAPTURE_AND_RETHROW() - } /// finalize_block + } /** * @post regardless of the success of commit block there is no active pending block @@ -2707,8 +2702,28 @@ struct controller_impl { if( s == controller::block_status::incomplete ) { log_irreversible(); - pacemaker->beat(); } + + // TODO: temp transistion to instant-finality, happens immediately after block with new_finalizer_policy + auto transition = [&](auto& fork_db, auto& head) -> bool { + const auto& bsp = std::get>(cb.bsp); + std::optional ext = bsp->block->extract_header_extension(instant_finality_extension::extension_id()); + if (ext) { + const auto& if_extension = std::get(*ext); + if (if_extension.new_finalizer_policy) { + ilog("Transition to instant finality happening after block ${b}", ("b", bsp->block_num())); + hs_irreversible_block_num = bsp->block_num(); + + log_irreversible(); + return true; + } + } + return false; + }; + if (block_data.apply_dpos(transition)) { + block_data.transistion_fork_db_to_if(cb.bsp); + } + } catch (...) { // dont bother resetting pending, instead abort the block reset_pending_on_exit.cancel(); @@ -3994,48 +4009,45 @@ std::optional controller::fetch_block_header_by_id( const b } signed_block_ptr controller::fetch_block_by_number( uint32_t block_num )const { try { - auto blk_state = fetch_block_state_by_number( block_num ); - if( blk_state ) { - return blk_state->block; - } + auto fetch_block = [&](auto& fork_db, auto& head) -> signed_block_ptr { + auto bsp = fork_db.search_on_branch( fork_db.head()->id(), block_num); + if (bsp) return bsp->block; + return {}; + }; + auto b = my->block_data.apply(fetch_block); + if (b) + return b; return my->blog.read_block_by_num(block_num); } FC_CAPTURE_AND_RETHROW( (block_num) ) } std::optional controller::fetch_block_header_by_number( uint32_t block_num )const { try { - auto blk_state = fetch_block_state_by_number( block_num ); - if( blk_state ) { - return blk_state->header; - } + auto fetch_block = [&](auto& fork_db, auto& head) -> signed_block_ptr { + auto bsp = fork_db.search_on_branch( fork_db.head()->id(), block_num); + if (bsp) return bsp->block; + return {}; + }; + auto b = my->block_data.apply(fetch_block); + if (b) + return *b; return my->blog.read_block_header_by_num(block_num); } FC_CAPTURE_AND_RETHROW( (block_num) ) } -block_state_legacy_ptr controller::fetch_block_state_by_id( block_id_type id )const { - // returns nullptr when in IF mode - auto get_block_state = [&](auto& fork_db, auto& head) -> block_state_legacy_ptr { return fork_db.get_block(id); }; - return my->block_data.apply_dpos(get_block_state); -} - -block_state_legacy_ptr controller::fetch_block_state_by_number( uint32_t block_num )const { - try { - // returns nullptr when in IF mode - auto fetch_block_state = [&](auto& fork_db, auto& head) -> block_state_legacy_ptr { - return fork_db.search_on_branch( fork_db.head()->id(), block_num); - }; - return my->block_data.apply_dpos(fetch_block_state); - } FC_CAPTURE_AND_RETHROW( (block_num) ) -} - block_id_type controller::get_block_id_for_num( uint32_t block_num )const { try { const auto& blog_head = my->blog.head(); bool find_in_blog = (blog_head && block_num <= blog_head->block_num()); if( !find_in_blog ) { - auto bsp = fetch_block_state_by_number( block_num ); - if( bsp ) return bsp->id(); + auto fetch_block_id = [&](auto& fork_db, auto& head) -> std::optional { + auto bsp = fork_db.search_on_branch( fork_db.head()->id(), block_num); + if (bsp) return bsp->id(); + return {}; + }; + auto id = my->block_data.apply>(fetch_block_id); + if (id) return *id; } auto id = my->blog.read_block_id_by_num(block_num); diff --git a/libraries/chain/include/eosio/chain/block_state_legacy.hpp b/libraries/chain/include/eosio/chain/block_state_legacy.hpp index fc267d813c..72de14c279 100644 --- a/libraries/chain/include/eosio/chain/block_state_legacy.hpp +++ b/libraries/chain/include/eosio/chain/block_state_legacy.hpp @@ -52,6 +52,7 @@ namespace eosio::chain { friend struct fc::reflector; friend struct controller_impl; friend struct completed_block; + friend struct block_state; bool is_pub_keys_recovered()const { return _pub_keys_recovered; } diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index c7ccd89a9c..02c2f264d5 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -287,10 +287,6 @@ namespace eosio::chain { std::optional fetch_block_header_by_number( uint32_t block_num )const; // thread-safe std::optional fetch_block_header_by_id( const block_id_type& id )const; - // return block_state_legacy from forkdb, thread-safe - block_state_legacy_ptr fetch_block_state_by_number( uint32_t block_num )const; - // return block_state_legacy from forkdb, thread-safe - block_state_legacy_ptr fetch_block_state_by_id( block_id_type id )const; // thread-safe block_id_type get_block_id_for_num( uint32_t block_num )const; From c3e8423f50323780365cc9def52385e7562d26bb Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 15 Jan 2024 09:31:05 -0600 Subject: [PATCH 25/36] GH-2045 Fix assert and use block exception type --- libraries/chain/block_header_state.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chain/block_header_state.cpp b/libraries/chain/block_header_state.cpp index 23499533a2..cce57cfcc1 100644 --- a/libraries/chain/block_header_state.cpp +++ b/libraries/chain/block_header_state.cpp @@ -182,7 +182,7 @@ block_header_state block_header_state::next(const signed_block_header& h, const // retrieve instant_finality_extension data from block header extension // -------------------------------------------------------------------- - EOS_ASSERT(exts.count(instant_finality_extension::extension_id() > 0), misc_exception, + EOS_ASSERT(exts.count(instant_finality_extension::extension_id()) > 0, invalid_block_header_extension, "Instant Finality Extension is expected to be present in all block headers after switch to IF"); auto if_entry = exts.lower_bound(instant_finality_extension::extension_id()); auto& if_ext = std::get(if_entry->second); From 07f1036f7eedd329de29f33b5e1350e6411d4923 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 15 Jan 2024 10:13:46 -0600 Subject: [PATCH 26/36] GH-2045 Add block_token as a wrapper of block_state and block_state_legacy. Also add block_exists as a performance improvement. --- libraries/chain/controller.cpp | 101 +++++++++++------- .../chain/include/eosio/chain/controller.hpp | 43 ++++---- .../chain/unapplied_transaction_queue.hpp | 14 +-- .../testing/include/eosio/testing/tester.hpp | 12 +-- libraries/testing/tester.cpp | 10 +- .../include/eosio/chain/plugin_interface.hpp | 2 +- plugins/chain_plugin/chain_plugin.cpp | 6 +- .../eosio/chain_plugin/chain_plugin.hpp | 2 +- plugins/net_plugin/net_plugin.cpp | 29 ++--- plugins/producer_plugin/producer_plugin.cpp | 23 ++-- tests/CMakeLists.txt | 2 + tests/TestHarness/Cluster.py | 4 + unittests/api_tests.cpp | 46 +++++--- unittests/block_tests.cpp | 8 +- unittests/chain_tests.cpp | 2 + unittests/forked_tests.cpp | 6 +- unittests/protocol_feature_tests.cpp | 4 +- .../unapplied_transaction_queue_tests.cpp | 36 +++++-- 18 files changed, 200 insertions(+), 150 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 1e8d5065bd..48e8d4b689 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -2835,7 +2835,7 @@ struct controller_impl { auto producer_block_id = bsp->id(); start_block( b->timestamp, b->confirmed, new_protocol_feature_activations, s, producer_block_id, fc::time_point::maximum() ); - // validated in create_block_state_future() + // validated in create_block_token() std::get(pending->_block_stage).trx_mroot_or_receipt_digests() = b->transaction_mroot; const bool existing_trxs_metas = !bsp->trxs_metas().empty(); @@ -2956,14 +2956,8 @@ struct controller_impl { // thread safe, expected to be called from thread other than the main thread - block_state_legacy_ptr create_block_state_i( const block_id_type& id, const signed_block_ptr& b, const block_header_state_legacy& prev ) { - // There is a small race condition at time of activation where create_block_state_i - // is called right before hs_irreversible_block_num is set. If that happens, - // the block is considered invalid, and the node will attempt to sync the block - // in the future and succeed - uint32_t instant_finality_lib = hs_irreversible_block_num.load(); - const bool instant_finality_active = instant_finality_lib > 0; - auto trx_mroot = calculate_trx_merkle( b->transactions, instant_finality_active ); + block_token create_block_state_i( const block_id_type& id, const signed_block_ptr& b, const block_header_state_legacy& prev ) { + auto trx_mroot = calculate_trx_merkle( b->transactions, false ); EOS_ASSERT( b->transaction_mroot == trx_mroot, block_validate_exception, "invalid block transaction merkle root ${b} != ${c}", ("b", b->transaction_mroot)("c", trx_mroot) ); @@ -2981,13 +2975,36 @@ struct controller_impl { EOS_ASSERT( id == bsp->id(), block_validate_exception, "provided id ${id} does not match block id ${bid}", ("id", id)("bid", bsp->id()) ); - return bsp; + return block_token{bsp}; } - std::future create_block_state_future( const block_id_type& id, const signed_block_ptr& b ) { + // thread safe, expected to be called from thread other than the main thread + block_token create_block_state_i( const block_id_type& id, const signed_block_ptr& b, const block_header_state& prev ) { + auto trx_mroot = calculate_trx_merkle( b->transactions, true ); + EOS_ASSERT( b->transaction_mroot == trx_mroot, block_validate_exception, + "invalid block transaction merkle root ${b} != ${c}", ("b", b->transaction_mroot)("c", trx_mroot) ); + + const bool skip_validate_signee = false; + auto bsp = std::make_shared( + prev, + b, + protocol_features.get_protocol_feature_set(), + [this]( block_timestamp_type timestamp, + const flat_set& cur_features, + const vector& new_features ) + { check_protocol_features( timestamp, cur_features, new_features ); }, + skip_validate_signee + ); + + EOS_ASSERT( id == bsp->id(), block_validate_exception, + "provided id ${id} does not match block id ${bid}", ("id", id)("bid", bsp->id()) ); + return block_token{bsp}; + } + + std::future create_block_token_future( const block_id_type& id, const signed_block_ptr& b ) { EOS_ASSERT( b, block_validate_exception, "null block" ); - auto f = [&](auto& fork_db, auto& head) -> std::future { + auto f = [&](auto& fork_db, auto& head) -> std::future { return post_async_task( thread_pool.get_executor(), [b, id, &fork_db, control=this]() { // no reason for a block_state if fork_db already knows about block auto existing = fork_db.get_block( id ); @@ -3001,14 +3018,14 @@ struct controller_impl { } ); }; - return block_data.apply_dpos>(f); // [greg todo] make it work with apply() + return block_data.apply>(f); // [greg todo] make it work with apply() } // thread safe, expected to be called from thread other than the main thread - block_state_legacy_ptr create_block_state( const block_id_type& id, const signed_block_ptr& b ) { + std::optional create_block_token( const block_id_type& id, const signed_block_ptr& b ) { EOS_ASSERT( b, block_validate_exception, "null block" ); - auto f = [&](auto& fork_db, auto& head) -> block_state_legacy_ptr { + auto f = [&](auto& fork_db, auto& head) -> std::optional { // no reason for a block_state if fork_db already knows about block auto existing = fork_db.get_block( id ); EOS_ASSERT( !existing, fork_database_exception, "we already know about this block: ${id}", ("id", id) ); @@ -3017,16 +3034,16 @@ struct controller_impl { auto prev = fork_db.get_block_header( b->previous ); if( !prev ) return {}; - return create_block_state_i( id, b, *prev ); // [greg todo] make it work with apply() - if `create_block_state` needed + return create_block_state_i( id, b, *prev ); }; - return block_data.apply_dpos(f); + return block_data.apply>(f); } template void push_block( controller::block_report& br, const BSP& bsp, - const forked_branch_callback_t& forked_branch_cb, + const forked_callback_t& forked_branch_cb, const trx_meta_cache_lookup& trx_lookup ) { controller::block_status s = controller::block_status::complete; @@ -3128,7 +3145,7 @@ struct controller_impl { template void maybe_switch_forks( controller::block_report& br, const BSP& new_head, controller::block_status s, - const forked_branch_callback_t& forked_branch_cb, const trx_meta_cache_lookup& trx_lookup ) + const forked_callback_t& forked_cb, const trx_meta_cache_lookup& trx_lookup ) { auto do_maybe_switch_forks = [&](auto& fork_db, auto& head) { bool head_changed = true; @@ -3157,9 +3174,15 @@ struct controller_impl { EOS_ASSERT( self.head_block_id() == branches.second.back()->header.previous, fork_database_exception, "loss of sync between fork_db and chainbase during fork switch" ); // _should_ never fail - if( forked_branch_cb ) - if constexpr (std::is_same_v>) - forked_branch_cb(branches.second); + if( forked_cb ) { + // forked_branch is in reverse order, maintain execution order + for( auto ritr = branches.second.rbegin(), rend = branches.second.rend(); ritr != rend; ++ritr ) { + const auto& bsptr = *ritr; + for( auto itr = bsptr->trxs_metas().begin(), end = bsptr->trxs_metas().end(); itr != end; ++itr ) { + forked_cb(*itr); + } + } + } } for( auto ritr = branches.first.rbegin(); ritr != branches.first.rend(); ++ritr ) { @@ -3799,33 +3822,23 @@ boost::asio::io_context& controller::get_thread_pool() { return my->thread_pool.get_executor(); } -std::future controller::create_block_state_future( const block_id_type& id, const signed_block_ptr& b ) { - return my->create_block_state_future( id, b ); +std::future controller::create_block_token_future( const block_id_type& id, const signed_block_ptr& b ) { + return my->create_block_token_future( id, b ); } -block_state_legacy_ptr controller::create_block_state( const block_id_type& id, const signed_block_ptr& b ) const { - return my->create_block_state( id, b ); +std::optional controller::create_block_token( const block_id_type& id, const signed_block_ptr& b ) const { + return my->create_block_token( id, b ); } -void controller::push_block( controller::block_report& br, - const block_state_legacy_ptr& bsp, - const forked_branch_callback_legacy& forked_branch_cb, +void controller::push_block( block_report& br, + const block_token& bt, + const forked_callback_t& forked_cb, const trx_meta_cache_lookup& trx_lookup ) { validate_db_available_size(); - my->push_block( br, bsp, forked_branch_cb, trx_lookup ); + std::visit([&](const auto& bsp) { my->push_block( br, bsp, forked_cb, trx_lookup); }, bt.bsp); } -void controller::push_block( controller::block_report& br, - const block_state_ptr& bsp, - const forked_branch_callback& forked_branch_cb, - const trx_meta_cache_lookup& trx_lookup ) -{ - validate_db_available_size(); - my->push_block( br, bsp, forked_branch_cb, trx_lookup ); -} - - transaction_trace_ptr controller::push_transaction( const transaction_metadata_ptr& trx, fc::time_point block_deadline, fc::microseconds max_transaction_time, uint32_t billed_cpu_time_us, bool explicit_billed_cpu_time, @@ -3994,6 +4007,14 @@ signed_block_ptr controller::fetch_block_by_id( const block_id_type& id )const { return signed_block_ptr(); } +bool controller::block_exists(const block_id_type&id) const { + auto sb_ptr = my->block_data.fork_db_fetch_block_by_id(id); + if( sb_ptr ) return true; + auto bptr = my->blog.read_block_header_by_num( block_header::num_from_id(id) ); + if( bptr && bptr->calculate_id() == id ) return true; + return false; +} + std::optional controller::fetch_block_header_by_id( const block_id_type& id )const { #if 0 // [greg todo] is the below code equivalent?? diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index 02c2f264d5..cef7c8df25 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -51,23 +51,24 @@ namespace eosio::chain { using resource_limits::resource_limits_manager; using apply_handler = std::function; - template - using branch_type_t = fork_database::branch_type; - - using branch_type_legacy = branch_type_t; - using branch_type = branch_type_t; - - template - using forked_branch_callback_t = std::function&)>; - - using forked_branch_callback_legacy = forked_branch_callback_t; - using forked_branch_callback = forked_branch_callback_t; + using forked_callback_t = std::function; // lookup transaction_metadata via supplied function to avoid re-creation using trx_meta_cache_lookup = std::function; using block_signal_params = std::tuple; + // Created via create_block_token(const block_id_type& id, const signed_block_ptr& b) + // Valid to request id and signed_block_ptr it was created from. + // Avoid using internal block_state/block_state_legacy as those types are internal to controller. + struct block_token { + std::variant bsp; + + uint32_t block_num() const { return std::visit([](const auto& bsp) { return bsp->block_num(); }, bsp); } + block_id_type id() const { return std::visit([](const auto& bsp) { return bsp->id(); }, bsp); } + signed_block_ptr block() const { return std::visit([](const auto& bsp) { return bsp->block; }, bsp); } + }; + enum class db_read_mode { HEAD, IRREVERSIBLE, @@ -186,26 +187,22 @@ namespace eosio::chain { void finish_block( block_report& br, const signer_callback_type& signer_callback ); void sign_block( const signer_callback_type& signer_callback ); void commit_block(); - + // thread-safe - std::future create_block_state_future( const block_id_type& id, const signed_block_ptr& b ); + std::future create_block_token_future( const block_id_type& id, const signed_block_ptr& b ); // thread-safe - block_state_legacy_ptr create_block_state( const block_id_type& id, const signed_block_ptr& b ) const; + // returns empty optional if block b is not immediately ready to be processed + std::optional create_block_token( const block_id_type& id, const signed_block_ptr& b ) const; /** * @param br returns statistics for block - * @param bsp block to push + * @param bt block to push, created by create_block_token * @param cb calls cb with forked applied transactions for each forked block * @param trx_lookup user provided lookup function for externally cached transaction_metadata */ void push_block( block_report& br, - const block_state_legacy_ptr& bsp, - const forked_branch_callback_legacy& cb, - const trx_meta_cache_lookup& trx_lookup ); - - void push_block( block_report& br, - const block_state_ptr& bsp, - const forked_branch_callback& cb, + const block_token& bt, + const forked_callback_t& cb, const trx_meta_cache_lookup& trx_lookup ); boost::asio::io_context& get_thread_pool(); @@ -284,6 +281,8 @@ namespace eosio::chain { // thread-safe signed_block_ptr fetch_block_by_id( const block_id_type& id )const; // thread-safe + bool block_exists( const block_id_type& id)const; + // thread-safe std::optional fetch_block_header_by_number( uint32_t block_num )const; // thread-safe std::optional fetch_block_header_by_id( const block_id_type& id )const; diff --git a/libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp b/libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp index 4a28f9eb51..e1231bedcb 100644 --- a/libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp +++ b/libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp @@ -133,17 +133,9 @@ class unapplied_transaction_queue { } } - template - void add_forked( const BRANCH_TYPE& forked_branch ) { - // forked_branch is in reverse order - for( auto ritr = forked_branch.rbegin(), rend = forked_branch.rend(); ritr != rend; ++ritr ) { - const auto& bsptr = *ritr; - for( auto itr = bsptr->trxs_metas().begin(), end = bsptr->trxs_metas().end(); itr != end; ++itr ) { - const auto& trx = *itr; - auto insert_itr = queue.insert( { trx, trx_enum_type::forked } ); - if( insert_itr.second ) added( insert_itr.first ); - } - } + void add_forked( const transaction_metadata_ptr& trx ) { + auto insert_itr = queue.insert( { trx, trx_enum_type::forked } ); + if( insert_itr.second ) added( insert_itr.first ); } void add_aborted( deque aborted_trxs ) { diff --git a/libraries/testing/include/eosio/testing/tester.hpp b/libraries/testing/include/eosio/testing/tester.hpp index eeabf71061..c359e761db 100644 --- a/libraries/testing/include/eosio/testing/tester.hpp +++ b/libraries/testing/include/eosio/testing/tester.hpp @@ -606,9 +606,9 @@ namespace eosio { namespace testing { signed_block_ptr produce_block( fc::microseconds skip_time = fc::milliseconds(config::block_interval_ms) )override { auto sb = _produce_block(skip_time, false); - auto bsf = validating_node->create_block_state_future( sb->calculate_id(), sb ); + auto btf = validating_node->create_block_token_future( sb->calculate_id(), sb ); controller::block_report br; - validating_node->push_block( br, bsf.get(), {}, trx_meta_cache_lookup{} ); + validating_node->push_block( br, btf.get(), {}, trx_meta_cache_lookup{} ); return sb; } @@ -618,17 +618,17 @@ namespace eosio { namespace testing { } void validate_push_block(const signed_block_ptr& sb) { - auto bsf = validating_node->create_block_state_future( sb->calculate_id(), sb ); + auto btf = validating_node->create_block_token_future( sb->calculate_id(), sb ); controller::block_report br; - validating_node->push_block( br, bsf.get(), {}, trx_meta_cache_lookup{} ); + validating_node->push_block( br, btf.get(), {}, trx_meta_cache_lookup{} ); } signed_block_ptr produce_empty_block( fc::microseconds skip_time = fc::milliseconds(config::block_interval_ms) )override { unapplied_transactions.add_aborted( control->abort_block() ); auto sb = _produce_block(skip_time, true); - auto bsf = validating_node->create_block_state_future( sb->calculate_id(), sb ); + auto btf = validating_node->create_block_token_future( sb->calculate_id(), sb ); controller::block_report br; - validating_node->push_block( br, bsf.get(), {}, trx_meta_cache_lookup{} ); + validating_node->push_block( br, btf.get(), {}, trx_meta_cache_lookup{} ); return sb; } diff --git a/libraries/testing/tester.cpp b/libraries/testing/tester.cpp index 80e098e689..d4b97fc002 100644 --- a/libraries/testing/tester.cpp +++ b/libraries/testing/tester.cpp @@ -376,11 +376,11 @@ namespace eosio { namespace testing { } void base_tester::push_block(signed_block_ptr b) { - auto bsf = control->create_block_state_future(b->calculate_id(), b); + auto btf = control->create_block_token_future(b->calculate_id(), b); unapplied_transactions.add_aborted( control->abort_block() ); controller::block_report br; - control->push_block( br, bsf.get(), [this]( const branch_type_legacy& forked_branch ) { - unapplied_transactions.add_forked( forked_branch ); + control->push_block( br, btf.get(), [this]( const transaction_metadata_ptr& trx ) { + unapplied_transactions.add_forked( trx ); }, [this]( const transaction_id_type& id ) { return unapplied_transactions.get_trx( id ); } ); @@ -1115,10 +1115,10 @@ namespace eosio { namespace testing { auto block = a.control->fetch_block_by_number(i); if( block ) { //&& !b.control->is_known_block(block->id()) ) { - auto bsf = b.control->create_block_state_future( block->calculate_id(), block ); + auto btf = b.control->create_block_token_future( block->calculate_id(), block ); b.control->abort_block(); controller::block_report br; - b.control->push_block(br, bsf.get(), {}, trx_meta_cache_lookup{}); //, eosio::chain::validation_steps::created_block); + b.control->push_block(br, btf.get(), {}, trx_meta_cache_lookup{}); //, eosio::chain::validation_steps::created_block); } } }; diff --git a/plugins/chain_interface/include/eosio/chain/plugin_interface.hpp b/plugins/chain_interface/include/eosio/chain/plugin_interface.hpp index 4fe1c247d4..2bd78ac800 100644 --- a/plugins/chain_interface/include/eosio/chain/plugin_interface.hpp +++ b/plugins/chain_interface/include/eosio/chain/plugin_interface.hpp @@ -33,7 +33,7 @@ namespace eosio::chain::plugin_interface { namespace incoming { namespace methods { // synchronously push a block/trx to a single provider, block_state_legacy_ptr may be null - using block_sync = method_decl&, const block_state_legacy_ptr&), first_provider_policy>; + using block_sync = method_decl&, const std::optional&), first_provider_policy>; using transaction_async = method_decl), first_provider_policy>; } } diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp index 56761924a6..41f897b224 100644 --- a/plugins/chain_plugin/chain_plugin.cpp +++ b/plugins/chain_plugin/chain_plugin.cpp @@ -1183,8 +1183,8 @@ chain_apis::read_only chain_plugin::get_read_only_api(const fc::microseconds& ht } -bool chain_plugin::accept_block(const signed_block_ptr& block, const block_id_type& id, const block_state_legacy_ptr& bsp ) { - return my->incoming_block_sync_method(block, id, bsp); +bool chain_plugin::accept_block(const signed_block_ptr& block, const block_id_type& id, const std::optional& obt ) { + return my->incoming_block_sync_method(block, id, obt); } void chain_plugin::accept_transaction(const chain::packed_transaction_ptr& trx, next_function next) { @@ -2013,7 +2013,7 @@ fc::variant read_only::get_block_info(const read_only::get_block_info_params& pa void read_write::push_block(read_write::push_block_params&& params, next_function next) { try { - app().get_method()(std::make_shared( std::move(params) ), std::optional{}, block_state_legacy_ptr{}); + app().get_method()(std::make_shared( std::move(params) ), std::optional{}, std::optional{}); } catch ( boost::interprocess::bad_alloc& ) { handle_db_exhaustion(); } catch ( const std::bad_alloc& ) { diff --git a/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp b/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp index cd1fd5b0aa..5e5d5b3701 100644 --- a/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp +++ b/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp @@ -1012,7 +1012,7 @@ class chain_plugin : public plugin { chain_apis::read_write get_read_write_api(const fc::microseconds& http_max_response_time); chain_apis::read_only get_read_only_api(const fc::microseconds& http_max_response_time) const; - bool accept_block( const chain::signed_block_ptr& block, const chain::block_id_type& id, const chain::block_state_legacy_ptr& bsp ); + bool accept_block( const chain::signed_block_ptr& block, const chain::block_id_type& id, const std::optional& obt ); void accept_transaction(const chain::packed_transaction_ptr& trx, chain::plugin_interface::next_function next); // Only call this after plugin_initialize()! diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 967686ef4f..7452479497 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -1100,7 +1100,7 @@ namespace eosio { // returns calculated number of blocks combined latency uint32_t calc_block_latency(); - void process_signed_block( const block_id_type& id, signed_block_ptr block, block_state_legacy_ptr bsp ); + void process_signed_block( const block_id_type& id, signed_block_ptr block, const std::optional& obt ); fc::variant_object get_logger_variant() const { fc::mutable_variant_object mvo; @@ -3718,7 +3718,7 @@ namespace eosio { controller& cc = my_impl->chain_plug->chain(); // may have come in on a different connection and posted into dispatcher strand before this one - if( my_impl->dispatcher->have_block( id ) || cc.fetch_block_by_id( id ) ) { // thread-safe + if( my_impl->dispatcher->have_block( id ) || cc.block_exists( id ) ) { // thread-safe my_impl->dispatcher->add_peer_block( id, c->connection_id ); c->strand.post( [c, id]() { my_impl->sync_master->sync_recv_block( c, id, block_header::num_from_id(id), false ); @@ -3726,11 +3726,11 @@ namespace eosio { return; } - block_state_legacy_ptr bsp; + std::optional obt; bool exception = false; try { // this may return null if block is not immediately ready to be processed - bsp = cc.create_block_state( id, ptr ); + obt = cc.create_block_token( id, ptr ); } catch( const fc::exception& ex ) { exception = true; fc_ilog( logger, "bad block exception connection ${cid}: #${n} ${id}...: ${m}", @@ -3749,17 +3749,18 @@ namespace eosio { } - uint32_t block_num = bsp ? bsp->block_num() : 0; + uint32_t block_num = obt ? obt->block_num() : 0; if( block_num != 0 ) { + assert(obt); fc_dlog( logger, "validated block header, broadcasting immediately, connection ${cid}, blk num = ${num}, id = ${id}", - ("cid", cid)("num", block_num)("id", bsp->id()) ); - my_impl->dispatcher->add_peer_block( bsp->id(), cid ); // no need to send back to sender - my_impl->dispatcher->bcast_block( bsp->block, bsp->id() ); + ("cid", cid)("num", block_num)("id", obt->id()) ); + my_impl->dispatcher->add_peer_block( obt->id(), cid ); // no need to send back to sender + my_impl->dispatcher->bcast_block( obt->block(), obt->id() ); } - app().executor().post(priority::medium, exec_queue::read_write, [ptr{std::move(ptr)}, bsp{std::move(bsp)}, id, c{std::move(c)}]() mutable { - c->process_signed_block( id, std::move(ptr), std::move(bsp) ); + app().executor().post(priority::medium, exec_queue::read_write, [ptr{std::move(ptr)}, obt{std::move(obt)}, id, c{std::move(c)}]() mutable { + c->process_signed_block( id, std::move(ptr), obt ); }); if( block_num != 0 ) { @@ -3770,14 +3771,14 @@ namespace eosio { } // called from application thread - void connection::process_signed_block( const block_id_type& blk_id, signed_block_ptr block, block_state_legacy_ptr bsp ) { + void connection::process_signed_block( const block_id_type& blk_id, signed_block_ptr block, const std::optional& obt ) { controller& cc = my_impl->chain_plug->chain(); uint32_t blk_num = block_header::num_from_id(blk_id); // use c in this method instead of this to highlight that all methods called on c-> must be thread safe connection_ptr c = shared_from_this(); try { - if( blk_num <= cc.last_irreversible_block_num() || cc.fetch_block_by_id(blk_id) ) { + if( blk_num <= cc.last_irreversible_block_num() || cc.block_exists(blk_id) ) { c->strand.post( [sync_master = my_impl->sync_master.get(), dispatcher = my_impl->dispatcher.get(), c, blk_id, blk_num]() { dispatcher->add_peer_block( blk_id, c->connection_id ); @@ -3794,12 +3795,12 @@ namespace eosio { fc::microseconds age( fc::time_point::now() - block->timestamp); fc_dlog( logger, "received signed_block: #${n} block age in secs = ${age}, connection ${cid}, ${v}", - ("n", blk_num)("age", age.to_seconds())("cid", c->connection_id)("v", bsp ? "pre-validated" : "validation pending") ); + ("n", blk_num)("age", age.to_seconds())("cid", c->connection_id)("v", obt ? "pre-validated" : "validation pending") ); go_away_reason reason = no_reason; bool accepted = false; try { - accepted = my_impl->chain_plug->accept_block(block, blk_id, bsp); + accepted = my_impl->chain_plug->accept_block(block, blk_id, obt); my_impl->update_chain_info(); } catch( const unlinkable_block_exception &ex) { fc_ilog(logger, "unlinkable_block_exception connection ${cid}: #${n} ${id}...: ${m}", diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index b8db00f4c0..a65bafa471 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -666,7 +666,7 @@ class producer_plugin_impl : public std::enable_shared_from_this& block_id, const block_state_legacy_ptr& bsp) { + bool on_incoming_block(const signed_block_ptr& block, const std::optional& block_id, const std::optional& obt) { auto& chain = chain_plug->chain(); if (in_producing_mode()) { fc_wlog(_log, "dropped incoming block #${num} id: ${id}", ("num", block->block_num())("id", block_id ? (*block_id).str() : "UNKNOWN")); @@ -687,16 +687,15 @@ class producer_plugin_impl : public std::enable_shared_from_thistimestamp < (now + fc::seconds(7)), block_from_the_future, "received a block from the future, ignoring it: ${id}", ("id", id)); - /* de-dupe here... no point in aborting block if we already know the block */ - auto existing = chain.fetch_block_by_id(id); - if (existing) { + // de-dupe here... no point in aborting block if we already know the block + if (chain.block_exists(id)) { return true; // return true because the block is valid } // start processing of block - std::future bsf; - if (!bsp) { - bsf = chain.create_block_state_future(id, block); + std::future btf; + if (!obt) { + btf = chain.create_block_token_future(id, block); } // abort the pending block @@ -711,11 +710,11 @@ class producer_plugin_impl : public std::enable_shared_from_this().register_provider( - [this](const signed_block_ptr& block, const std::optional& block_id, const block_state_legacy_ptr& bsp) { - return on_incoming_block(block, block_id, bsp); + [this](const signed_block_ptr& block, const std::optional& block_id, const std::optional& obt) { + return on_incoming_block(block, block_id, obt); }); _incoming_transaction_async_provider = diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b288bc9a2d..2ccb07831b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -88,6 +88,8 @@ add_test(NAME nodeos_sanity_test COMMAND tests/nodeos_run_test.py -v --sanity-te set_property(TEST nodeos_sanity_test PROPERTY LABELS nonparallelizable_tests) add_test(NAME nodeos_run_test COMMAND tests/nodeos_run_test.py -v ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST nodeos_run_test PROPERTY LABELS nonparallelizable_tests) +add_test(NAME nodeos_if_test COMMAND tests/nodeos_run_test.py -v --activate-if ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +set_property(TEST nodeos_if_test PROPERTY LABELS nonparallelizable_tests) add_test(NAME block_log_util_test COMMAND tests/block_log_util_test.py -v ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST block_log_util_test PROPERTY LABELS nonparallelizable_tests) add_test(NAME block_log_retain_blocks_test COMMAND tests/block_log_retain_blocks_test.py -v ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) diff --git a/tests/TestHarness/Cluster.py b/tests/TestHarness/Cluster.py index ef846ff27b..8988f7f133 100644 --- a/tests/TestHarness/Cluster.py +++ b/tests/TestHarness/Cluster.py @@ -1187,6 +1187,10 @@ def createSystemAccount(accountName): if not biosNode.waitForTransactionsInBlock(transIds): Utils.Print('ERROR: Failed to validate creation of system accounts') return None + # + # Could activate instant finality here, but have to wait for finality which with all the producers takes a long time + # if activateIF: + # self.activateInstantFinality(launcher) eosioTokenAccount = copy.deepcopy(eosioAccount) eosioTokenAccount.name = 'eosio.token' diff --git a/unittests/api_tests.cpp b/unittests/api_tests.cpp index fd36f47c40..b11200c772 100644 --- a/unittests/api_tests.cpp +++ b/unittests/api_tests.cpp @@ -3885,25 +3885,39 @@ BOOST_AUTO_TEST_CASE(set_finalizer_test) { try { BOOST_TEST(fin_policy->finalizers.size() == finalizers.size()); BOOST_TEST(fin_policy->generation == 1); BOOST_TEST(fin_policy->threshold == finalizers.size() / 3 * 2 + 1); -#if 0 // update after transition is complete: https://github.com/AntelopeIO/leap/issues/1911 - // old dpos still in affect until block is irreversible - BOOST_TEST(block->confirmed == 0); - block_state_legacy_ptr block_state = t.control->fetch_block_state_by_id(block->calculate_id()); - BOOST_REQUIRE(!!block_state); - BOOST_TEST(block_state->dpos_irreversible_blocknum != hs_dpos_irreversible_blocknum); - - block = t.produce_block(); // only one producer so now this block is irreversible, next block will be hotstuff - BOOST_TEST(block->confirmed == 0); - block_state = t.control->fetch_block_state_by_id(block->calculate_id()); - BOOST_REQUIRE(!!block_state); - BOOST_TEST(block_state->dpos_irreversible_blocknum != hs_dpos_irreversible_blocknum); + // currently transition happens immediately after set_finalizer block + + // TODO: update after transition is complete: https://github.com/AntelopeIO/leap/issues/1911 + + // // old dpos still in affect until block is irreversible + // BOOST_TEST(block->confirmed == 0); + // block_state_legacy_ptr block_state = t.control->fetch_block_state_by_id(block->calculate_id()); + // BOOST_REQUIRE(!!block_state); + // BOOST_TEST(block_state->dpos_irreversible_blocknum != hs_dpos_irreversible_blocknum); + + // block = t.produce_block(); // only one producer so now this block is irreversible, next block will be hotstuff + // BOOST_TEST(block->confirmed == 0); + // block_state = t.control->fetch_block_state_by_id(block->calculate_id()); + // BOOST_REQUIRE(!!block_state); + // BOOST_TEST(block_state->dpos_irreversible_blocknum != hs_dpos_irreversible_blocknum); block = t.produce_block(); // hotstuff now active BOOST_TEST(block->confirmed == std::numeric_limits::max()); - block_state = t.control->fetch_block_state_by_id(block->calculate_id()); - BOOST_REQUIRE(!!block_state); - BOOST_TEST(block_state->dpos_irreversible_blocknum == hs_dpos_irreversible_blocknum); -#endif + auto fb = t.control->fetch_block_by_id(block->calculate_id()); + BOOST_REQUIRE(!!fb); + BOOST_TEST(fb == block); + ext = fb->extract_header_extension(instant_finality_extension::extension_id()); + BOOST_REQUIRE(ext); + + // and another on top of a instant-finality block + block = t.produce_block(); + BOOST_TEST(block->confirmed == std::numeric_limits::max()); + fb = t.control->fetch_block_by_id(block->calculate_id()); + BOOST_REQUIRE(!!fb); + BOOST_TEST(fb == block); + ext = fb->extract_header_extension(instant_finality_extension::extension_id()); + BOOST_REQUIRE(ext); + } FC_LOG_AND_RETHROW() } BOOST_AUTO_TEST_SUITE_END() diff --git a/unittests/block_tests.cpp b/unittests/block_tests.cpp index 8e11f270e2..5ffb9d8408 100644 --- a/unittests/block_tests.cpp +++ b/unittests/block_tests.cpp @@ -44,10 +44,10 @@ BOOST_AUTO_TEST_CASE(block_with_invalid_tx_test) // Push block with invalid transaction to other chain tester validator; - auto bsf = validator.control->create_block_state_future( copy_b->calculate_id(), copy_b ); + auto btf = validator.control->create_block_token_future( copy_b->calculate_id(), copy_b ); validator.control->abort_block(); controller::block_report br; - BOOST_REQUIRE_EXCEPTION(validator.control->push_block( br, bsf.get(), {}, trx_meta_cache_lookup{} ), fc::exception , + BOOST_REQUIRE_EXCEPTION(validator.control->push_block( br, btf.get(), {}, trx_meta_cache_lookup{} ), fc::exception , [] (const fc::exception &e)->bool { return e.code() == account_name_exists_exception::code_value ; }) ; @@ -83,10 +83,10 @@ BOOST_AUTO_TEST_CASE(block_with_invalid_tx_mroot_test) // Push block with invalid transaction to other chain tester validator; - auto bsf = validator.control->create_block_state_future( copy_b->calculate_id(), copy_b ); + auto btf = validator.control->create_block_token_future( copy_b->calculate_id(), copy_b ); validator.control->abort_block(); controller::block_report br; - BOOST_REQUIRE_EXCEPTION(validator.control->push_block( br, bsf.get(), {}, trx_meta_cache_lookup{} ), fc::exception, + BOOST_REQUIRE_EXCEPTION(validator.control->push_block( br, btf.get(), {}, trx_meta_cache_lookup{} ), fc::exception, [] (const fc::exception &e)->bool { return e.code() == block_validate_exception::code_value && e.to_detail_string().find("invalid block transaction merkle root") != std::string::npos; diff --git a/unittests/chain_tests.cpp b/unittests/chain_tests.cpp index 45ad505115..bc2a973e12 100644 --- a/unittests/chain_tests.cpp +++ b/unittests/chain_tests.cpp @@ -161,6 +161,7 @@ BOOST_AUTO_TEST_CASE( signal_validated_blocks ) try { auto block_num = block->block_num(); BOOST_CHECK(block); BOOST_CHECK(chain.control->fetch_block_by_id(id) == block); + BOOST_CHECK(chain.control->block_exists(id)); BOOST_CHECK(chain.control->fetch_block_by_number(block_num) == block); BOOST_REQUIRE(chain.control->fetch_block_header_by_number(block_num)); BOOST_CHECK(chain.control->fetch_block_header_by_number(block_num)->calculate_id() == id); @@ -176,6 +177,7 @@ BOOST_AUTO_TEST_CASE( signal_validated_blocks ) try { auto block_num = block->block_num(); BOOST_CHECK(block); BOOST_CHECK(validator.control->fetch_block_by_id(id) == block); + BOOST_CHECK(validator.control->block_exists(id)); BOOST_CHECK(validator.control->fetch_block_by_number(block_num) == block); BOOST_REQUIRE(validator.control->fetch_block_header_by_number(block_num)); BOOST_CHECK(validator.control->fetch_block_header_by_number(block_num)->calculate_id() == id); diff --git a/unittests/forked_tests.cpp b/unittests/forked_tests.cpp index 2392663921..66dc2fc17d 100644 --- a/unittests/forked_tests.cpp +++ b/unittests/forked_tests.cpp @@ -266,10 +266,10 @@ BOOST_AUTO_TEST_CASE( forking ) try { signed_block bad_block = std::move(*b); bad_block.action_mroot = bad_block.previous; auto bad_id = bad_block.calculate_id(); - auto bad_block_bsf = c.control->create_block_state_future( bad_id, std::make_shared(std::move(bad_block)) ); + auto bad_block_btf = c.control->create_block_token_future( bad_id, std::make_shared(std::move(bad_block)) ); c.control->abort_block(); controller::block_report br; - BOOST_REQUIRE_EXCEPTION(c.control->push_block( br, bad_block_bsf.get(), {}, trx_meta_cache_lookup{} ), fc::exception, + BOOST_REQUIRE_EXCEPTION(c.control->push_block( br, bad_block_btf.get(), {}, trx_meta_cache_lookup{} ), fc::exception, [] (const fc::exception &ex)->bool { return ex.to_detail_string().find("block signed by unexpected key") != std::string::npos; }); @@ -496,6 +496,7 @@ BOOST_AUTO_TEST_CASE( irreversible_mode ) try { { auto b = irreversible.control->fetch_block_by_id( fork_first_block_id ); BOOST_REQUIRE( b && b->calculate_id() == fork_first_block_id ); + BOOST_TEST( irreversible.control->block_exists(fork_first_block_id) ); } main.produce_block(); @@ -509,6 +510,7 @@ BOOST_AUTO_TEST_CASE( irreversible_mode ) try { { auto b = irreversible.control->fetch_block_by_id( fork_first_block_id ); BOOST_REQUIRE( !b ); + BOOST_TEST( irreversible.control->block_exists(fork_first_block_id) ); } } FC_LOG_AND_RETHROW() diff --git a/unittests/protocol_feature_tests.cpp b/unittests/protocol_feature_tests.cpp index 45c76ab339..38fbbe40a7 100644 --- a/unittests/protocol_feature_tests.cpp +++ b/unittests/protocol_feature_tests.cpp @@ -2252,12 +2252,12 @@ BOOST_AUTO_TEST_CASE( block_validation_after_stage_1_test ) { try { tester2.produce_block(); // Push the block with delayed transaction to the second chain - auto bsf = tester2.control->create_block_state_future( copy_b->calculate_id(), copy_b ); + auto btf = tester2.control->create_block_token_future( copy_b->calculate_id(), copy_b ); tester2.control->abort_block(); controller::block_report br; // The block is invalidated - BOOST_REQUIRE_EXCEPTION(tester2.control->push_block( br, bsf.get(), {}, trx_meta_cache_lookup{} ), + BOOST_REQUIRE_EXCEPTION(tester2.control->push_block( br, btf.get(), {}, trx_meta_cache_lookup{} ), fc::exception, fc_exception_message_starts_with("transaction cannot be delayed") ); diff --git a/unittests/unapplied_transaction_queue_tests.cpp b/unittests/unapplied_transaction_queue_tests.cpp index d2f683dc32..53dd773647 100644 --- a/unittests/unapplied_transaction_queue_tests.cpp +++ b/unittests/unapplied_transaction_queue_tests.cpp @@ -78,6 +78,20 @@ auto create_test_block_state( deque trx_metas ) { return bsp; } +using branch_type_legacy = fork_database::branch_type; + +template +void add_forked( unapplied_transaction_queue& queue, const BRANCH_TYPE& forked_branch ) { + // forked_branch is in reverse order + for( auto ritr = forked_branch.rbegin(), rend = forked_branch.rend(); ritr != rend; ++ritr ) { + const auto& bsptr = *ritr; + for( auto itr = bsptr->trxs_metas().begin(), end = bsptr->trxs_metas().end(); itr != end; ++itr ) { + const auto& trx = *itr; + queue.add_forked(trx); + } + } +} + // given a current itr make sure expected number of items are iterated over void verify_order( unapplied_transaction_queue& q, unapplied_transaction_queue::iterator itr, size_t expected ) { size_t size = 0; @@ -136,7 +150,7 @@ BOOST_AUTO_TEST_CASE( unapplied_transaction_queue_test ) try { auto bs1 = create_test_block_state( { trx1, trx2 } ); auto bs2 = create_test_block_state( { trx3, trx4, trx5 } ); auto bs3 = create_test_block_state( { trx6 } ); - q.add_forked( branch_type_legacy{ bs3, bs2, bs1, bs1 } ); // bs1 duplicate ignored + add_forked( q, branch_type_legacy{ bs3, bs2, bs1, bs1 } ); // bs1 duplicate ignored BOOST_CHECK( q.size() == 6u ); BOOST_REQUIRE( next( q ) == trx1 ); BOOST_CHECK( q.size() == 5u ); @@ -155,9 +169,9 @@ BOOST_AUTO_TEST_CASE( unapplied_transaction_queue_test ) try { // fifo forked auto bs4 = create_test_block_state( { trx7 } ); - q.add_forked( branch_type_legacy{ bs1 } ); - q.add_forked( branch_type_legacy{ bs3, bs2 } ); - q.add_forked( branch_type_legacy{ bs4 } ); + add_forked( q, branch_type_legacy{ bs1 } ); + add_forked( q, branch_type_legacy{ bs3, bs2 } ); + add_forked( q, branch_type_legacy{ bs4 } ); BOOST_CHECK( q.size() == 7u ); BOOST_REQUIRE( next( q ) == trx1 ); BOOST_CHECK( q.size() == 6u ); @@ -189,10 +203,10 @@ BOOST_AUTO_TEST_CASE( unapplied_transaction_queue_test ) try { // fifo forked, multi forks auto bs5 = create_test_block_state( { trx11, trx12, trx13 } ); auto bs6 = create_test_block_state( { trx11, trx15 } ); - q.add_forked( branch_type_legacy{ bs3, bs2, bs1 } ); - q.add_forked( branch_type_legacy{ bs4 } ); - q.add_forked( branch_type_legacy{ bs3, bs2 } ); // dups ignored - q.add_forked( branch_type_legacy{ bs6, bs5 } ); + add_forked( q, branch_type_legacy{ bs3, bs2, bs1 } ); + add_forked( q, branch_type_legacy{ bs4 } ); + add_forked( q, branch_type_legacy{ bs3, bs2 } ); // dups ignored + add_forked( q, branch_type_legacy{ bs6, bs5 } ); BOOST_CHECK_EQUAL( q.size(), 11u ); BOOST_REQUIRE( next( q ) == trx1 ); BOOST_CHECK( q.size() == 10u ); @@ -220,10 +234,10 @@ BOOST_AUTO_TEST_CASE( unapplied_transaction_queue_test ) try { BOOST_CHECK( q.empty() ); // altogether, order fifo: forked, aborted - q.add_forked( branch_type_legacy{ bs3, bs2, bs1 } ); + add_forked( q, branch_type_legacy{ bs3, bs2, bs1 } ); q.add_aborted( { trx9, trx14 } ); q.add_aborted( { trx18, trx19 } ); - q.add_forked( branch_type_legacy{ bs6, bs5, bs4 } ); + add_forked( q, branch_type_legacy{ bs6, bs5, bs4 } ); // verify order verify_order( q, q.begin(), 15 ); // verify type order @@ -289,7 +303,7 @@ BOOST_AUTO_TEST_CASE( unapplied_transaction_queue_test ) try { BOOST_REQUIRE( next( q ) == trx22 ); BOOST_CHECK( q.empty() ); - q.add_forked( branch_type_legacy{ bs3, bs2, bs1 } ); + add_forked( q, branch_type_legacy{ bs3, bs2, bs1 } ); q.add_aborted( { trx9, trx11 } ); q.clear(); BOOST_CHECK( q.empty() ); From 9b39982a2286fdaff178b5644423495b97a85d53 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 15 Jan 2024 10:40:51 -0600 Subject: [PATCH 27/36] GH-2045 cleanup implementation --- libraries/chain/controller.cpp | 43 +++++++++++++++------------------- unittests/forked_tests.cpp | 2 +- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 48e8d4b689..90459ac4ab 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -291,6 +291,22 @@ struct block_data_t { }, v); } + signed_block_ptr fork_db_fetch_block_by_num(uint32_t block_num) const { + return std::visit([&](const auto& bd) -> signed_block_ptr { + auto bsp = bd.fork_db.search_on_branch(bd.fork_db.head()->id(), block_num); + if (bsp) return bsp->block; + return {}; + }, v); + } + + std::optional fork_db_fetch_block_id(uint32_t block_num) const { + return std::visit([&](const auto& bd) -> std::optional { + auto bsp = bd.fork_db.search_on_branch(bd.fork_db.head()->id(), block_num); + if (bsp) return bsp->id(); + return {}; + }, v); + } + template R apply(F& f) { if constexpr (std::is_same_v) @@ -4016,26 +4032,15 @@ bool controller::block_exists(const block_id_type&id) const { } std::optional controller::fetch_block_header_by_id( const block_id_type& id )const { -#if 0 - // [greg todo] is the below code equivalent?? - auto state = my->fork_db.get_block(id); - if( state && state->block ) return state->header; -#else auto sb_ptr = my->block_data.fork_db_fetch_block_by_id(id); if( sb_ptr ) return *static_cast(sb_ptr.get()); -#endif auto result = my->blog.read_block_header_by_num( block_header::num_from_id(id) ); if( result && result->calculate_id() == id ) return result; return {}; } signed_block_ptr controller::fetch_block_by_number( uint32_t block_num )const { try { - auto fetch_block = [&](auto& fork_db, auto& head) -> signed_block_ptr { - auto bsp = fork_db.search_on_branch( fork_db.head()->id(), block_num); - if (bsp) return bsp->block; - return {}; - }; - auto b = my->block_data.apply(fetch_block); + auto b = my->block_data.fork_db_fetch_block_by_num( block_num ); if (b) return b; @@ -4043,12 +4048,7 @@ signed_block_ptr controller::fetch_block_by_number( uint32_t block_num )const { } FC_CAPTURE_AND_RETHROW( (block_num) ) } std::optional controller::fetch_block_header_by_number( uint32_t block_num )const { try { - auto fetch_block = [&](auto& fork_db, auto& head) -> signed_block_ptr { - auto bsp = fork_db.search_on_branch( fork_db.head()->id(), block_num); - if (bsp) return bsp->block; - return {}; - }; - auto b = my->block_data.apply(fetch_block); + auto b = my->block_data.fork_db_fetch_block_by_num(block_num); if (b) return *b; @@ -4062,12 +4062,7 @@ block_id_type controller::get_block_id_for_num( uint32_t block_num )const { try bool find_in_blog = (blog_head && block_num <= blog_head->block_num()); if( !find_in_blog ) { - auto fetch_block_id = [&](auto& fork_db, auto& head) -> std::optional { - auto bsp = fork_db.search_on_branch( fork_db.head()->id(), block_num); - if (bsp) return bsp->id(); - return {}; - }; - auto id = my->block_data.apply>(fetch_block_id); + std::optional id = my->block_data.fork_db_fetch_block_id(block_num); if (id) return *id; } diff --git a/unittests/forked_tests.cpp b/unittests/forked_tests.cpp index 66dc2fc17d..3800b090c6 100644 --- a/unittests/forked_tests.cpp +++ b/unittests/forked_tests.cpp @@ -510,7 +510,7 @@ BOOST_AUTO_TEST_CASE( irreversible_mode ) try { { auto b = irreversible.control->fetch_block_by_id( fork_first_block_id ); BOOST_REQUIRE( !b ); - BOOST_TEST( irreversible.control->block_exists(fork_first_block_id) ); + BOOST_TEST( !irreversible.control->block_exists(fork_first_block_id) ); } } FC_LOG_AND_RETHROW() From 03f232286a701154c29008c8b8eed5b16d71c951 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 15 Jan 2024 11:13:01 -0600 Subject: [PATCH 28/36] GH-2045 Fix proposer_policy switching time. Enable instant-finality producer schedule test. --- libraries/chain/block_header_state.cpp | 2 +- unittests/producer_schedule_hs_tests.cpp | 39 ++++++++++-------------- 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/libraries/chain/block_header_state.cpp b/libraries/chain/block_header_state.cpp index cce57cfcc1..8f2255e134 100644 --- a/libraries/chain/block_header_state.cpp +++ b/libraries/chain/block_header_state.cpp @@ -109,7 +109,7 @@ block_header_state block_header_state::next(block_header_state_input& input) con if (input.new_proposer_policy) { // called when assembling the block - result.proposer_policies[result.header.timestamp] = input.new_proposer_policy; + result.proposer_policies[input.new_proposer_policy->active_time] = input.new_proposer_policy; } // finalizer policy diff --git a/unittests/producer_schedule_hs_tests.cpp b/unittests/producer_schedule_hs_tests.cpp index b1188e9084..681627a42b 100644 --- a/unittests/producer_schedule_hs_tests.cpp +++ b/unittests/producer_schedule_hs_tests.cpp @@ -3,13 +3,11 @@ #include -#include "fork_test_utilities.hpp" - using namespace eosio::testing; using namespace eosio::chain; using mvo = fc::mutable_variant_object; -BOOST_AUTO_TEST_SUITE(producer_schedule_hs_tests) +BOOST_AUTO_TEST_SUITE(producer_schedule_if_tests) namespace { @@ -20,19 +18,16 @@ inline account_name get_expected_producer(const vector& sche }; } // anonymous namespace -#if 0 - -// [greg todo] Enable test when https://github.com/AntelopeIO/leap/issues/1980 is completed -BOOST_FIXTURE_TEST_CASE( verify_producer_schedule_after_hotstuff_activation, validating_tester ) try { +BOOST_FIXTURE_TEST_CASE( verify_producer_schedule_after_instant_finality_activation, validating_tester ) try { // Utility function to ensure that producer schedule work as expected const auto& confirm_schedule_correctness = [&](const vector& new_prod_schd, uint32_t expected_schd_ver, uint32_t expected_block_num = 0) { const uint32_t check_duration = 100; // number of blocks bool scheduled_changed_to_new = false; for (uint32_t i = 0; i < check_duration; ++i) { - const auto current_schedule = control->head_block_state()->active_schedule.producers; - if (new_prod_schd == current_schedule) { + const auto current_schedule = control->active_producers(); + if (new_prod_schd == current_schedule.producers) { scheduled_changed_to_new = true; if (expected_block_num != 0) BOOST_TEST(control->head_block_num() == expected_block_num); @@ -42,7 +37,7 @@ BOOST_FIXTURE_TEST_CASE( verify_producer_schedule_after_hotstuff_activation, val // Check if the producer is the same as what we expect const auto block_time = control->head_block_time(); - const auto& expected_producer = get_expected_producer(current_schedule, block_time); + const auto& expected_producer = get_expected_producer(current_schedule.producers, block_time); BOOST_TEST(control->head_block_producer() == expected_producer); if (scheduled_changed_to_new) @@ -69,17 +64,17 @@ BOOST_FIXTURE_TEST_CASE( verify_producer_schedule_after_hotstuff_activation, val }; create_accounts(producers); - // activate hotstuff + // activate instant_finality set_finalizers(producers); auto block = produce_block(); // this block contains the header extension of the finalizer set - BOOST_TEST(lib == 3); + BOOST_TEST(lib == 4); // TODO: currently lib advances immediately on set_finalizers // ---- Test first set of producers ---- // Send set prods action and confirm schedule correctness set_producers(producers); const auto first_prod_schd = get_producer_authorities(producers); - // TODO: update expected when lib for hotstuff is working, will change from 22 at that time - confirm_schedule_correctness(first_prod_schd, 1, 22); + // TODO: update expected when lib for instant_finality is working, will change from 26 at that time, 4+12+12 + confirm_schedule_correctness(first_prod_schd, 1, 26); // ---- Test second set of producers ---- vector second_set_of_producer = { @@ -88,8 +83,8 @@ BOOST_FIXTURE_TEST_CASE( verify_producer_schedule_after_hotstuff_activation, val // Send set prods action and confirm schedule correctness set_producers(second_set_of_producer); const auto second_prod_schd = get_producer_authorities(second_set_of_producer); - // TODO: update expected when lib for hotstuff is working, will change from 44 at that time - confirm_schedule_correctness(second_prod_schd, 2, 44); + // TODO: update expected when lib for instant_finality is working, will change from 50 at that time, 26+12+12 + confirm_schedule_correctness(second_prod_schd, 2, 50); // ---- Test deliberately miss some blocks ---- const int64_t num_of_missed_blocks = 5000; @@ -111,9 +106,7 @@ BOOST_FIXTURE_TEST_CASE( verify_producer_schedule_after_hotstuff_activation, val } FC_LOG_AND_RETHROW() -#endif - -/** TODO: Enable tests after hotstuff LIB is working +/** TODO: Enable tests after instant_finality LIB is working BOOST_FIXTURE_TEST_CASE( producer_schedule_promotion_test, validating_tester ) try { create_accounts( {"alice"_n,"bob"_n,"carol"_n} ); @@ -121,7 +114,7 @@ BOOST_FIXTURE_TEST_CASE( producer_schedule_promotion_test, validating_tester ) t produce_block(); } - // activate hotstuff + // activate instant_finality set_finalizers({"alice"_n,"bob"_n,"carol"_n}); auto block = produce_block(); // this block contains the header extension of the finalizer set @@ -190,7 +183,7 @@ BOOST_AUTO_TEST_CASE( producer_watermark_test ) try { c.create_accounts( {"alice"_n,"bob"_n,"carol"_n} ); c.produce_block(); - // activate hotstuff + // activate instant_finality c.set_finalizers({"alice"_n,"bob"_n,"carol"_n}); auto block = c.produce_block(); // this block contains the header extension of the finalizer set @@ -309,7 +302,7 @@ BOOST_FIXTURE_TEST_CASE( producer_one_of_n_test, validating_tester ) try { create_accounts( {"alice"_n,"bob"_n} ); produce_block(); - // activate hotstuff + // activate instant_finality set_finalizers({"alice"_n,"bob"_n}); auto block = produce_block(); // this block contains the header extension of the finalizer set @@ -331,7 +324,7 @@ BOOST_FIXTURE_TEST_CASE( producer_m_of_n_test, validating_tester ) try { create_accounts( {"alice"_n,"bob"_n} ); produce_block(); - // activate hotstuff + // activate instant_finality set_finalizers({"alice"_n,"bob"_n}); auto block = produce_block(); // this block contains the header extension of the finalizer set From 7962634ef49497350cb8fbf5c27112bdd481ea94 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 15 Jan 2024 12:41:59 -0600 Subject: [PATCH 29/36] GH-2045 Serialize all block_header_state and block_state data --- .../chain/include/eosio/chain/block_header_state.hpp | 8 +++++--- libraries/chain/include/eosio/chain/block_state.hpp | 3 +-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/libraries/chain/include/eosio/chain/block_header_state.hpp b/libraries/chain/include/eosio/chain/block_header_state.hpp index 2780d1b709..f18e3137a7 100644 --- a/libraries/chain/include/eosio/chain/block_header_state.hpp +++ b/libraries/chain/include/eosio/chain/block_header_state.hpp @@ -98,6 +98,8 @@ using block_header_state_ptr = std::shared_ptr; } -// [greg todo] which members need to be serialized to disk when saving fork_db -// obviously many are missing below. -FC_REFLECT( eosio::chain::block_header_state, (id)) +FC_REFLECT( eosio::chain::block_header_state_core, + (last_final_block_num)(final_on_strong_qc_block_num)(last_qc_block_num)(finalizer_policy_generation)) +FC_REFLECT( eosio::chain::block_header_state, + (id)(header)(activated_protocol_features)(core)(proposal_mtree)(finality_mtree) + (active_finalizer_policy)(active_proposer_policy)(proposer_policies)(finalizer_policies)(header_exts)) diff --git a/libraries/chain/include/eosio/chain/block_state.hpp b/libraries/chain/include/eosio/chain/block_state.hpp index 39cb9a49ab..badb713489 100644 --- a/libraries/chain/include/eosio/chain/block_state.hpp +++ b/libraries/chain/include/eosio/chain/block_state.hpp @@ -57,5 +57,4 @@ using block_state_ptr = std::shared_ptr; } // namespace eosio::chain -// [greg todo] which members need to be serialized to disk when saving fork_db -FC_REFLECT_DERIVED( eosio::chain::block_state, (eosio::chain::block_header_state), (block)(validated) ) +FC_REFLECT_DERIVED( eosio::chain::block_state, (eosio::chain::block_header_state), (block)(validated)(strong_digest)(weak_digest)(pending_qc)(valid_qc) ) From 7c59cf36e2c2722edaf7fbffaef3ee894a5b417c Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 15 Jan 2024 12:50:02 -0600 Subject: [PATCH 30/36] GH-2045 Fix spelling --- libraries/chain/block_state.cpp | 8 ++++---- libraries/chain/controller.cpp | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/libraries/chain/block_state.cpp b/libraries/chain/block_state.cpp index 8630a4bcca..b750e68292 100644 --- a/libraries/chain/block_state.cpp +++ b/libraries/chain/block_state.cpp @@ -21,20 +21,20 @@ block_state::block_state(const block_header_state& bhs, dequetransactions = std::move(trx_receipts); } -// Used for transition from dbpos to instant-finality +// Used for transition from dpos to instant-finality block_state::block_state(const block_state_legacy& bsp) { block_header_state::id = bsp.id(); header = bsp.header; activated_protocol_features = bsp.activated_protocol_features; std::optional ext = bsp.block->extract_header_extension(instant_finality_extension::extension_id()); - assert(ext); // required by current transistion mechanism + assert(ext); // required by current transition mechanism const auto& if_extension = std::get(*ext); - assert(if_extension.new_finalizer_policy); // required by current transistion mechanism + assert(if_extension.new_finalizer_policy); // required by current transition mechanism active_finalizer_policy = std::make_shared(*if_extension.new_finalizer_policy); active_proposer_policy = std::make_shared(); active_proposer_policy->active_time = bsp.timestamp(); active_proposer_policy->proposer_schedule = bsp.active_schedule; - header_exts = bsp.header_exts; // not needed, but copy over just in case + header_exts = bsp.header_exts; block = bsp.block; validated = bsp.is_valid(); pub_keys_recovered = bsp._pub_keys_recovered; diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 90459ac4ab..10d987d5ce 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -178,7 +178,7 @@ struct block_data_t { block_timestamp_type head_block_time() const { return std::visit([](const auto& bd) { return bd.head->timestamp(); }, v); } account_name head_block_producer() const { return std::visit([](const auto& bd) { return bd.head->producer(); }, v); } - void transistion_fork_db_to_if(const auto& vbsp) { + void transition_fork_db_to_if(const auto& vbsp) { std::visit([](auto& bd) { bd.fork_db.close(); }, v); auto bsp = std::make_shared(*std::get(vbsp)); v.emplace(std::visit([](const auto& bd) { return bd.fork_db.get_data_dir(); }, v)); @@ -2720,7 +2720,7 @@ struct controller_impl { log_irreversible(); } - // TODO: temp transistion to instant-finality, happens immediately after block with new_finalizer_policy + // TODO: temp transition to instant-finality, happens immediately after block with new_finalizer_policy auto transition = [&](auto& fork_db, auto& head) -> bool { const auto& bsp = std::get>(cb.bsp); std::optional ext = bsp->block->extract_header_extension(instant_finality_extension::extension_id()); @@ -2737,7 +2737,7 @@ struct controller_impl { return false; }; if (block_data.apply_dpos(transition)) { - block_data.transistion_fork_db_to_if(cb.bsp); + block_data.transition_fork_db_to_if(cb.bsp); } } catch (...) { From d0a1e075d00258d92e3093df45dc1d7f0c077c50 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 15 Jan 2024 13:03:59 -0600 Subject: [PATCH 31/36] GH-2045 Don't export qcs --- libraries/chain/include/eosio/chain/block_state.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/chain/include/eosio/chain/block_state.hpp b/libraries/chain/include/eosio/chain/block_state.hpp index badb713489..ff48e4fb5e 100644 --- a/libraries/chain/include/eosio/chain/block_state.hpp +++ b/libraries/chain/include/eosio/chain/block_state.hpp @@ -57,4 +57,5 @@ using block_state_ptr = std::shared_ptr; } // namespace eosio::chain -FC_REFLECT_DERIVED( eosio::chain::block_state, (eosio::chain::block_header_state), (block)(validated)(strong_digest)(weak_digest)(pending_qc)(valid_qc) ) +// not exporting pending_qc or valid_qc +FC_REFLECT_DERIVED( eosio::chain::block_state, (eosio::chain::block_header_state), (block)(validated)(strong_digest)(weak_digest) ) From b5dfdd402420f9744ececa7f36553e6cab9a6c8e Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 15 Jan 2024 15:32:47 -0600 Subject: [PATCH 32/36] GH-2045 Fix for next_producers when no pending --- libraries/chain/controller.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 10d987d5ce..1c4d585e62 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -207,6 +207,17 @@ struct block_data_t { [](const block_data_new_t&) -> const producer_authority_schedule* { return nullptr; } }, v); } + + const producer_authority_schedule* next_producers() { + return std::visit(overloaded{ + [](const block_data_legacy_t& bd) -> const producer_authority_schedule* { + return bd.head->pending_schedule_auth(); + }, + [](const block_data_new_t& bd) -> const producer_authority_schedule* { + return bd.head->proposer_policies.empty() ? nullptr : &bd.head->proposer_policies.begin()->second->proposer_schedule; + } + }, v); + } const block_id_type& head_block_id() const { return std::visit([](const auto& bd) -> const block_id_type& { return bd.head->id(); }, v); @@ -4210,6 +4221,9 @@ std::optional controller::proposed_producers_legacy } const producer_authority_schedule* controller::next_producers()const { + if( !(my->pending) ) + return my->block_data.next_producers(); + return my->pending->next_producers(); } From 35705b3af2ff28fd21842cb5f356df3101aa73cf Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 15 Jan 2024 15:43:20 -0600 Subject: [PATCH 33/36] GH-2045 Removed a warning and renamed hs_irreversible_block_num --- libraries/chain/controller.cpp | 16 ++++++---------- libraries/chain/hotstuff/chain_pacemaker.cpp | 2 +- .../chain/include/eosio/chain/controller.hpp | 2 +- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 1c4d585e62..63dd913629 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -1013,7 +1013,7 @@ struct controller_impl { std::optional pending; block_data_t block_data; // includes `head` aand `fork_db` std::optional pacemaker; - std::atomic hs_irreversible_block_num{0}; + std::atomic if_irreversible_block_num{0}; resource_limits_manager resource_limits; subjective_billing subjective_bill; authorization_manager authorization; @@ -1196,8 +1196,8 @@ struct controller_impl { ("lib_num", lib_num)("bn", fork_db_root_block_num()) ); } - const uint32_t hs_lib = hs_irreversible_block_num; - const uint32_t new_lib = hs_lib > 0 ? hs_lib : fork_db_head_irreversible_blocknum(); + const uint32_t if_lib = if_irreversible_block_num; + const uint32_t new_lib = if_lib > 0 ? if_lib : fork_db_head_irreversible_blocknum(); if( new_lib <= lib_num ) return; @@ -2482,9 +2482,6 @@ struct controller_impl { { EOS_ASSERT( !pending, block_validate_exception, "pending block already exists" ); - // can change during start_block, so use same value throughout - uint32_t hs_lib = hs_irreversible_block_num.load(); - emit( self.block_start, head_block_num() + 1 ); // at block level, no transaction specific logging is possible @@ -2739,7 +2736,7 @@ struct controller_impl { const auto& if_extension = std::get(*ext); if (if_extension.new_finalizer_policy) { ilog("Transition to instant finality happening after block ${b}", ("b", bsp->block_num())); - hs_irreversible_block_num = bsp->block_num(); + if_irreversible_block_num = bsp->block_num(); log_irreversible(); return true; @@ -4000,10 +3997,10 @@ std::optional controller::pending_producer_block_id()const { return my->pending->_producer_block_id; } -void controller::set_hs_irreversible_block_num(uint32_t block_num) { +void controller::set_if_irreversible_block_num(uint32_t block_num) { // needs to be set by qc_chain at startup and as irreversible changes assert(block_num > 0); - my->hs_irreversible_block_num = block_num; + my->if_irreversible_block_num = block_num; } uint32_t controller::last_irreversible_block_num() const { @@ -4197,7 +4194,6 @@ const producer_authority_schedule& controller::active_producers()const { if( !(my->pending) ) return my->block_data.head_active_schedule_auth(); -#warning todo: support active/pending_producers correctly when in IF mode (see assembled_block and completed_block stages) return my->pending->active_producers(); } diff --git a/libraries/chain/hotstuff/chain_pacemaker.cpp b/libraries/chain/hotstuff/chain_pacemaker.cpp index 2f3a84c41c..ebe34117be 100644 --- a/libraries/chain/hotstuff/chain_pacemaker.cpp +++ b/libraries/chain/hotstuff/chain_pacemaker.cpp @@ -176,7 +176,7 @@ namespace eosio::chain { ilog("Switching to instant finality at block ${b}", ("b", block->block_num())); // switching from dpos to hotstuff, all nodes will switch at same block height // block header extension is set in finish_block to value set by host function set_finalizers - _chain->set_hs_irreversible_block_num(block->block_num()); // can be any value <= dpos lib + _chain->set_if_irreversible_block_num(block->block_num()); // can be any value <= dpos lib } auto if_extension = std::get(*ext); #warning Revisit after finalizer policy change design is complete as this is not necessarily when we will change active finalizer policy. diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index cef7c8df25..4b726a3a2d 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -270,7 +270,7 @@ namespace eosio::chain { // Called by qc_chain to indicate the current irreversible block num // After hotstuff is activated, this should be called on startup by qc_chain - void set_hs_irreversible_block_num(uint32_t block_num); + void set_if_irreversible_block_num(uint32_t block_num); uint32_t last_irreversible_block_num() const; block_id_type last_irreversible_block_id() const; From 6b6249635c5190bb916ca7c04ec12e862d4b8e75 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 15 Jan 2024 16:05:33 -0600 Subject: [PATCH 34/36] GH-2045 Misc cleanup --- libraries/chain/controller.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 63dd913629..086bdc8066 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -187,6 +187,7 @@ struct block_data_t { [&](block_data_new_t& bd) { bd.head = bsp; bd.fork_db.reset(*bd.head); + bd.fork_db.open({}); // no-op } }, v); } @@ -310,7 +311,7 @@ struct block_data_t { }, v); } - std::optional fork_db_fetch_block_id(uint32_t block_num) const { + std::optional fork_db_fetch_block_id_by_num(uint32_t block_num) const { return std::visit([&](const auto& bd) -> std::optional { auto bsp = bd.fork_db.search_on_branch(bd.fork_db.head()->id(), block_num); if (bsp) return bsp->id(); @@ -3038,11 +3039,11 @@ struct controller_impl { EOS_ASSERT( prev, unlinkable_block_exception, "unlinkable block ${id}", ("id", id)("previous", b->previous) ); - return control->create_block_state_i( id, b, *prev ); // [greg todo] make it work with apply() (if `create_block_state_future` needed) + return control->create_block_state_i( id, b, *prev ); } ); }; - return block_data.apply>(f); // [greg todo] make it work with apply() + return block_data.apply>(f); } // thread safe, expected to be called from thread other than the main thread @@ -4032,10 +4033,10 @@ signed_block_ptr controller::fetch_block_by_id( const block_id_type& id )const { } bool controller::block_exists(const block_id_type&id) const { - auto sb_ptr = my->block_data.fork_db_fetch_block_by_id(id); + signed_block_ptr sb_ptr = my->block_data.fork_db_fetch_block_by_id(id); if( sb_ptr ) return true; - auto bptr = my->blog.read_block_header_by_num( block_header::num_from_id(id) ); - if( bptr && bptr->calculate_id() == id ) return true; + std::optional sbh = my->blog.read_block_header_by_num( block_header::num_from_id(id) ); + if( sbh && sbh->calculate_id() == id ) return true; return false; } @@ -4070,7 +4071,7 @@ block_id_type controller::get_block_id_for_num( uint32_t block_num )const { try bool find_in_blog = (blog_head && block_num <= blog_head->block_num()); if( !find_in_blog ) { - std::optional id = my->block_data.fork_db_fetch_block_id(block_num); + std::optional id = my->block_data.fork_db_fetch_block_id_by_num(block_num); if (id) return *id; } From 852f69ebae84977b92fc06db754bc86949b10046 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 15 Jan 2024 16:30:54 -0600 Subject: [PATCH 35/36] GH-2045 No need to close or open fork_db --- libraries/chain/controller.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 086bdc8066..00992e1172 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -179,7 +179,7 @@ struct block_data_t { account_name head_block_producer() const { return std::visit([](const auto& bd) { return bd.head->producer(); }, v); } void transition_fork_db_to_if(const auto& vbsp) { - std::visit([](auto& bd) { bd.fork_db.close(); }, v); + // no need to close fork_db because we don't want to write anything out, file is removed on open auto bsp = std::make_shared(*std::get(vbsp)); v.emplace(std::visit([](const auto& bd) { return bd.fork_db.get_data_dir(); }, v)); std::visit(overloaded{ @@ -187,7 +187,6 @@ struct block_data_t { [&](block_data_new_t& bd) { bd.head = bsp; bd.fork_db.reset(*bd.head); - bd.fork_db.open({}); // no-op } }, v); } From 161d585903e5d24699fdc9b8ca003d30cece6ed8 Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Mon, 15 Jan 2024 20:59:27 -0500 Subject: [PATCH 36/36] Remove unneeded include. --- libraries/chain/controller.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index eacb250315..63e1aa500c 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -43,7 +43,6 @@ #include #include #include -#include namespace eosio::chain {