Add collator options (#1052)

* Set collator options from validator console

* Fix compilation error in manager-disk

* Defer all messages if out msg queue is too big

* Fix checking queue size in collator

---------

Co-authored-by: SpyCheese <[email protected]>

crypto/block/block.tlb

@@ -801,7 +801,7 @@ size_limits_config#01 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells
  max_ext_msg_size:uint32 max_ext_msg_depth:uint16 = SizeLimitsConfig;
 size_limits_config_v2#02 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells:uint32 max_vm_data_depth:uint16
  max_ext_msg_size:uint32 max_ext_msg_depth:uint16 max_acc_state_cells:uint32 max_acc_state_bits:uint32
- max_acc_public_libraries:uint32 = SizeLimitsConfig;
+ max_acc_public_libraries:uint32 defer_out_queue_size_limit:uint32 = SizeLimitsConfig;
 _ SizeLimitsConfig = ConfigParam 43;
 
 // key is [ wc:int32 addr:uint256 ]

crypto/block/mc-config.cpp

@@ -1956,6 +1956,7 @@ td::Result<SizeLimitsConfig> Config::do_get_size_limits_config(td::Ref<vm::CellS
  limits.max_acc_state_bits = rec.max_acc_state_bits;
  limits.max_acc_state_cells = rec.max_acc_state_cells;
  limits.max_acc_public_libraries = rec.max_acc_public_libraries;
+ limits.defer_out_queue_size_limit = rec.defer_out_queue_size_limit;
  };
  gen::SizeLimitsConfig::Record_size_limits_config rec_v1;
  gen::SizeLimitsConfig::Record_size_limits_config_v2 rec_v2;

crypto/block/mc-config.h

@@ -395,6 +395,7 @@ struct SizeLimitsConfig {
  td::uint32 max_acc_state_cells = 1 << 16;
  td::uint32 max_acc_state_bits = (1 << 16) * 1023;
  td::uint32 max_acc_public_libraries = 256;
+ td::uint32 defer_out_queue_size_limit = 256;
 };
 
 struct CatchainValidatorsConfig {

tl/generate/scheme/ton_api.tl

@@ -606,6 +606,11 @@ engine.validator.customOverlayNode adnl_id:int256 msg_sender:Bool msg_sender_pri
 engine.validator.customOverlay name:string nodes:(vector engine.validator.customOverlayNode) = engine.validator.CustomOverlay;
 engine.validator.customOverlaysConfig overlays:(vector engine.validator.customOverlay) = engine.validator.CustomOverlaysConfig;
 
+engine.validator.collatorOptions
+ deferring_enabled:Bool defer_messages_after:int defer_out_queue_size_limit:long
+ dispatch_phase_2_max_total:int dispatch_phase_3_max_total:int
+ dispatch_phase_2_max_per_initiator:int dispatch_phase_3_max_per_initiator:int = engine.validator.CollatorOptions;
+
 ---functions---
 ---types---
 
@@ -715,6 +720,8 @@ engine.validator.showCustomOverlays = engine.validator.CustomOverlaysConfig;
 
 engine.validator.setStateSerializerEnabled enabled:Bool = engine.validator.Success;
 
+engine.validator.setCollatorOptionsJson json:string = engine.validator.Success;
+
 ---types---
 
 storage.pong = storage.Pong;

validator-engine-console/validator-engine-console-query.cpp

@@ -1203,3 +1203,42 @@ td::Status SetStateSerializerEnabledQuery::receive(td::BufferSlice data) {
  td::TerminalIO::out() << "success\n";
  return td::Status::OK();
 }
+
+td::Status SetCollatorOptionsJsonQuery::run() {
+ TRY_RESULT_ASSIGN(file_name_, tokenizer_.get_token<std::string>());
+ TRY_STATUS(tokenizer_.check_endl());
+ return td::Status::OK();
+}
+
+td::Status SetCollatorOptionsJsonQuery::send() {
+ TRY_RESULT(data, td::read_file(file_name_));
+ auto b =
+ ton::create_serialize_tl_object<ton::ton_api::engine_validator_setCollatorOptionsJson>(data.as_slice().str());
+ td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
+ return td::Status::OK();
+}
+
+td::Status SetCollatorOptionsJsonQuery::receive(td::BufferSlice data) {
+ TRY_RESULT_PREFIX(f, ton::fetch_tl_object<ton::ton_api::engine_validator_success>(data.as_slice(), true),
+ "received incorrect answer: ");
+ td::TerminalIO::out() << "success\n";
+ return td::Status::OK();
+}
+
+td::Status ResetCollatorOptionsQuery::run() {
+ TRY_STATUS(tokenizer_.check_endl());
+ return td::Status::OK();
+}
+
+td::Status ResetCollatorOptionsQuery::send() {
+ auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_setCollatorOptionsJson>("{}");
+ td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
+ return td::Status::OK();
+}
+
+td::Status ResetCollatorOptionsQuery::receive(td::BufferSlice data) {
+ TRY_RESULT_PREFIX(f, ton::fetch_tl_object<ton::ton_api::engine_validator_success>(data.as_slice(), true),
+ "received incorrect answer: ");
+ td::TerminalIO::out() << "success\n";
+ return td::Status::OK();
+}

validator-engine-console/validator-engine-console-query.h

@@ -1229,3 +1229,44 @@ class SetStateSerializerEnabledQuery : public Query {
  private:
  bool enabled_;
 };
+
+class SetCollatorOptionsJsonQuery : public Query {
+ public:
+ SetCollatorOptionsJsonQuery(td::actor::ActorId<ValidatorEngineConsole> console, Tokenizer tokenizer)
+ : Query(console, std::move(tokenizer)) {
+ }
+ td::Status run() override;
+ td::Status send() override;
+ td::Status receive(td::BufferSlice data) override;
+ static std::string get_name() {
+ return "setcollatoroptionsjson";
+ }
+ static std::string get_help() {
+ return "setcollatoroptionsjson <filename>\tset collator options from file <filename>";
+ }
+ std::string name() const override {
+ return get_name();
+ }
+
+ private:
+ std::string file_name_;
+};
+
+class ResetCollatorOptionsQuery : public Query {
+ public:
+ ResetCollatorOptionsQuery(td::actor::ActorId<ValidatorEngineConsole> console, Tokenizer tokenizer)
+ : Query(console, std::move(tokenizer)) {
+ }
+ td::Status run() override;
+ td::Status send() override;
+ td::Status receive(td::BufferSlice data) override;
+ static std::string get_name() {
+ return "resetcollatoroptions";
+ }
+ static std::string get_help() {
+ return "resetcollatoroptions\tset collator options to default values";
+ }
+ std::string name() const override {
+ return get_name();
+ }
+};

validator-engine-console/validator-engine-console.cpp

@@ -147,6 +147,8 @@ void ValidatorEngineConsole::run() {
  add_query_runner(std::make_unique<QueryRunnerImpl<DelCustomOverlayQuery>>());
  add_query_runner(std::make_unique<QueryRunnerImpl<ShowCustomOverlaysQuery>>());
  add_query_runner(std::make_unique<QueryRunnerImpl<SetStateSerializerEnabledQuery>>());
+ add_query_runner(std::make_unique<QueryRunnerImpl<SetCollatorOptionsJsonQuery>>());
+ add_query_runner(std::make_unique<QueryRunnerImpl<ResetCollatorOptionsQuery>>());
 }
 
 bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise<td::BufferSlice> promise) {

validator-engine/validator-engine.cpp

@@ -1823,6 +1823,7 @@ void ValidatorEngine::started_overlays() {
 void ValidatorEngine::start_validator() {
  validator_options_.write().set_allow_blockchain_init(config_.validators.size() > 0);
  validator_options_.write().set_state_serializer_enabled(config_.state_serializer_enabled);
+ load_collator_options();
 
  validator_manager_ = ton::validator::ValidatorManagerFactory::create(
  validator_options_, db_root_, keyring_.get(), adnl_.get(), rldp_.get(), overlay_manager_.get());
@@ -2414,6 +2415,69 @@ void ValidatorEngine::del_custom_overlay_from_config(std::string name, td::Promi
  promise.set_error(td::Status::Error(PSTRING() << "no overlay \"" << name << "\" in config"));
 }
 
+static td::Result<td::Ref<ton::validator::CollatorOptions>> parse_collator_options(td::MutableSlice json_str) {
+ td::Ref<ton::validator::CollatorOptions> ref{true};
+ ton::validator::CollatorOptions& opts = ref.write();
+
+ // Set default values (from_json leaves missing fields as is)
+ ton::ton_api::engine_validator_collatorOptions f;
+ f.deferring_enabled_ = opts.deferring_enabled;
+ f.defer_out_queue_size_limit_ = opts.defer_out_queue_size_limit;
+ f.defer_messages_after_ = opts.defer_messages_after;
+ f.dispatch_phase_2_max_total_ = opts.dispatch_phase_2_max_total;
+ f.dispatch_phase_3_max_total_ = opts.dispatch_phase_3_max_total;
+ f.dispatch_phase_2_max_per_initiator_ = opts.dispatch_phase_2_max_per_initiator;
+ f.dispatch_phase_3_max_per_initiator_ =
+ opts.dispatch_phase_3_max_per_initiator ? opts.dispatch_phase_3_max_per_initiator.value() : -1;
+
+ TRY_RESULT_PREFIX(json, td::json_decode(json_str), "failed to parse json: ");
+ TRY_STATUS_PREFIX(ton::ton_api::from_json(f, json.get_object()), "json does not fit TL scheme: ");
+
+ if (f.defer_messages_after_ <= 0) {
+ return td::Status::Error("defer_messages_after should be positive");
+ }
+ if (f.defer_out_queue_size_limit_ < 0) {
+ return td::Status::Error("defer_out_queue_size_limit should be non-negative");
+ }
+ if (f.dispatch_phase_2_max_total_ < 0) {
+ return td::Status::Error("dispatch_phase_2_max_total should be non-negative");
+ }
+ if (f.dispatch_phase_3_max_total_ < 0) {
+ return td::Status::Error("dispatch_phase_3_max_total should be non-negative");
+ }
+ if (f.dispatch_phase_2_max_per_initiator_ < 0) {
+ return td::Status::Error("dispatch_phase_2_max_per_initiator should be non-negative");
+ }
+
+ opts.deferring_enabled = f.deferring_enabled_;
+ opts.defer_messages_after = f.defer_messages_after_;
+ opts.defer_out_queue_size_limit = f.defer_out_queue_size_limit_;
+ opts.dispatch_phase_2_max_total = f.dispatch_phase_2_max_total_;
+ opts.dispatch_phase_3_max_total = f.dispatch_phase_3_max_total_;
+ opts.dispatch_phase_2_max_per_initiator = f.dispatch_phase_2_max_per_initiator_;
+ if (f.dispatch_phase_3_max_per_initiator_ >= 0) {
+ opts.dispatch_phase_3_max_per_initiator = f.dispatch_phase_3_max_per_initiator_;
+ } else {
+ opts.dispatch_phase_3_max_per_initiator = {};
+ }
+
+ return ref;
+}
+
+void ValidatorEngine::load_collator_options() {
+ auto r_data = td::read_file(collator_options_file());
+ if (r_data.is_error()) {
+ return;
+ }
+ td::BufferSlice data = r_data.move_as_ok();
+ auto r_collator_options = parse_collator_options(data.as_slice());
+ if (r_collator_options.is_error()) {
+ LOG(ERROR) << "Failed to read collator options from file: " << r_collator_options.move_as_error();
+ return;
+ }
+ validator_options_.write().set_collator_options(r_collator_options.move_as_ok());
+}
+
 void ValidatorEngine::check_key(ton::PublicKeyHash id, td::Promise<td::Unit> promise) {
  if (keys_.count(id) == 1) {
  promise.set_value(td::Unit());
@@ -3684,6 +3748,33 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setStateS
  });
 }
 
+void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setCollatorOptionsJson &query,
+ td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm,
+ td::Promise<td::BufferSlice> promise) {
+ if (!(perm & ValidatorEnginePermissions::vep_modify)) {
+ promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized")));
+ return;
+ }
+ if (!started_) {
+ promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started")));
+ return;
+ }
+ auto r_collator_options = parse_collator_options(query.json_);
+ if (r_collator_options.is_error()) {
+ promise.set_value(create_control_query_error(r_collator_options.move_as_error_prefix("failed to parse json: ")));
+ return;
+ }
+ auto S = td::write_file(collator_options_file(), query.json_);
+ if (S.is_error()) {
+ promise.set_value(create_control_query_error(r_collator_options.move_as_error_prefix("failed to write file: ")));
+ return;
+ }
+ validator_options_.write().set_collator_options(r_collator_options.move_as_ok());
+ td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options,
+ validator_options_);
+ promise.set_value(ton::create_serialize_tl_object<ton::ton_api::engine_validator_success>());
+}
+
 void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src,
  ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data,
  td::Promise<td::BufferSlice> promise) {

validator-engine/validator-engine.hpp

@@ -384,12 +384,16 @@ class ValidatorEngine : public td::actor::Actor {
  std::string custom_overlays_config_file() const {
  return db_root_ + "/custom-overlays.json";
  }
+ std::string collator_options_file() const {
+ return db_root_ + "/collator-options.json";
+ }
 
  void load_custom_overlays_config();
  td::Status write_custom_overlays_config();
  void add_custom_overlay_to_config(
  ton::tl_object_ptr<ton::ton_api::engine_validator_customOverlay> overlay, td::Promise<td::Unit> promise);
  void del_custom_overlay_from_config(std::string name, td::Promise<td::Unit> promise);
+ void load_collator_options();
 
  void check_key(ton::PublicKeyHash id, td::Promise<td::Unit> promise);
 
@@ -477,6 +481,8 @@ class ValidatorEngine : public td::actor::Actor {
  ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise);
  void run_control_query(ton::ton_api::engine_validator_setStateSerializerEnabled &query, td::BufferSlice data,
  ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise);
+ void run_control_query(ton::ton_api::engine_validator_setCollatorOptionsJson &query, td::BufferSlice data,
+ ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise);
  template <class T>
  void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm,
  td::Promise<td::BufferSlice> promise) {

validator/fabric.h

@@ -80,8 +80,8 @@ void run_validate_query(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_maste
  td::Promise<ValidateCandidateResult> promise, bool is_fake = false);
 void run_collate_query(ShardIdFull shard, td::uint32 min_ts, const BlockIdExt& min_masterchain_block_id,
  std::vector<BlockIdExt> prev, Ed25519_PublicKey local_id, td::Ref<ValidatorSet> validator_set,
- td::actor::ActorId<ValidatorManager> manager, td::Timestamp timeout,
- td::Promise<BlockCandidate> promise);
+ td::Ref<CollatorOptions> collator_opts, td::actor::ActorId<ValidatorManager> manager,
+ td::Timestamp timeout, td::Promise<BlockCandidate> promise);
 void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector<BlockIdExt> prev,
  td::actor::ActorId<ValidatorManager> manager, td::Timestamp timeout,
  td::Promise<BlockCandidate> promise);

validator/impl/collator-impl.h

@@ -71,6 +71,7 @@ class Collator final : public td::actor::Actor {
  std::vector<Ref<ShardState>> prev_states;
  std::vector<Ref<BlockData>> prev_block_data;
  Ed25519_PublicKey created_by_;
+ Ref<CollatorOptions> collator_opts_;
  Ref<ValidatorSet> validator_set_;
  td::actor::ActorId<ValidatorManager> manager;
  td::Timestamp timeout;
@@ -90,7 +91,8 @@ class Collator final : public td::actor::Actor {
  public:
  Collator(ShardIdFull shard, bool is_hardfork, td::uint32 min_ts, BlockIdExt min_masterchain_block_id,
  std::vector<BlockIdExt> prev, Ref<ValidatorSet> validator_set, Ed25519_PublicKey collator_id,
- td::actor::ActorId<ValidatorManager> manager, td::Timestamp timeout, td::Promise<BlockCandidate> promise);
+ Ref<CollatorOptions> collator_opts, td::actor::ActorId<ValidatorManager> manager, td::Timestamp timeout,
+ td::Promise<BlockCandidate> promise);
  ~Collator() override = default;
  bool is_busy() const {
  return busy_;
@@ -195,6 +197,7 @@ class Collator final : public td::actor::Actor {
  std::unique_ptr<vm::AugmentedDictionary> in_msg_dict, out_msg_dict, out_msg_queue_, sibling_out_msg_queue_;
  std::map<StdSmcAddress, size_t> unprocessed_deferred_messages_; // number of messages from dispatch queue in new_msgs
  td::uint64 out_msg_queue_size_ = 0;
+ td::uint64 old_out_msg_queue_size_ = 0;
  bool have_out_msg_queue_size_in_state_ = false;
  std::unique_ptr<vm::Dictionary> ihr_pending;
  std::shared_ptr<block::MsgProcessedUptoCollection> processed_upto_, sibling_processed_upto_;
@@ -211,6 +214,8 @@ class Collator final : public td::actor::Actor {
  unsigned dispatch_queue_ops_{0};
  std::map<StdSmcAddress, LogicalTime> last_dispatch_queue_emitted_lt_;
  bool have_unprocessed_account_dispatch_queue_ = true;
+ td::uint64 defer_out_queue_size_limit_;
+ td::uint64 hard_defer_out_queue_size_limit_;
 
  bool msg_metadata_enabled_ = false;
  bool deferring_messages_enabled_ = false;