Use candidates from DB in wait block data

tl/generate/scheme/ton_api.tl

@@ -524,6 +524,7 @@ db.filedb.key.proof block_id:tonNode.blockIdExt = db.filedb.Key;
 db.filedb.key.proofLink block_id:tonNode.blockIdExt = db.filedb.Key;
 db.filedb.key.signatures block_id:tonNode.blockIdExt = db.filedb.Key;
 db.filedb.key.candidate id:db.candidate.id = db.filedb.Key;
+db.filedb.key.candidateRef id:tonNode.blockIdExt = db.filedb.Key;
 db.filedb.key.blockInfo block_id:tonNode.blockIdExt = db.filedb.Key;
 
 db.filedb.value key:db.filedb.Key prev:int256 next:int256 file_hash:int256 = db.filedb.Value;

validator/collator-node.cpp

@@ -149,7 +149,7 @@ void CollatorNode::new_masterchain_block_notification(td::Ref<MasterchainState>
  }
  }
 
- for (auto it = future_validator_groups_.begin(); it != future_validator_groups_.end(); ++it) {
+ for (auto it = future_validator_groups_.begin(); it != future_validator_groups_.end();) {
  if (get_future_validator_group(it->first.first, it->first.second).is_ok()) {
  ++it;
  } else {

validator/db/fileref.cpp

@@ -213,6 +213,28 @@ std::string CandidateShort::filename_short() const {
  return PSTRING() << "candidate_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex();
 }
 
+CandidateRefShort CandidateRef::shortref() const {
+ return CandidateRefShort{block_id.id, hash()};
+}
+
+std::string CandidateRef::filename() const {
+ return PSTRING() << "candidateref_" << block_id.to_str();
+}
+
+std::string CandidateRef::filename_short() const {
+ char s[33];
+ sprintf(s, "%llx", static_cast<long long>(block_id.id.shard));
+ return PSTRING() << "candidateref_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_"
+ << hash().to_hex();
+}
+
+std::string CandidateRefShort::filename_short() const {
+ char s[33];
+ sprintf(s, "%llx", static_cast<long long>(block_id.shard));
+ return PSTRING() << "candidateref_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_"
+ << hash().to_hex();
+}
+
 BlockInfoShort BlockInfo::shortref() const {
  return BlockInfoShort{block_id.id, hash()};
 }
@@ -259,6 +281,9 @@ FileReference::FileReference(tl_object_ptr<ton_api::db_filedb_Key> key) {
  ref_ = fileref::Candidate{PublicKey{key.id_->source_}, create_block_id(key.id_->id_),
  key.id_->collated_data_file_hash_};
  },
+ [&](const ton_api::db_filedb_key_candidateRef& key) {
+ ref_ = fileref::CandidateRef{create_block_id(key.id_)};
+ },
  [&](const ton_api::db_filedb_key_blockInfo& key) {
  ref_ = fileref::BlockInfo{create_block_id(key.block_id_)};
  }));

validator/db/fileref.hpp

@@ -278,6 +278,38 @@ class Candidate {
  FileHash collated_data_file_hash;
 };
 
+class CandidateRefShort {
+ public:
+ FileHash hash() const {
+ return hashv;
+ }
+ ShardIdFull shard() const {
+ return block_id.shard_full();
+ }
+ std::string filename_short() const;
+
+ BlockId block_id;
+ FileHash hashv;
+};
+
+class CandidateRef {
+ public:
+ tl_object_ptr<ton_api::db_filedb_Key> tl() const {
+ return create_tl_object<ton_api::db_filedb_key_candidateRef>(create_tl_block_id(block_id));
+ }
+ FileHash hash() const {
+ return create_hash_tl_object<ton_api::db_filedb_key_candidateRef>(create_tl_block_id(block_id));
+ }
+ ShardIdFull shard() const {
+ return block_id.shard_full();
+ }
+ CandidateRefShort shortref() const;
+ std::string filename() const;
+ std::string filename_short() const;
+
+ BlockIdExt block_id;
+};
+
 class BlockInfoShort {
  public:
  FileHash hash() const {
@@ -316,7 +348,7 @@ class FileReferenceShort {
  private:
  td::Variant<fileref::Empty, fileref::BlockShort, fileref::ZeroStateShort, fileref::PersistentStateShort,
  fileref::ProofShort, fileref::ProofShort, fileref::ProofLinkShort, fileref::SignaturesShort,
- fileref::CandidateShort, fileref::BlockInfoShort>
+ fileref::CandidateShort, fileref::CandidateRefShort, fileref::BlockInfoShort>
  ref_;
 
  public:
@@ -340,7 +372,8 @@ class FileReferenceShort {
 class FileReference {
  private:
  td::Variant<fileref::Empty, fileref::Block, fileref::ZeroState, fileref::PersistentState, fileref::Proof,
- fileref::Proof, fileref::ProofLink, fileref::Signatures, fileref::Candidate, fileref::BlockInfo>
+ fileref::Proof, fileref::ProofLink, fileref::Signatures, fileref::Candidate, fileref::CandidateRef,
+ fileref::BlockInfo>
  ref_;
 
  public:

validator/db/rootdb.cpp

@@ -177,21 +177,21 @@ void RootDb::get_block_proof_link(ConstBlockHandle handle, td::Promise<td::Ref<P
 }
 
 void RootDb::store_block_candidate(BlockCandidate candidate, td::Promise<td::Unit> promise) {
+ auto source = PublicKey{pubkeys::Ed25519{candidate.pubkey.as_bits256()}};
  auto obj = create_serialize_tl_object<ton_api::db_candidate>(
- PublicKey{pubkeys::Ed25519{candidate.pubkey.as_bits256()}}.tl(), create_tl_block_id(candidate.id),
-  std::move(candidate.data), std::move(candidate.collated_data));
-
- auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result<td::Unit> R) mutable {
- if (R.is_error()) {
- promise.set_error(R.move_as_error());
- } else {
- promise.set_value(td::Unit());
- }
- });
+ source.tl(), create_tl_block_id(candidate.id), std::move(candidate.data), std::move(candidate.collated_data));
+ auto P = td::PromiseCreator::lambda(
+ [archive_db = archive_db_.get(), promise = std::move(promise), block_id = candidate.id, source,
+  collated_file_hash = candidate.collated_file_hash](td::Result<td::Unit> R) mutable {
+  TRY_RESULT_PROMISE(promise, _, std::move(R));
+  td::actor::send_closure(archive_db, &ArchiveManager::add_temp_file_short, fileref::CandidateRef{block_id},
+  create_serialize_tl_object<ton_api::db_candidate_id>(
+  source.tl(), create_tl_block_id(block_id), collated_file_hash),
+  std::move(promise));
+  });
  td::actor::send_closure(archive_db_, &ArchiveManager::add_temp_file_short,
- fileref::Candidate{PublicKey{pubkeys::Ed25519{candidate.pubkey.as_bits256()}}, candidate.id,
- candidate.collated_file_hash},
- std::move(obj), std::move(P));
+ fileref::Candidate{source, candidate.id, candidate.collated_file_hash}, std::move(obj),
+ std::move(P));
 }
 
 void RootDb::get_block_candidate(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash,
@@ -215,6 +215,17 @@ void RootDb::get_block_candidate(PublicKey source, BlockIdExt id, FileHash colla
  fileref::Candidate{source, id, collated_data_file_hash}, std::move(P));
 }
 
+void RootDb::get_block_candidate_by_block_id(BlockIdExt id, td::Promise<BlockCandidate> promise) {
+ td::actor::send_closure(
+ archive_db_, &ArchiveManager::get_temp_file_short, fileref::CandidateRef{id},
+ [SelfId = actor_id(this), promise = std::move(promise)](td::Result<td::BufferSlice> R) mutable {
+ TRY_RESULT_PROMISE(promise, data, std::move(R));
+ TRY_RESULT_PROMISE(promise, f, fetch_tl_object<ton_api::db_candidate_id>(data, true));
+ td::actor::send_closure(SelfId, &RootDb::get_block_candidate, PublicKey{f->source_}, create_block_id(f->id_),
+ f->collated_data_file_hash_, std::move(promise));
+ });
+}
+
 void RootDb::store_block_state(BlockHandle handle, td::Ref<ShardState> state,
  td::Promise<td::Ref<ShardState>> promise) {
  if (handle->moved_to_archive()) {

validator/db/rootdb.hpp

@@ -58,6 +58,7 @@ class RootDb : public Db {
  void store_block_candidate(BlockCandidate candidate, td::Promise<td::Unit> promise) override;
  void get_block_candidate(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash,
  td::Promise<BlockCandidate> promise) override;
+ void get_block_candidate_by_block_id(BlockIdExt id, td::Promise<BlockCandidate> promise) override;
 
  void store_block_state(BlockHandle handle, td::Ref<ShardState> state,
  td::Promise<td::Ref<ShardState>> promise) override;

validator/downloaders/wait-block-data.cpp

@@ -106,13 +106,24 @@ void WaitBlockData::start() {
  });
 
  td::actor::send_closure(manager_, &ValidatorManager::try_get_static_file, handle_->id().file_hash, std::move(P));
+ } else if (try_get_candidate_) {
+ try_get_candidate_ = false;
+ td::actor::send_closure(
+ manager_, &ValidatorManager::get_candidate_data_by_block_id_from_db, handle_->id(),
+ [SelfId = actor_id(this), id = handle_->id()](td::Result<td::BufferSlice> R) {
+ if (R.is_error()) {
+ td::actor::send_closure(SelfId, &WaitBlockData::start);
+ } else {
+ td::actor::send_closure(SelfId, &WaitBlockData::loaded_data, ReceivedBlock{id, R.move_as_ok()});
+ }
+ });
  } else {
  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<ReceivedBlock> R) {
  if (R.is_error()) {
  td::actor::send_closure(SelfId, &WaitBlockData::failed_to_get_block_data_from_net,
  R.move_as_error_prefix("net error: "));
  } else {
- td::actor::send_closure(SelfId, &WaitBlockData::got_data_from_net, R.move_as_ok());
+ td::actor::send_closure(SelfId, &WaitBlockData::loaded_data, R.move_as_ok());
  }
  });
 
@@ -137,16 +148,16 @@ void WaitBlockData::failed_to_get_block_data_from_net(td::Status reason) {
  td::Timestamp::in(0.1));
 }
 
-void WaitBlockData::got_data_from_net(ReceivedBlock block) {
+void WaitBlockData::loaded_data(ReceivedBlock block) {
  auto X = create_block(std::move(block));
  if (X.is_error()) {
  failed_to_get_block_data_from_net(X.move_as_error_prefix("bad block from net: "));
  return;
  }
- got_block_data_from_net(X.move_as_ok());
+ loaded_block_data(X.move_as_ok());
 }
 
-void WaitBlockData::got_block_data_from_net(td::Ref<BlockData> block) {
+void WaitBlockData::loaded_block_data(td::Ref<BlockData> block) {
  if (data_.not_null()) {
  return;
  }

validator/downloaders/wait-block-data.hpp

@@ -30,15 +30,16 @@ class ValidatorManager;
 class WaitBlockData : public td::actor::Actor {
  public:
  WaitBlockData(BlockHandle handle, td::uint32 priority, td::actor::ActorId<ValidatorManager> manager,
- td::Timestamp timeout, td::Promise<td::Ref<BlockData>> promise)
+ td::Timestamp timeout, bool try_get_candidate, td::Promise<td::Ref<BlockData>> promise)
  : handle_(std::move(handle))
  , priority_(priority)
  , manager_(manager)
  , timeout_(timeout)
+ , try_get_candidate_(try_get_candidate)
  , promise_(std::move(promise))
  , perf_timer_("waitdata", 1.0, [manager](double duration) {
-  send_closure(manager, &ValidatorManager::add_perf_timer_stat, "waitdata", duration);
-  }) {
+ send_closure(manager, &ValidatorManager::add_perf_timer_stat, "waitdata", duration);
+ }) {
  }
 
  void update_timeout(td::Timestamp timeout, td::uint32 priority) {
@@ -57,8 +58,8 @@ class WaitBlockData : public td::actor::Actor {
  void set_is_hardfork(bool value);
  void start();
  void got_block_data_from_db(td::Ref<BlockData> data);
- void got_data_from_net(ReceivedBlock data);
- void got_block_data_from_net(td::Ref<BlockData> block);
+ void loaded_data(ReceivedBlock data);
+ void loaded_block_data(td::Ref<BlockData> block);
  void checked_proof_link();
  void failed_to_get_block_data_from_net(td::Status reason);
 
@@ -73,6 +74,7 @@ class WaitBlockData : public td::actor::Actor {
 
  td::actor::ActorId<ValidatorManager> manager_;
  td::Timestamp timeout_;
+ bool try_get_candidate_;
  td::Promise<td::Ref<BlockData>> promise_;
 
  td::Ref<BlockData> data_;

validator/interfaces/db.h

@@ -46,6 +46,7 @@ class Db : public td::actor::Actor {
  virtual void store_block_candidate(BlockCandidate candidate, td::Promise<td::Unit> promise) = 0;
  virtual void get_block_candidate(ton::PublicKey source, BlockIdExt id, FileHash collated_data_file_hash,
  td::Promise<BlockCandidate> promise) = 0;
+ virtual void get_block_candidate_by_block_id(BlockIdExt id, td::Promise<BlockCandidate> promise) = 0;
 
  virtual void store_block_state(BlockHandle handle, td::Ref<ShardState> state,
  td::Promise<td::Ref<ShardState>> promise) = 0;

validator/manager-disk.cpp

@@ -589,6 +589,11 @@ void ValidatorManagerImpl::get_block_candidate_from_db(PublicKey source, BlockId
  td::actor::send_closure(db_, &Db::get_block_candidate, source, id, collated_data_file_hash, std::move(promise));
 }
 
+void ValidatorManagerImpl::get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise<td::BufferSlice> promise) {
+ td::actor::send_closure(db_, &Db::get_block_candidate_by_block_id, id,
+ promise.wrap([](BlockCandidate &&b) { return std::move(b.data); }));
+}
+
 void ValidatorManagerImpl::get_block_proof_from_db(ConstBlockHandle handle, td::Promise<td::Ref<Proof>> promise) {
  td::actor::send_closure(db_, &Db::get_block_proof, std::move(handle), std::move(promise));
 }

validator/manager-disk.hpp

@@ -217,6 +217,7 @@ class ValidatorManagerImpl : public ValidatorManager {
  void get_shard_state_from_db_short(BlockIdExt block_id, td::Promise<td::Ref<ShardState>> promise) override;
  void get_block_candidate_from_db(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash,
  td::Promise<BlockCandidate> promise) override;
+ void get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise<td::BufferSlice> promise) override;
  void get_block_proof_from_db(ConstBlockHandle handle, td::Promise<td::Ref<Proof>> promise) override;
  void get_block_proof_from_db_short(BlockIdExt id, td::Promise<td::Ref<Proof>> promise) override;
  void get_block_proof_link_from_db(ConstBlockHandle handle, td::Promise<td::Ref<ProofLink>> promise) override;

validator/manager-hardfork.cpp

@@ -415,6 +415,11 @@ void ValidatorManagerImpl::get_block_candidate_from_db(PublicKey source, BlockId
  td::actor::send_closure(db_, &Db::get_block_candidate, source, id, collated_data_file_hash, std::move(promise));
 }
 
+void ValidatorManagerImpl::get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise<td::BufferSlice> promise) {
+ td::actor::send_closure(db_, &Db::get_block_candidate_by_block_id, id,
+ promise.wrap([](BlockCandidate &&b) { return std::move(b.data); }));
+}
+
 void ValidatorManagerImpl::get_block_proof_from_db(ConstBlockHandle handle, td::Promise<td::Ref<Proof>> promise) {
  td::actor::send_closure(db_, &Db::get_block_proof, std::move(handle), std::move(promise));
 }

validator/manager-hardfork.hpp

@@ -263,6 +263,7 @@ class ValidatorManagerImpl : public ValidatorManager {
  void get_shard_state_from_db_short(BlockIdExt block_id, td::Promise<td::Ref<ShardState>> promise) override;
  void get_block_candidate_from_db(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash,
  td::Promise<BlockCandidate> promise) override;
+ void get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise<td::BufferSlice> promise) override;
  void get_block_proof_from_db(ConstBlockHandle handle, td::Promise<td::Ref<Proof>> promise) override;
  void get_block_proof_from_db_short(BlockIdExt id, td::Promise<td::Ref<Proof>> promise) override;
  void get_block_proof_link_from_db(ConstBlockHandle handle, td::Promise<td::Ref<ProofLink>> promise) override;

validator/manager.cpp

@@ -622,7 +622,7 @@ void ValidatorManagerImpl::add_cached_block_candidate(ReceivedBlock block) {
  if (it != wait_block_data_.end()) {
  auto r_block = create_block(cached_block_candidates_[id].clone());
  if (r_block.is_ok()) {
- td::actor::send_closure(it->second.actor_, &WaitBlockData::got_block_data_from_net, r_block.move_as_ok());
+ td::actor::send_closure(it->second.actor_, &WaitBlockData::loaded_block_data, r_block.move_as_ok());
  }
  }
  }
@@ -852,7 +852,8 @@ void ValidatorManagerImpl::wait_block_data(BlockHandle handle, td::uint32 priori
  td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_data, handle, std::move(R));
  });
  auto id = td::actor::create_actor<WaitBlockData>("waitdata", handle, priority, actor_id(this),
- td::Timestamp::at(timeout.at() + 10.0), std::move(P))
+ td::Timestamp::at(timeout.at() + 10.0),
+ is_shard_collator(handle->id().shard_full()), std::move(P))
  .release();
  wait_block_data_[handle->id()].actor_ = id;
  it = wait_block_data_.find(handle->id());
@@ -1177,6 +1178,16 @@ void ValidatorManagerImpl::get_block_candidate_from_db(PublicKey source, BlockId
  td::actor::send_closure(db_, &Db::get_block_candidate, source, id, collated_data_file_hash, std::move(promise));
 }
 
+void ValidatorManagerImpl::get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise<td::BufferSlice> promise) {
+ auto it = cached_block_candidates_.find(id);
+ if (it != cached_block_candidates_.end()) {
+ promise.set_result(it->second.data.clone());
+ return;
+ }
+ td::actor::send_closure(db_, &Db::get_block_candidate_by_block_id, id,
+ promise.wrap([](BlockCandidate &&b) { return std::move(b.data); }));
+}
+
 void ValidatorManagerImpl::get_block_proof_from_db(ConstBlockHandle handle, td::Promise<td::Ref<Proof>> promise) {
  td::actor::send_closure(db_, &Db::get_block_proof, std::move(handle), std::move(promise));
 }
@@ -1289,9 +1300,9 @@ void ValidatorManagerImpl::finished_wait_data(BlockHandle handle, td::Result<td:
  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result<td::Ref<BlockData>> R) {
  td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_data, handle, std::move(R));
  });
- auto id =
- td::actor::create_actor<WaitBlockData>("waitdata", handle, X.second, actor_id(this), X.first, std::move(P))
- .release();
+ auto id = td::actor::create_actor<WaitBlockData>("waitdata", handle, X.second, actor_id(this), X.first,
+  is_shard_collator(handle->id().shard_full()), std::move(P))
+  .release();
  it->second.actor_ = id;
  return;
  }

validator/manager.hpp

@@ -479,6 +479,7 @@ class ValidatorManagerImpl : public ValidatorManager {
  void get_shard_state_from_db_short(BlockIdExt block_id, td::Promise<td::Ref<ShardState>> promise) override;
  void get_block_candidate_from_db(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash,
  td::Promise<BlockCandidate> promise) override;
+ void get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise<td::BufferSlice> promise) override;
  void get_block_proof_from_db(ConstBlockHandle handle, td::Promise<td::Ref<Proof>> promise) override;
  void get_block_proof_from_db_short(BlockIdExt id, td::Promise<td::Ref<Proof>> promise) override;
  void get_block_proof_link_from_db(ConstBlockHandle handle, td::Promise<td::Ref<ProofLink>> promise) override;

validator/validator.h

@@ -262,6 +262,7 @@ class ValidatorManagerInterface : public td::actor::Actor {
  virtual void get_block_data_from_db_short(BlockIdExt block_id, td::Promise<td::Ref<BlockData>> promise) = 0;
  virtual void get_block_candidate_from_db(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash,
  td::Promise<BlockCandidate> promise) = 0;
+ virtual void get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise<td::BufferSlice> promise) = 0;
  virtual void get_shard_state_from_db(ConstBlockHandle handle, td::Promise<td::Ref<ShardState>> promise) = 0;
  virtual void get_shard_state_from_db_short(BlockIdExt block_id, td::Promise<td::Ref<ShardState>> promise) = 0;
  virtual void get_block_proof_from_db(ConstBlockHandle handle, td::Promise<td::Ref<Proof>> promise) = 0;