diff --git a/CMakeLists.txt b/CMakeLists.txt index 665fc2e9d..573bc3a32 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -79,7 +79,7 @@ else() set(HAVE_SSE42 FALSE) endif() -set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED TRUE) set(CMAKE_CXX_EXTENSIONS FALSE) @@ -333,6 +333,10 @@ add_cxx_compiler_flag("-Wno-sign-conversion") add_cxx_compiler_flag("-Qunused-arguments") add_cxx_compiler_flag("-Wno-unused-private-field") add_cxx_compiler_flag("-Wno-redundant-move") + +#add_cxx_compiler_flag("-Wno-unused-function") +#add_cxx_compiler_flag("-Wno-unused-variable") +#add_cxx_compiler_flag("-Wno-shorten-64-to-32") #add_cxx_compiler_flag("-Werror") #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -isystem /usr/include/c++/v1") diff --git a/assembly/nix/linux-x86-64-tonlib.nix b/assembly/nix/linux-x86-64-tonlib.nix index afcbe3ba6..f961eea21 100644 --- a/assembly/nix/linux-x86-64-tonlib.nix +++ b/assembly/nix/linux-x86-64-tonlib.nix @@ -60,7 +60,7 @@ stdenv227.mkDerivation { dontAddStaticConfigureFlags = false; cmakeFlags = [ - "-DTON_USE_ABSEIL=OFF" + "-DTON_USE_ABSEIL=ON" "-DNIX=ON" ]; diff --git a/catchain/catchain-receiver.cpp b/catchain/catchain-receiver.cpp index a6ecf0611..edef90659 100644 --- a/catchain/catchain-receiver.cpp +++ b/catchain/catchain-receiver.cpp @@ -27,6 +27,8 @@ #include "catchain-receiver.hpp" +#include "td/utils/ThreadSafeCounter.h" + namespace ton { namespace catchain { @@ -369,7 +371,7 @@ void CatChainReceiverImpl::add_block(td::BufferSlice payload, std::vectorheight_ + 1; auto max_block_height = get_max_block_height(opts_, sources_.size()); - if (height > max_block_height) { + if (td::narrow_cast(height) > max_block_height) { VLOG(CATCHAIN_WARNING) << this << ": cannot create block: max height exceeded (" << max_block_height << ")"; active_send_ = false; return; @@ -685,6 +687,7 @@ void CatChainReceiverImpl::receive_query_from_overlay(adnl::AdnlNodeIdShort src, promise.set_error(td::Status::Error(ErrorCode::notready, "db not read")); return; } + TD_PERF_COUNTER(catchain_query_process); td::PerfWarningTimer t{"catchain query process", 0.05}; auto F = fetch_tl_object(data.clone(), true); if (F.is_error()) { diff --git a/common/delay.h b/common/delay.h index 9b98c58e5..3df8e7d86 100644 --- a/common/delay.h +++ b/common/delay.h @@ -49,4 +49,29 @@ template void delay_action(T promise, td::Timestamp timeout) { DelayedAction::create(std::move(promise), timeout); } + +template +class AsyncApply : public td::actor::Actor { + public: + AsyncApply(PromiseT promise, ValueT value) : promise_(std::move(promise)), value_(std::move(value)){ + } + + void start_up() override { + promise_(std::move(value_)); + stop(); + } + + static void create(td::Slice name, PromiseT promise, ValueT value ) { + td::actor::create_actor(PSLICE() << "async:" << name, std::move(promise), std::move(value)).release(); + } + + private: + PromiseT promise_; + ValueT value_; +}; + +template +void async_apply(td::Slice name, PromiseT &&promise, ValueT &&value) { + AsyncApply::create(name, std::forward(promise), std::forward(value)); +} } // namespace ton diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index b3291602a..c2a737149 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -151,6 +151,7 @@ set(TON_DB_SOURCE vm/db/CellHashTable.h vm/db/CellStorage.h vm/db/TonDb.h + vm/db/InMemoryBagOfCellsDb.cpp ) set(FIFT_SOURCE diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index d682271fb..16b4897ac 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -320,7 +320,7 @@ ton::ValidatorSessionConfig Config::get_consensus_config() const { c.max_block_size = r.max_block_bytes; c.max_collated_data_size = r.max_collated_bytes; }; - auto set_v2 = [&] (auto& r) { + auto set_v2 = [&](auto& r) { set_v1(r); c.new_catchain_ids = r.new_catchain_ids; }; @@ -1940,7 +1940,7 @@ td::Result Config::get_size_limits_config() const { td::Result Config::do_get_size_limits_config(td::Ref cs) { SizeLimitsConfig limits; if (cs.is_null()) { - return limits; // default values + return limits; // default values } auto unpack_v1 = [&](auto& rec) { limits.max_msg_bits = rec.max_msg_bits; @@ -2299,17 +2299,14 @@ td::Result> ConfigInfo::get_prev_blocks_info() const { if (shard->sgn() < 0) { shard &= ((td::make_refint(1) << 64) - 1); } - return vm::make_tuple_ref( - td::make_refint(block_id.id.workchain), - std::move(shard), - td::make_refint(block_id.id.seqno), - td::bits_to_refint(block_id.root_hash.bits(), 256), - td::bits_to_refint(block_id.file_hash.bits(), 256)); + return vm::make_tuple_ref(td::make_refint(block_id.id.workchain), std::move(shard), + td::make_refint(block_id.id.seqno), td::bits_to_refint(block_id.root_hash.bits(), 256), + td::bits_to_refint(block_id.file_hash.bits(), 256)); }; std::vector last_mc_blocks; last_mc_blocks.push_back(block_id_to_tuple(block_id)); - for (ton::BlockSeqno seqno = block_id.id.seqno; seqno > 0 && last_mc_blocks.size() < 16; ) { + for (ton::BlockSeqno seqno = block_id.id.seqno; seqno > 0 && last_mc_blocks.size() < 16;) { --seqno; ton::BlockIdExt block_id; if (!get_old_mc_block_id(seqno, block_id)) { @@ -2323,9 +2320,8 @@ td::Result> ConfigInfo::get_prev_blocks_info() const { if (!get_last_key_block(last_key_block, last_key_block_lt)) { return td::Status::Error("cannot fetch last key block"); } - return vm::make_tuple_ref( - td::make_cnt_ref>(std::move(last_mc_blocks)), - block_id_to_tuple(last_key_block)); + return vm::make_tuple_ref(td::make_cnt_ref>(std::move(last_mc_blocks)), + block_id_to_tuple(last_key_block)); } td::optional PrecompiledContractsConfig::get_contract( diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index 94ab291bf..2c4de0c65 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -197,6 +197,7 @@ struct McShardHash : public McShardHashI { : blk_(blk), start_lt_(start_lt), end_lt_(end_lt) { } McShardHash(const McShardHash&) = default; + McShardHash& operator=(const McShardHash&) = default; bool is_valid() const { return blk_.is_valid(); } @@ -545,7 +546,10 @@ class Config { }; public: - enum { needValidatorSet = 16, needSpecialSmc = 32, needWorkchainInfo = 256, needCapabilities = 512 }; + static constexpr int needValidatorSet = 16; + static constexpr int needSpecialSmc = 32; + static constexpr int needWorkchainInfo = 256; + static constexpr int needCapabilities = 512; int mode{0}; ton::BlockIdExt block_id; @@ -682,14 +686,12 @@ class Config { class ConfigInfo : public Config, public ShardConfig { public: - enum { - needStateRoot = 1, - needLibraries = 2, - needStateExtraRoot = 4, - needShardHashes = 8, - needAccountsRoot = 64, - needPrevBlocks = 128 - }; + static constexpr int needStateRoot = 1; + static constexpr int needLibraries = 2; + static constexpr int needStateExtraRoot = 4; + static constexpr int needShardHashes = 8; + static constexpr int needAccountsRoot = 64; + static constexpr int needPrevBlocks = 128; ton::BlockSeqno vert_seqno{~0U}; int global_id_{0}; ton::UnixTime utime{0}; diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index dbf0199e7..2e4cda220 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -2860,22 +2860,26 @@ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, vm::CellStorageStat storage_stat; storage_stat.limit_cells = size_limits.max_acc_state_cells; storage_stat.limit_bits = size_limits.max_acc_state_bits; - td::Timer timer; - auto add_used_storage = [&](const td::Ref& cell) -> td::Status { - if (cell.not_null()) { - TRY_RESULT(res, storage_stat.add_used_storage(cell)); - if (res.max_merkle_depth > max_allowed_merkle_depth) { - return td::Status::Error("too big merkle depth"); + { + TD_PERF_COUNTER(transaction_storage_stat_a); + td::Timer timer; + auto add_used_storage = [&](const td::Ref& cell) -> td::Status { + if (cell.not_null()) { + TRY_RESULT(res, storage_stat.add_used_storage(cell)); + if (res.max_merkle_depth > max_allowed_merkle_depth) { + return td::Status::Error("too big merkle depth"); + } } + return td::Status::OK(); + }; + TRY_STATUS(add_used_storage(new_code)); + TRY_STATUS(add_used_storage(new_data)); + TRY_STATUS(add_used_storage(new_library)); + if (timer.elapsed() > 0.1) { + LOG(INFO) << "Compute used storage took " << timer.elapsed() << "s"; } - return td::Status::OK(); - }; - TRY_STATUS(add_used_storage(new_code)); - TRY_STATUS(add_used_storage(new_data)); - TRY_STATUS(add_used_storage(new_library)); - if (timer.elapsed() > 0.1) { - LOG(INFO) << "Compute used storage took " << timer.elapsed() << "s"; } + if (acc_status == Account::acc_active) { storage_stat.clear_limit(); } else { @@ -3156,6 +3160,7 @@ bool Transaction::compute_state() { if (new_stats) { stats = new_stats.unwrap(); } else { + TD_PERF_COUNTER(transaction_storage_stat_b); td::Timer timer; stats.add_used_storage(Ref(storage)).ensure(); if (timer.elapsed() > 0.1) { diff --git a/crypto/common/bigint.hpp b/crypto/common/bigint.hpp index f8756d9f6..4061f82d3 100644 --- a/crypto/common/bigint.hpp +++ b/crypto/common/bigint.hpp @@ -2294,11 +2294,11 @@ std::string AnyIntView::to_dec_string_destroy_any() { stack.push_back(divmod_short_any(Tr::max_pow10)); } while (sgn()); char slice[word_bits * 97879 / 325147 + 2]; - std::sprintf(slice, "%lld", stack.back()); + std::snprintf(slice, sizeof(slice), "%lld", stack.back()); s += slice; stack.pop_back(); while (stack.size()) { - std::sprintf(slice, "%018lld", stack.back()); + std::snprintf(slice, sizeof(slice), "%018lld", stack.back()); s += slice; stack.pop_back(); } diff --git a/crypto/common/refcnt.cpp b/crypto/common/refcnt.cpp index 7ef0857cc..c6d1b8884 100644 --- a/crypto/common/refcnt.cpp +++ b/crypto/common/refcnt.cpp @@ -29,6 +29,7 @@ Ref CntObject::clone() const { namespace detail { struct SafeDeleter { public: + thread_local static td::int64 delete_count; void retire(const CntObject *ptr) { if (is_active_) { to_delete_.push_back(ptr); @@ -39,9 +40,11 @@ struct SafeDeleter { is_active_ = false; }; delete ptr; + delete_count++; while (!to_delete_.empty()) { auto *ptr = to_delete_.back(); to_delete_.pop_back(); + delete_count++; delete ptr; } } @@ -50,6 +53,7 @@ struct SafeDeleter { std::vector to_delete_; bool is_active_{false}; }; +thread_local td::int64 SafeDeleter::delete_count{0}; TD_THREAD_LOCAL SafeDeleter *deleter; void safe_delete(const CntObject *ptr) { @@ -57,4 +61,7 @@ void safe_delete(const CntObject *ptr) { deleter->retire(ptr); } } // namespace detail +int64 ref_get_delete_count() { + return detail::SafeDeleter::delete_count; +} } // namespace td diff --git a/crypto/common/refcnt.hpp b/crypto/common/refcnt.hpp index ef50c3b9a..953cc7797 100644 --- a/crypto/common/refcnt.hpp +++ b/crypto/common/refcnt.hpp @@ -472,5 +472,6 @@ template void swap(Ref& r1, Ref& r2) { r1.swap(r2); } +int64 ref_get_delete_count(); } // namespace td diff --git a/crypto/fift/words.cpp b/crypto/fift/words.cpp index 324f492c0..cbe1dbcad 100644 --- a/crypto/fift/words.cpp +++ b/crypto/fift/words.cpp @@ -3425,7 +3425,7 @@ void import_cmdline_args(Dictionary& d, std::string arg0, int n, const char* con cmdline_args->set(std::move(list)); for (int i = 1; i <= n; i++) { char buffer[14]; - sprintf(buffer, "$%d ", i); + snprintf(buffer, sizeof(buffer), "$%d ", i); d.def_stack_word(buffer, std::bind(interpret_get_fixed_cmdline_arg, _1, i)); } } diff --git a/crypto/func/analyzer.cpp b/crypto/func/analyzer.cpp index fb05bbb4b..9d59cf9ad 100644 --- a/crypto/func/analyzer.cpp +++ b/crypto/func/analyzer.cpp @@ -81,7 +81,7 @@ bool CodeBlob::compute_used_code_vars(std::unique_ptr& ops_ptr, const VarDes func_assert(ops_ptr->cl == Op::_Nop); return ops_ptr->set_var_info(var_info); } - return compute_used_code_vars(ops_ptr->next, var_info, edit) | ops_ptr->compute_used_vars(*this, edit); + return int(compute_used_code_vars(ops_ptr->next, var_info, edit)) | int(ops_ptr->compute_used_vars(*this, edit)); } bool operator==(const VarDescrList& x, const VarDescrList& y) { @@ -584,7 +584,7 @@ bool prune_unreachable(std::unique_ptr& ops) { ops = std::move(op.block1); return prune_unreachable(ops); } else { - reach = prune_unreachable(op.block0) | prune_unreachable(op.block1); + reach = int(prune_unreachable(op.block0)) | int(prune_unreachable(op.block1)); } break; } @@ -660,7 +660,7 @@ bool prune_unreachable(std::unique_ptr& ops) { break; } case Op::_TryCatch: { - reach = prune_unreachable(op.block0) | prune_unreachable(op.block1); + reach = int(prune_unreachable(op.block0)) | int(prune_unreachable(op.block1)); break; } default: @@ -892,15 +892,15 @@ bool Op::mark_noreturn() { return set_noreturn(true); case _If: case _TryCatch: - return set_noreturn((block0->mark_noreturn() & (block1 && block1->mark_noreturn())) | next->mark_noreturn()); + return set_noreturn((int(block0->mark_noreturn()) & int(block1 && block1->mark_noreturn())) | int(next->mark_noreturn())); case _Again: block0->mark_noreturn(); return set_noreturn(true); case _Until: - return set_noreturn(block0->mark_noreturn() | next->mark_noreturn()); + return set_noreturn(int(block0->mark_noreturn()) | int(next->mark_noreturn())); case _While: block1->mark_noreturn(); - return set_noreturn(block0->mark_noreturn() | next->mark_noreturn()); + return set_noreturn(int(block0->mark_noreturn()) | int(next->mark_noreturn())); case _Repeat: block0->mark_noreturn(); return set_noreturn(next->mark_noreturn()); diff --git a/crypto/openssl/digest.hpp b/crypto/openssl/digest.hpp index 5c232df99..2adeef1d7 100644 --- a/crypto/openssl/digest.hpp +++ b/crypto/openssl/digest.hpp @@ -48,6 +48,7 @@ struct OpensslEVP_SHA512 { template class HashCtx { + EVP_MD_CTX *base_ctx{nullptr}; EVP_MD_CTX *ctx{nullptr}; void init(); void clear(); @@ -77,16 +78,20 @@ class HashCtx { template void HashCtx::init() { ctx = EVP_MD_CTX_create(); + base_ctx = EVP_MD_CTX_create(); + EVP_DigestInit_ex(base_ctx, H::get_evp(), 0); reset(); } template void HashCtx::reset() { - EVP_DigestInit_ex(ctx, H::get_evp(), 0); + EVP_MD_CTX_copy_ex(ctx, base_ctx); } template void HashCtx::clear() { + EVP_MD_CTX_destroy(base_ctx); + base_ctx = nullptr; EVP_MD_CTX_destroy(ctx); ctx = nullptr; } diff --git a/crypto/test/Ed25519.cpp b/crypto/test/Ed25519.cpp index 45e8891f4..131bfe92e 100644 --- a/crypto/test/Ed25519.cpp +++ b/crypto/test/Ed25519.cpp @@ -17,6 +17,8 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "crypto/Ed25519.h" +#include "ellcurve/Ed25519.h" + #include "td/utils/logging.h" #include "td/utils/misc.h" #include "td/utils/Slice.h" @@ -24,6 +26,8 @@ #include "td/utils/JsonBuilder.h" #include "wycheproof.h" +#include "keys/keys.hpp" +#include "td/utils/benchmark.h" #include #include @@ -217,3 +221,36 @@ TEST(Crypto, almost_zero) { } } } + +BENCH(ed25519_sign, "ed25519_sign") { + auto private_key = td::Ed25519::generate_private_key().move_as_ok(); + std::string hash_to_sign(32, 'a'); + for (int i = 0; i < n; i++) { + private_key.sign(hash_to_sign).ensure(); + } +} + +BENCH(ed25519_shared_secret, "ed25519_shared_secret") { + auto private_key_a = td::Ed25519::generate_private_key().move_as_ok(); + auto private_key_b = td::Ed25519::generate_private_key().move_as_ok(); + auto public_key_b = private_key_a.get_public_key().move_as_ok(); + for (int i = 0; i < n; i++) { + td::Ed25519::compute_shared_secret(public_key_b, private_key_a).ensure(); + } +} + +BENCH(ed25519_verify, "ed25519_verify") { + auto private_key = td::Ed25519::generate_private_key().move_as_ok(); + std::string hash_to_sign(32, 'a'); + auto public_key = private_key.get_public_key().move_as_ok(); + auto signature = private_key.sign(hash_to_sign).move_as_ok(); + for (int i = 0; i < n; i++) { + public_key.verify_signature(hash_to_sign, signature).ensure(); + } +} + +TEST(Crypto, ed25519_benchmark) { + bench(ed25519_signBench()); + bench(ed25519_shared_secretBench()); + bench(ed25519_verifyBench()); +} \ No newline at end of file diff --git a/crypto/test/test-db.cpp b/crypto/test/test-db.cpp index 35727ee36..d4059f051 100644 --- a/crypto/test/test-db.cpp +++ b/crypto/test/test-db.cpp @@ -54,10 +54,15 @@ #include #include +#include #include #include "openssl/digest.hpp" +#include "vm/dict.h" + +#include +#include namespace vm { @@ -82,9 +87,23 @@ int get_random_serialization_mode(T &rnd) { return modes[rnd.fast(0, (int)modes.size() - 1)]; } -class BenchSha256 : public td::Benchmark { +class BenchSha : public td::Benchmark { public: + explicit BenchSha(size_t n) : str_(n, 'a') { + } std::string get_description() const override { + return PSTRING() << get_name() << " length=" << str_.size(); + } + + virtual std::string get_name() const = 0; + + protected: + std::string str_; +}; +class BenchSha256 : public BenchSha { + public: + using BenchSha::BenchSha; + std::string get_name() const override { return "SHA256"; } @@ -92,7 +111,7 @@ class BenchSha256 : public td::Benchmark { int res = 0; for (int i = 0; i < n; i++) { digest::SHA256 hasher; - hasher.feed("abcd", 4); + hasher.feed(str_); unsigned char buf[32]; hasher.extract(buf); res += buf[0]; @@ -100,10 +119,12 @@ class BenchSha256 : public td::Benchmark { td::do_not_optimize_away(res); } }; -class BenchSha256Reuse : public td::Benchmark { +class BenchSha256Reuse : public BenchSha { public: - std::string get_description() const override { - return "SHA256 reuse"; + using BenchSha::BenchSha; + + std::string get_name() const override { + return "SHA256 reuse (used in DataCell)"; } void run(int n) override { @@ -111,7 +132,7 @@ class BenchSha256Reuse : public td::Benchmark { digest::SHA256 hasher; for (int i = 0; i < n; i++) { hasher.reset(); - hasher.feed("abcd", 4); + hasher.feed(str_); unsigned char buf[32]; hasher.extract(buf); res += buf[0]; @@ -119,28 +140,46 @@ class BenchSha256Reuse : public td::Benchmark { td::do_not_optimize_away(res); } }; -class BenchSha256Low : public td::Benchmark { +class BenchSha256Low : public BenchSha { public: - std::string get_description() const override { + using BenchSha::BenchSha; + + std::string get_name() const override { return "SHA256 low level"; } +// Use the old method to check for performance degradation +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#elif defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4996) // Disable deprecated warning for MSVC +#endif void run(int n) override { int res = 0; - td::Sha256State ctx; + SHA256_CTX ctx; for (int i = 0; i < n; i++) { - ctx.init(); - ctx.feed("abcd"); + SHA256_Init(&ctx); + SHA256_Update(&ctx, str_.data(), str_.size()); unsigned char buf[32]; - ctx.extract(td::MutableSlice{buf, 32}); + SHA256_Final(buf, &ctx); res += buf[0]; } td::do_not_optimize_away(res); } }; -class BenchSha256Tdlib : public td::Benchmark { +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic pop +#elif defined(_MSC_VER) +#pragma warning(pop) +#endif + +class BenchSha256Tdlib : public BenchSha { public: - std::string get_description() const override { + using BenchSha::BenchSha; + + std::string get_name() const override { return "SHA256 TDLib"; } @@ -150,7 +189,7 @@ class BenchSha256Tdlib : public td::Benchmark { for (int i = 0; i < n; i++) { td::init_thread_local(ctx); ctx->init(); - ctx->feed("abcd"); + ctx->feed(str_); unsigned char buf[32]; ctx->extract(td::MutableSlice(buf, 32), false); res += buf[0]; @@ -158,11 +197,61 @@ class BenchSha256Tdlib : public td::Benchmark { td::do_not_optimize_away(res); } }; + +template +void bench_threaded(F &&f) { + class Threaded : public td::Benchmark { + public: + explicit Threaded(F &&f) : f_(std::move(f)), base(f_()) { + } + F f_; + std::decay_t base; + + std::string get_description() const override { + return base.get_description() + " threaded"; + } + + void run(int n) override { + std::atomic task_i{0}; + int chunk_size = 1024; + int num_threads = 16; + n *= num_threads; + std::vector threads; + for (int i = 0; i < num_threads; i++) { + threads.emplace_back([&]() mutable { + auto bench = f_(); + while (true) { + i = task_i.fetch_add(chunk_size, std::memory_order_relaxed); + auto i_end = std::min(n, i + chunk_size); + if (i > n) { + break; + } + bench.run(i_end - i); + } + }); + } + for (auto &thread : threads) { + thread.join(); + } + }; + }; + bench(Threaded(std::forward(f))); +} TEST(Cell, sha_benchmark) { - bench(BenchSha256Tdlib()); - bench(BenchSha256Low()); - bench(BenchSha256Reuse()); - bench(BenchSha256()); + for (size_t n : {4, 64, 128}) { + bench(BenchSha256Tdlib(n)); + bench(BenchSha256Low(n)); + bench(BenchSha256Reuse(n)); + bench(BenchSha256(n)); + } +} +TEST(Cell, sha_benchmark_threaded) { + for (size_t n : {4, 64, 128}) { + bench_threaded([n] { return BenchSha256Tdlib(n); }); + bench_threaded([n]() { return BenchSha256Low(n); }); + bench_threaded([n]() { return BenchSha256Reuse(n); }); + bench_threaded([n]() { return BenchSha256(n); }); + } } std::string serialize_boc(Ref cell, int mode = 31) { @@ -762,16 +851,70 @@ TEST(TonDb, BocMultipleRoots) { } }; -TEST(TonDb, DynamicBoc) { +TEST(TonDb, InMemoryDynamicBocSimple) { + auto counter = [] { return td::NamedThreadSafeCounter::get_default().get_counter("DataCell").sum(); }; + auto before = counter(); + SCOPE_EXIT { + LOG_CHECK(before == counter()) << before << " vs " << counter(); + ; + }; + td::Random::Xorshift128plus rnd{123}; + auto kv = std::make_shared(); + CellStorer storer(*kv); + + auto boc = DynamicBagOfCellsDb::create_in_memory(kv.get(), {}); + + auto empty_cell = vm::CellBuilder().finalize(); + boc->inc(empty_cell); + boc->prepare_commit().ensure(); + boc->commit(storer).ensure(); + auto got_empty_cell = boc->load_cell(empty_cell->get_hash().as_slice()).move_as_ok(); + ASSERT_EQ(empty_cell->get_hash(), got_empty_cell->get_hash()); + + boc->dec(empty_cell); + + auto one_ref_cell = vm::CellBuilder().store_ref(empty_cell).finalize(); + boc->inc(one_ref_cell); + boc->prepare_commit().ensure(); + boc->commit(storer).ensure(); + auto got_one_ref_cell = boc->load_cell(one_ref_cell->get_hash().as_slice()).move_as_ok(); + ASSERT_EQ(one_ref_cell->get_hash(), got_one_ref_cell->get_hash()); + boc = DynamicBagOfCellsDb::create_in_memory(kv.get(), {}); + + auto random_ref_cell = gen_random_cell(3, rnd); + boc->inc(random_ref_cell); + boc->prepare_commit().ensure(); + boc->commit(storer).ensure(); + auto got_random_ref_cell = boc->load_cell(random_ref_cell->get_hash().as_slice()).move_as_ok(); + ASSERT_EQ(random_ref_cell->get_hash(), got_random_ref_cell->get_hash()); + boc = DynamicBagOfCellsDb::create_in_memory(kv.get(), {}); +} + +void test_dynamic_boc(std::optional o_in_memory) { + auto counter = [] { return td::NamedThreadSafeCounter::get_default().get_counter("DataCell").sum(); }; + auto before = counter(); + SCOPE_EXIT { + LOG_CHECK((o_in_memory && o_in_memory->use_arena) || before == counter()) << before << " vs " << counter(); + ; + }; td::Random::Xorshift128plus rnd{123}; std::string old_root_hash; std::string old_root_serialization; auto kv = std::make_shared(); - auto dboc = DynamicBagOfCellsDb::create(); + auto create_dboc = [&]() { + if (o_in_memory) { + auto res = DynamicBagOfCellsDb::create_in_memory(kv.get(), *o_in_memory); + auto roots_n = old_root_hash.empty() ? 0 : 1; + ASSERT_EQ(roots_n, res->get_stats().ok().roots_total_count); + return res; + } + return DynamicBagOfCellsDb::create(); + }; + auto dboc = create_dboc(); dboc->set_loader(std::make_unique(kv)); for (int t = 1000; t >= 0; t--) { if (rnd() % 10 == 0) { - dboc = DynamicBagOfCellsDb::create(); + dboc = create_dboc(); } dboc->set_loader(std::make_unique(kv)); Ref old_root; @@ -795,29 +938,64 @@ TEST(TonDb, DynamicBoc) { if (t != 0) { dboc->inc(cell); } - dboc->prepare_commit(); + dboc->prepare_commit().ensure(); { CellStorer cell_storer(*kv); - dboc->commit(cell_storer); + dboc->commit(cell_storer).ensure(); } } ASSERT_EQ(0u, kv->count("").ok()); +} + +template +void with_all_boc_options(F &&f) { + LOG(INFO) << "Test dynamic boc"; + LOG(INFO) << "\ton disk"; + f({}); + for (auto use_arena : {false, true}) { + for (auto less_memory : {false, true}) { + LOG(INFO) << "\tuse_arena=" << use_arena << " less_memory=" << less_memory; + f(DynamicBagOfCellsDb::CreateInMemoryOptions{.extra_threads = std::thread::hardware_concurrency(), + .verbose = false, + .use_arena = use_arena, + .use_less_memory_during_creation = less_memory}); + } + } +} +TEST(TonDb, DynamicBoc) { + with_all_boc_options(test_dynamic_boc); }; -TEST(TonDb, DynamicBoc2) { +void test_dynamic_boc2(std::optional o_in_memory) { int VERBOSITY_NAME(boc) = VERBOSITY_NAME(DEBUG) + 10; td::Random::Xorshift128plus rnd{123}; int total_roots = 10000; int max_roots = 20; - std::vector root_hashes(max_roots); - std::vector> roots(max_roots); int last_commit_at = 0; int first_root_id = 0; int last_root_id = 0; auto kv = std::make_shared(); - auto dboc = DynamicBagOfCellsDb::create(); + auto create_dboc = [&](td::int64 root_n) { + if (o_in_memory) { + auto res = DynamicBagOfCellsDb::create_in_memory(kv.get(), *o_in_memory); + auto stats = res->get_stats().move_as_ok(); + ASSERT_EQ(root_n, stats.roots_total_count); + VLOG(boc) << "reset roots_n=" << stats.roots_total_count << " cells_n=" << stats.cells_total_count; + return res; + } + return DynamicBagOfCellsDb::create(); + }; + auto dboc = create_dboc(0); dboc->set_loader(std::make_unique(kv)); + auto counter = [] { return td::NamedThreadSafeCounter::get_default().get_counter("DataCell").sum(); }; + auto before = counter(); + SCOPE_EXIT { + LOG_CHECK((o_in_memory && o_in_memory->use_arena) || before == counter()) << before << " vs " << counter(); + }; + + std::vector> roots(max_roots); + std::vector root_hashes(max_roots); auto add_root = [&](Ref root) { dboc->inc(root); root_hashes[last_root_id % max_roots] = (root->get_hash().as_slice().str()); @@ -825,18 +1003,23 @@ TEST(TonDb, DynamicBoc2) { last_root_id++; }; - auto get_root = [&](int root_id) { + auto get_root = [&](int root_id) -> Ref { VLOG(boc) << " from older root #" << root_id; auto from_root = roots[root_id % max_roots]; if (from_root.is_null()) { VLOG(boc) << " from db"; auto from_root_hash = root_hashes[root_id % max_roots]; - from_root = dboc->load_cell(from_root_hash).move_as_ok(); + if (o_in_memory && (rnd() % 2 == 0)) { + from_root = dboc->load_root(from_root_hash).move_as_ok(); + } else { + from_root = dboc->load_cell(from_root_hash).move_as_ok(); + } } else { VLOG(boc) << "FROM MEMORY"; } return from_root; }; + std::map root_cnt; auto new_root = [&] { if (last_root_id == total_roots) { return; @@ -850,7 +1033,9 @@ TEST(TonDb, DynamicBoc2) { from_root = get_root(rnd.fast(first_root_id, last_root_id - 1)); } VLOG(boc) << " ..."; - add_root(gen_random_cell(rnd.fast(1, 20), from_root, rnd)); + auto new_root = gen_random_cell(rnd.fast(1, 20), from_root, rnd); + root_cnt[new_root->get_hash()]++; + add_root(std::move(new_root)); VLOG(boc) << " OK"; }; @@ -870,7 +1055,7 @@ TEST(TonDb, DynamicBoc2) { auto reset = [&] { VLOG(boc) << "reset"; commit(); - dboc = DynamicBagOfCellsDb::create(); + dboc = create_dboc(td::int64(root_cnt.size())); dboc->set_loader(std::make_unique(kv)); }; @@ -879,7 +1064,15 @@ TEST(TonDb, DynamicBoc2) { if (first_root_id == last_root_id) { return; } - dboc->dec(get_root(first_root_id)); + auto old_root = get_root(first_root_id); + auto it = root_cnt.find(old_root->get_hash()); + it->second--; + CHECK(it->second >= 0); + if (it->second == 0) { + root_cnt.erase(it); + } + + dboc->dec(std::move(old_root)); first_root_id++; VLOG(boc) << " OK"; }; @@ -893,6 +1086,10 @@ TEST(TonDb, DynamicBoc2) { ASSERT_EQ(0u, kv->count("").ok()); } +TEST(TonDb, DynamicBoc2) { + with_all_boc_options(test_dynamic_boc2); +} + template td::Status test_boc_deserializer(std::vector> cells, int mode) { auto total_data_cells_before = vm::DataCell::get_total_data_cells(); @@ -1848,7 +2045,7 @@ TEST(TonDb, CompactArrayOld) { SCOPE_EXIT { ton_db->commit_transaction(std::move(txn)); }; - auto smart = txn->begin_smartcontract(""); + auto smart = txn->begin_smartcontract(); SCOPE_EXIT { txn->commit_smartcontract(std::move(smart)); }; @@ -1875,7 +2072,7 @@ TEST(TonDb, CompactArrayOld) { SCOPE_EXIT { ton_db->commit_transaction(std::move(txn)); }; - auto smart = txn->begin_smartcontract(""); + auto smart = txn->begin_smartcontract(); //smart->validate_meta(); SCOPE_EXIT { txn->commit_smartcontract(std::move(smart)); @@ -1896,7 +2093,7 @@ TEST(TonDb, CompactArrayOld) { SCOPE_EXIT { ton_db->abort_transaction(std::move(txn)); }; - auto smart = txn->begin_smartcontract(""); + auto smart = txn->begin_smartcontract(); SCOPE_EXIT { txn->abort_smartcontract(std::move(smart)); }; @@ -1950,14 +2147,15 @@ TEST(TonDb, BocRespectsUsageCell) { ASSERT_STREQ(serialization, serialization_of_virtualized_cell); } -TEST(TonDb, DynamicBocRespectsUsageCell) { +void test_dynamic_boc_respectes_usage_cell(std::optional o_in_memory) { td::Random::Xorshift128plus rnd(123); auto cell = vm::gen_random_cell(20, rnd, true); auto usage_tree = std::make_shared(); auto usage_cell = vm::UsageCell::create(cell, usage_tree->root_ptr()); auto kv = std::make_shared(); - auto dboc = vm::DynamicBagOfCellsDb::create(); + auto dboc = o_in_memory ? vm::DynamicBagOfCellsDb::create_in_memory(kv.get(), *o_in_memory) + : vm::DynamicBagOfCellsDb::create(); dboc->set_loader(std::make_unique(kv)); dboc->inc(usage_cell); { @@ -1972,6 +2170,42 @@ TEST(TonDb, DynamicBocRespectsUsageCell) { ASSERT_STREQ(serialization, serialization_of_virtualized_cell); } +TEST(TonDb, DynamicBocRespectsUsageCell) { + vm::with_all_boc_options(test_dynamic_boc_respectes_usage_cell); +} + +TEST(TonDb, LargeBocSerializer) { + td::Random::Xorshift128plus rnd{123}; + size_t n = 1000000; + std::vector data(n); + std::iota(data.begin(), data.end(), 0); + vm::CompactArray arr(data); + auto root = arr.root(); + std::string path = "serialization"; + td::unlink(path).ignore(); + auto fd = td::FileFd::open(path, td::FileFd::Flags::Create | td::FileFd::Flags::Truncate | td::FileFd::Flags::Write) + .move_as_ok(); + std_boc_serialize_to_file(root, fd, 31); + fd.close(); + auto a = td::read_file_str(path).move_as_ok(); + + auto kv = std::make_shared(); + auto dboc = vm::DynamicBagOfCellsDb::create(); + dboc->set_loader(std::make_unique(kv)); + dboc->inc(root); + dboc->prepare_commit(); + vm::CellStorer cell_storer(*kv); + dboc->commit(cell_storer); + dboc->set_loader(std::make_unique(kv)); + td::unlink(path).ignore(); + fd = td::FileFd::open(path, td::FileFd::Flags::Create | td::FileFd::Flags::Truncate | td::FileFd::Flags::Write) + .move_as_ok(); + std_boc_serialize_to_file_large(dboc->get_cell_db_reader(), root->get_hash(), fd, 31); + fd.close(); + auto b = td::read_file_str(path).move_as_ok(); + CHECK(a == b); +} + TEST(TonDb, DoNotMakeListsPrunned) { auto cell = vm::CellBuilder().store_bytes("abc").finalize(); auto is_prunned = [&](const td::Ref &cell) { return true; }; @@ -2020,7 +2254,7 @@ TEST(TonDb, CellStat) { ASSERT_EQ(stat.cells, new_stat.get_stat().cells); ASSERT_EQ(stat.bits, new_stat.get_stat().bits); - CHECK(usage_tree.unique()); + CHECK(usage_tree.use_count() == 1); usage_tree.reset(); td::Ref C, BC, C_proof; std::shared_ptr usage_tree_B; @@ -2057,7 +2291,6 @@ TEST(Ref, AtomicRef) { int threads_n = 10; std::vector nodes(threads_n); std::vector threads(threads_n); - int thread_id = 0; for (auto &thread : threads) { thread = td::thread([&] { for (int i = 0; i < 1000000; i++) { @@ -2072,7 +2305,6 @@ TEST(Ref, AtomicRef) { } } }); - thread_id++; } for (auto &thread : threads) { thread.join(); diff --git a/crypto/tl/tlbc-gen-cpp.cpp b/crypto/tl/tlbc-gen-cpp.cpp index 6edd0a121..55b4a1c05 100644 --- a/crypto/tl/tlbc-gen-cpp.cpp +++ b/crypto/tl/tlbc-gen-cpp.cpp @@ -1316,7 +1316,7 @@ void CppTypeCode::clear_context() { std::string CppTypeCode::new_tmp_var() { char buffer[16]; while (true) { - sprintf(buffer, "t%d", ++tmp_ints); + snprintf(buffer, sizeof(buffer), "t%d", ++tmp_ints); if (tmp_cpp_ids.is_good_ident(buffer) && local_cpp_ids.is_good_ident(buffer)) { break; } diff --git a/crypto/tl/tlbc.cpp b/crypto/tl/tlbc.cpp index b48bc472e..0050e1610 100644 --- a/crypto/tl/tlbc.cpp +++ b/crypto/tl/tlbc.cpp @@ -420,7 +420,7 @@ void AdmissibilityInfo::operator|=(const AdmissibilityInfo& other) { std::size_t i, j, n = info.size(), n1 = other.info.size(); assert(n1 && !(n1 & (n1 - 1))); for (i = j = 0; i < n; i++) { - info[i] = info[i] | other.info[j]; + info[i] = info[i] || other.info[j]; j = (j + 1) & (n1 - 1); } } @@ -2511,7 +2511,7 @@ void define_builtins() { Bits_type = define_builtin_type("bits", "#", false, 1023, 0, true, 0); for (int i = 1; i <= 257; i++) { char buff[8]; - sprintf(buff, "uint%d", i); + snprintf(buff, sizeof(buff), "uint%d", i); define_builtin_type(buff + 1, "", false, i, i, true, -1); if (i < 257) { define_builtin_type(buff, "", false, i, i, true, 1); @@ -2519,7 +2519,7 @@ void define_builtins() { } for (int i = 1; i <= 1023; i++) { char buff[12]; - sprintf(buff, "bits%d", i); + snprintf(buff, sizeof(buff), "bits%d", i); define_builtin_type(buff, "", false, i, i, true, 0); } Eq_type = define_builtin_type("=", "##", false, 0, 0, true); diff --git a/crypto/vm/atom.cpp b/crypto/vm/atom.cpp index a7d5486a1..dbf1b16f5 100644 --- a/crypto/vm/atom.cpp +++ b/crypto/vm/atom.cpp @@ -35,7 +35,7 @@ void Atom::print_to(std::ostream& os) const { std::string Atom::make_name() const { char buffer[16]; - sprintf(buffer, "atom#%d", index_); + snprintf(buffer, sizeof(buffer), "atom#%d", index_); return buffer; } diff --git a/crypto/vm/boc.cpp b/crypto/vm/boc.cpp index 3e15f62b6..7ec8bdd1d 100644 --- a/crypto/vm/boc.cpp +++ b/crypto/vm/boc.cpp @@ -183,6 +183,9 @@ int BagOfCells::add_root(td::Ref add_root) { // Changes in this function may require corresponding changes in crypto/vm/large-boc-serializer.cpp td::Status BagOfCells::import_cells() { + if (logger_ptr_) { + logger_ptr_->start_stage("import_cells"); + } cells_clear(); for (auto& root : roots) { auto res = import_cell(root.cell, 0); @@ -196,6 +199,9 @@ td::Status BagOfCells::import_cells() { //LOG(INFO) << "[cells: " << cell_count << ", refs: " << int_refs << ", bytes: " << data_bytes //<< ", internal hashes: " << int_hashes << ", top hashes: " << top_hashes << "]"; CHECK(cell_count != 0); + if (logger_ptr_) { + logger_ptr_->finish_stage(PSLICE() << cell_count << " cells"); + } return td::Status::OK(); } @@ -207,6 +213,9 @@ td::Result BagOfCells::import_cell(td::Ref cell, int depth) { if (cell.is_null()) { return td::Status::Error("error while importing a cell into a bag of cells: cell is null"); } + if (logger_ptr_) { + TRY_STATUS(logger_ptr_->on_cell_processed()); + } auto it = cells.find(cell->get_hash()); if (it != cells.end()) { auto pos = it->second; @@ -436,17 +445,19 @@ std::size_t BagOfCells::estimate_serialized_size(int mode) { return res.ok(); } -BagOfCells& BagOfCells::serialize(int mode) { +td::Status BagOfCells::serialize(int mode) { std::size_t size_est = estimate_serialized_size(mode); if (!size_est) { serialized.clear(); - return *this; + return td::Status::OK(); } serialized.resize(size_est); - if (serialize_to(const_cast(serialized.data()), serialized.size(), mode) != size_est) { + TRY_RESULT(size, serialize_to(const_cast(serialized.data()), serialized.size(), mode)); + if (size != size_est) { serialized.clear(); + return td::Status::Error("serialization failed"); } - return *this; + return td::Status::OK(); } std::string BagOfCells::serialize_to_string(int mode) { @@ -456,8 +467,8 @@ std::string BagOfCells::serialize_to_string(int mode) { } std::string res; res.resize(size_est, 0); - if (serialize_to(const_cast(reinterpret_cast(res.data())), res.size(), mode) == - res.size()) { + if (serialize_to(const_cast(reinterpret_cast(res.data())), res.size(), mode) + .move_as_ok() == res.size()) { return res; } else { return {}; @@ -470,8 +481,9 @@ td::Result BagOfCells::serialize_to_slice(int mode) { return td::Status::Error("no cells to serialize to this bag of cells"); } td::BufferSlice res(size_est); - if (serialize_to(const_cast(reinterpret_cast(res.data())), res.size(), mode) == - res.size()) { + TRY_RESULT(size, serialize_to(const_cast(reinterpret_cast(res.data())), + res.size(), mode)); + if (size == res.size()) { return std::move(res); } else { return td::Status::Error("error while serializing a bag of cells: actual serialized size differs from estimated"); @@ -494,14 +506,10 @@ std::string BagOfCells::extract_string() const { // cell_data:(tot_cells_size * [ uint8 ]) // = BagOfCells; // Changes in this function may require corresponding changes in crypto/vm/large-boc-serializer.cpp -template -std::size_t BagOfCells::serialize_to_impl(WriterT& writer, int mode) { - auto store_ref = [&](unsigned long long value) { - writer.store_uint(value, info.ref_byte_size); - }; - auto store_offset = [&](unsigned long long value) { - writer.store_uint(value, info.offset_byte_size); - }; +template +td::Result BagOfCells::serialize_to_impl(WriterT& writer, int mode) { + auto store_ref = [&](unsigned long long value) { writer.store_uint(value, info.ref_byte_size); }; + auto store_offset = [&](unsigned long long value) { writer.store_uint(value, info.offset_byte_size); }; writer.store_uint(info.magic, 4); @@ -536,6 +544,9 @@ std::size_t BagOfCells::serialize_to_impl(WriterT& writer, int mode) { DCHECK((unsigned)cell_count == cell_list_.size()); if (info.has_index) { std::size_t offs = 0; + if (logger_ptr_) { + logger_ptr_->start_stage("generate_index"); + } for (int i = cell_count - 1; i >= 0; --i) { const Ref& dc = cell_list_[i].dc_ref; bool with_hash = (mode & Mode::WithIntHashes) && !cell_list_[i].wt; @@ -548,11 +559,20 @@ std::size_t BagOfCells::serialize_to_impl(WriterT& writer, int mode) { fixed_offset = offs * 2 + cell_list_[i].should_cache; } store_offset(fixed_offset); + if (logger_ptr_) { + TRY_STATUS(logger_ptr_->on_cell_processed()); + } + } + if (logger_ptr_) { + logger_ptr_->finish_stage(""); } DCHECK(offs == info.data_size); } DCHECK(writer.position() == info.data_offset); size_t keep_position = writer.position(); + if (logger_ptr_) { + logger_ptr_->start_stage("serialize"); + } for (int i = 0; i < cell_count; ++i) { const auto& dc_info = cell_list_[cell_count - 1 - i]; const Ref& dc = dc_info.dc_ref; @@ -572,6 +592,9 @@ std::size_t BagOfCells::serialize_to_impl(WriterT& writer, int mode) { // std::cerr << ' ' << k; } // std::cerr << std::endl; + if (logger_ptr_) { + TRY_STATUS(logger_ptr_->on_cell_processed()); + } } writer.chk(); DCHECK(writer.position() - keep_position == info.data_size); @@ -580,11 +603,14 @@ std::size_t BagOfCells::serialize_to_impl(WriterT& writer, int mode) { unsigned crc = writer.get_crc32(); writer.store_uint(td::bswap32(crc), 4); } + if (logger_ptr_) { + logger_ptr_->finish_stage(PSLICE() << cell_count << " cells, " << writer.position() << " bytes"); + } DCHECK(writer.empty()); return writer.position(); } -std::size_t BagOfCells::serialize_to(unsigned char* buffer, std::size_t buff_size, int mode) { +td::Result BagOfCells::serialize_to(unsigned char* buffer, std::size_t buff_size, int mode) { std::size_t size_est = estimate_serialized_size(mode); if (!size_est || size_est > buff_size) { return 0; @@ -599,7 +625,7 @@ td::Status BagOfCells::serialize_to_file(td::FileFd& fd, int mode) { return td::Status::Error("no cells to serialize to this bag of cells"); } boc_writers::FileWriter writer{fd, size_est}; - size_t s = serialize_to_impl(writer, mode); + TRY_RESULT(s, serialize_to_impl(writer, mode)); TRY_STATUS(writer.finalize()); if (s != size_est) { return td::Status::Error("error while serializing a bag of cells: actual serialized size differs from estimated"); @@ -1001,6 +1027,21 @@ td::Result std_boc_serialize_multi(std::vector> roots } return boc.serialize_to_slice(mode); } +td::Status std_boc_serialize_to_file(Ref root, td::FileFd& fd, int mode, + td::CancellationToken cancellation_token) { + if (root.is_null()) { + return td::Status::Error("cannot serialize a null cell reference into a bag of cells"); + } + td::Timer timer; + BagOfCellsLogger logger(std::move(cancellation_token)); + BagOfCells boc; + boc.set_logger(&logger); + boc.add_root(std::move(root)); + TRY_STATUS(boc.import_cells()); + TRY_STATUS(boc.serialize_to_file(fd, mode)); + LOG(ERROR) << "serialization took " << timer.elapsed() << "s"; + return td::Status::OK(); +} /* * diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index bdd0a06f1..8adf240fb 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -27,6 +27,8 @@ #include "td/utils/buffer.h" #include "td/utils/HashMap.h" #include "td/utils/HashSet.h" +#include "td/utils/Time.h" +#include "td/utils/Timer.h" #include "td/utils/port/FileFd.h" namespace vm { @@ -199,6 +201,43 @@ struct CellSerializationInfo { td::Result> create_data_cell(td::Slice data, td::Span> refs) const; }; +class BagOfCellsLogger { + public: + BagOfCellsLogger() = default; + explicit BagOfCellsLogger(td::CancellationToken cancellation_token) + : cancellation_token_(std::move(cancellation_token)) { + } + + void start_stage(std::string stage) { + log_speed_at_ = td::Timestamp::in(LOG_SPEED_PERIOD); + processed_cells_ = 0; + timer_ = {}; + stage_ = std::move(stage); + } + void finish_stage(td::Slice desc) { + LOG(ERROR) << "serializer: " << stage_ << " took " << timer_.elapsed() << "s, " << desc; + } + td::Status on_cell_processed() { + ++processed_cells_; + if (processed_cells_ % 1000 == 0) { + TRY_STATUS(cancellation_token_.check()); + } + if (log_speed_at_.is_in_past()) { + log_speed_at_ += LOG_SPEED_PERIOD; + LOG(WARNING) << "serializer: " << stage_ << " " << (double)processed_cells_ / LOG_SPEED_PERIOD << " cells/s"; + processed_cells_ = 0; + } + return td::Status::OK(); + } + + private: + std::string stage_; + td::Timer timer_; + td::CancellationToken cancellation_token_; + td::Timestamp log_speed_at_; + size_t processed_cells_ = 0; + static constexpr double LOG_SPEED_PERIOD = 120.0; +}; class BagOfCells { public: enum { hash_bytes = vm::Cell::hash_bytes, default_max_roots = 16384 }; @@ -283,6 +322,7 @@ class BagOfCells { const unsigned char* index_ptr{nullptr}; const unsigned char* data_ptr{nullptr}; std::vector custom_index; + BagOfCellsLogger* logger_ptr_{nullptr}; public: void clear(); @@ -292,14 +332,17 @@ class BagOfCells { int add_root(td::Ref add_root); td::Status import_cells() TD_WARN_UNUSED_RESULT; BagOfCells() = default; + void set_logger(BagOfCellsLogger* logger_ptr) { + logger_ptr_ = logger_ptr; + } std::size_t estimate_serialized_size(int mode = 0); - BagOfCells& serialize(int mode = 0); - std::string serialize_to_string(int mode = 0); + td::Status serialize(int mode = 0); + td::string serialize_to_string(int mode = 0); td::Result serialize_to_slice(int mode = 0); - std::size_t serialize_to(unsigned char* buffer, std::size_t buff_size, int mode = 0); + td::Result serialize_to(unsigned char* buffer, std::size_t buff_size, int mode = 0); td::Status serialize_to_file(td::FileFd& fd, int mode = 0); - template - std::size_t serialize_to_impl(WriterT& writer, int mode = 0); + template + td::Result serialize_to_impl(WriterT& writer, int mode = 0); std::string extract_string() const; td::Result deserialize(const td::Slice& data, int max_roots = default_max_roots); @@ -345,6 +388,8 @@ td::Result>> std_boc_deserialize_multi(td::Slice data, int max_roots = BagOfCells::default_max_roots); td::Result std_boc_serialize_multi(std::vector> root, int mode = 0); +td::Status std_boc_serialize_to_file(Ref root, td::FileFd& fd, int mode = 0, + td::CancellationToken cancellation_token = {}); td::Status std_boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, td::FileFd& fd, int mode = 0, td::CancellationToken cancellation_token = {}); diff --git a/crypto/vm/cells/Cell.h b/crypto/vm/cells/Cell.h index 03f73b14e..a75371dbb 100644 --- a/crypto/vm/cells/Cell.h +++ b/crypto/vm/cells/Cell.h @@ -19,6 +19,7 @@ #pragma once #include "common/refcnt.hpp" #include "common/bitstring.h" +#include "td/utils/HashSet.h" #include "vm/cells/CellHash.h" #include "vm/cells/CellTraits.h" @@ -86,4 +87,31 @@ class Cell : public CellTraits { }; std::ostream& operator<<(std::ostream& os, const Cell& c); + +using is_transparent = void; // Pred to use +inline vm::CellHash as_cell_hash(const Ref& cell) { + return cell->get_hash(); +} +inline vm::CellHash as_cell_hash(td::Slice hash) { + return vm::CellHash::from_slice(hash); +} +inline vm::CellHash as_cell_hash(vm::CellHash hash) { + return hash; +} +struct CellEqF { + using is_transparent = void; // Pred to use + template + bool operator()(const A& a, const B& b) const { + return as_cell_hash(a) == as_cell_hash(b); + } +}; +struct CellHashF { + using is_transparent = void; // Pred to use + using transparent_key_equal = CellEqF; + template + size_t operator()(const T& value) const { + return cell_hash_slice_hash(as_cell_hash(value).as_slice()); + } +}; +using CellHashSet = td::HashSet, CellHashF, CellEqF>; } // namespace vm diff --git a/crypto/vm/cells/CellBuilder.cpp b/crypto/vm/cells/CellBuilder.cpp index 43058531e..772b5f6b7 100644 --- a/crypto/vm/cells/CellBuilder.cpp +++ b/crypto/vm/cells/CellBuilder.cpp @@ -617,7 +617,7 @@ std::string CellBuilder::to_hex() const { int len = serialize(buff, sizeof(buff)); char hex_buff[Cell::max_serialized_bytes * 2 + 1]; for (int i = 0; i < len; i++) { - sprintf(hex_buff + 2 * i, "%02x", buff[i]); + snprintf(hex_buff + 2 * i, sizeof(hex_buff) - 2 * i, "%02x", buff[i]); } return hex_buff; } diff --git a/crypto/vm/cells/CellHash.h b/crypto/vm/cells/CellHash.h index 2cbc0e8b3..9435675f1 100644 --- a/crypto/vm/cells/CellHash.h +++ b/crypto/vm/cells/CellHash.h @@ -74,13 +74,17 @@ struct CellHash { }; } // namespace vm +inline size_t cell_hash_slice_hash(td::Slice hash) { + // use offset 8, because in db keys are grouped by first bytes. + return td::as(hash.substr(8, 8).ubegin()); +} namespace std { template <> struct hash { typedef vm::CellHash argument_type; typedef std::size_t result_type; result_type operator()(argument_type const& s) const noexcept { - return td::as(s.as_slice().ubegin()); + return cell_hash_slice_hash(s.as_slice()); } }; } // namespace std diff --git a/crypto/vm/cells/CellSlice.cpp b/crypto/vm/cells/CellSlice.cpp index 4be1667e7..466bcd8d1 100644 --- a/crypto/vm/cells/CellSlice.cpp +++ b/crypto/vm/cells/CellSlice.cpp @@ -976,7 +976,7 @@ void CellSlice::dump(std::ostream& os, int level, bool endl) const { os << "; refs: " << refs_st << ".." << refs_en; if (level > 2) { char tmp[64]; - std::sprintf(tmp, "; ptr=data+%ld; z=%016llx", + std::snprintf(tmp, sizeof(tmp), "; ptr=data+%ld; z=%016llx", static_cast(ptr && cell.not_null() ? ptr - cell->get_data() : -1), static_cast(z)); os << tmp << " (have " << size() << " bits; " << zd << " preloaded)"; } diff --git a/crypto/vm/cells/CellWithStorage.h b/crypto/vm/cells/CellWithStorage.h index 1830c37d2..f3fedfc31 100644 --- a/crypto/vm/cells/CellWithStorage.h +++ b/crypto/vm/cells/CellWithStorage.h @@ -20,6 +20,15 @@ namespace vm { namespace detail { + +template +struct DefaultAllocator { + template + std::unique_ptr make_unique(ArgsT&&... args) { + return std::make_unique(std::forward(args)...); + } +}; + template class CellWithArrayStorage : public CellT { public: @@ -29,14 +38,14 @@ class CellWithArrayStorage : public CellT { ~CellWithArrayStorage() { CellT::destroy_storage(get_storage()); } - template - static std::unique_ptr create(size_t storage_size, ArgsT&&... args) { + template + static auto create(Allocator allocator, size_t storage_size, ArgsT&&... args) { static_assert(CellT::max_storage_size <= 40 * 8, ""); //size = 128 + 32 + 8; auto size = (storage_size + 7) / 8; #define CASE(size) \ case (size): \ - return std::make_unique>(std::forward(args)...); + return allocator. template make_unique>(std::forward(args)...); #define CASE2(offset) CASE(offset) CASE(offset + 1) #define CASE8(offset) CASE2(offset) CASE2(offset + 2) CASE2(offset + 4) CASE2(offset + 6) #define CASE32(offset) CASE8(offset) CASE8(offset + 8) CASE8(offset + 16) CASE8(offset + 24) @@ -48,6 +57,10 @@ class CellWithArrayStorage : public CellT { LOG(FATAL) << "TOO BIG " << storage_size; UNREACHABLE(); } + template + static std::unique_ptr create(size_t storage_size, ArgsT&&... args) { + return create(DefaultAllocator{}, storage_size, std::forward(args)...); + } private: alignas(alignof(void*)) char storage_[Size]; diff --git a/crypto/vm/cells/DataCell.cpp b/crypto/vm/cells/DataCell.cpp index 8aac575bf..4dd301616 100644 --- a/crypto/vm/cells/DataCell.cpp +++ b/crypto/vm/cells/DataCell.cpp @@ -25,7 +25,44 @@ #include "vm/cells/CellWithStorage.h" namespace vm { +thread_local bool DataCell::use_arena = false; + +namespace { +template +struct ArenaAllocator { + template + std::unique_ptr make_unique(ArgsT&&... args) { + auto* ptr = fast_alloc(sizeof(T)); + T* obj = new (ptr) T(std::forward(args)...); + return std::unique_ptr(obj); + } +private: + td::MutableSlice alloc_batch() { + size_t batch_size = 1 << 20; + auto batch = std::make_unique(batch_size); + return td::MutableSlice(batch.release(), batch_size); + } + char* fast_alloc(size_t size) { + thread_local td::MutableSlice batch; + auto aligned_size = (size + 7) / 8 * 8; + if (batch.size() < size) { + batch = alloc_batch(); + } + auto res = batch.begin(); + batch.remove_prefix(aligned_size); + return res; + } +}; +} std::unique_ptr DataCell::create_empty_data_cell(Info info) { + if (use_arena) { + ArenaAllocator allocator; + auto res = detail::CellWithArrayStorage::create(allocator, info.get_storage_size(), info); + // this is dangerous + Ref(res.get()).release(); + return res; + } + return detail::CellWithUniquePtrStorage::create(info.get_storage_size(), info); } @@ -359,7 +396,7 @@ std::string DataCell::to_hex() const { int len = serialize(buff, sizeof(buff)); char hex_buff[max_serialized_bytes * 2 + 1]; for (int i = 0; i < len; i++) { - sprintf(hex_buff + 2 * i, "%02x", buff[i]); + snprintf(hex_buff + 2 * i, sizeof(hex_buff) - 2 * i, "%02x", buff[i]); } return hex_buff; } diff --git a/crypto/vm/cells/DataCell.h b/crypto/vm/cells/DataCell.h index 933ff04a3..6d3c845fc 100644 --- a/crypto/vm/cells/DataCell.h +++ b/crypto/vm/cells/DataCell.h @@ -27,6 +27,9 @@ namespace vm { class DataCell : public Cell { public: + // NB: cells created with use_arena=true are never freed + static thread_local bool use_arena; + DataCell(const DataCell& other) = delete; ~DataCell() override; @@ -121,10 +124,6 @@ class DataCell : public Cell { void destroy_storage(char* storage); explicit DataCell(Info info); - Cell* get_ref_raw_ptr(unsigned idx) const { - DCHECK(idx < get_refs_cnt()); - return info_.get_refs(get_storage())[idx]; - } public: td::Result load_cell() const override { @@ -152,6 +151,20 @@ class DataCell : public Cell { return Ref(get_ref_raw_ptr(idx)); } + Cell* get_ref_raw_ptr(unsigned idx) const { + DCHECK(idx < get_refs_cnt()); + return info_.get_refs(get_storage())[idx]; + } + + Ref reset_ref_unsafe(unsigned idx, Ref ref, bool check_hash = true) { + CHECK(idx < get_refs_cnt()); + auto refs = info_.get_refs(get_storage()); + CHECK(!check_hash || refs[idx]->get_hash() == ref->get_hash()); + auto res = Ref(refs[idx], Ref::acquire_t{}); // call destructor + refs[idx] = ref.release(); + return res; + } + td::uint32 get_virtualization() const override { return info_.virtualization_; } @@ -173,6 +186,9 @@ class DataCell : public Cell { return ((get_bits() + 23) >> 3) + (with_hashes ? get_level_mask().get_hashes_count() * (hash_bytes + depth_bytes) : 0); } + size_t get_storage_size() const { + return info_.get_storage_size(); + } int serialize(unsigned char* buff, int buff_size, bool with_hashes = false) const; std::string serialize() const; std::string to_hex() const; @@ -207,6 +223,9 @@ class DataCell : public Cell { }; std::ostream& operator<<(std::ostream& os, const DataCell& c); +inline CellHash as_cell_hash(const Ref& cell) { + return cell->get_hash(); +} } // namespace vm diff --git a/crypto/vm/cells/PrunnedCell.h b/crypto/vm/cells/PrunnedCell.h index e8434ae47..a58b245cc 100644 --- a/crypto/vm/cells/PrunnedCell.h +++ b/crypto/vm/cells/PrunnedCell.h @@ -30,18 +30,27 @@ struct PrunnedCellInfo { template class PrunnedCell : public Cell { public: + ExtraT& get_extra() { + return extra_; + } const ExtraT& get_extra() const { return extra_; } static td::Result>> create(const PrunnedCellInfo& prunned_cell_info, ExtraT&& extra) { + return create(detail::DefaultAllocator>(), prunned_cell_info, std::forward(extra)); + } + + template + static td::Result>> create(AllocatorT allocator, const PrunnedCellInfo& prunned_cell_info, + ExtraT&& extra) { auto level_mask = prunned_cell_info.level_mask; if (level_mask.get_level() > max_level) { return td::Status::Error("Level is too big"); } Info info(level_mask); auto prunned_cell = - detail::CellWithUniquePtrStorage>::create(info.get_storage_size(), info, std::move(extra)); + detail::CellWithArrayStorage>::create(allocator, info.get_storage_size(), info, std::move(extra)); TRY_STATUS(prunned_cell->init(prunned_cell_info)); return Ref>(prunned_cell.release(), typename Ref>::acquire_t{}); } @@ -51,6 +60,7 @@ class PrunnedCell : public Cell { } protected: + static constexpr auto max_storage_size = (max_level + 1) * (hash_bytes + sizeof(td::uint16)); struct Info { Info(LevelMask level_mask) { level_mask_ = level_mask.get_mask() & 7; diff --git a/crypto/vm/db/CellHashTable.h b/crypto/vm/db/CellHashTable.h index 7d0308b74..92affce39 100644 --- a/crypto/vm/db/CellHashTable.h +++ b/crypto/vm/db/CellHashTable.h @@ -19,7 +19,7 @@ #pragma once #include "td/utils/Slice.h" - +#include "td/utils/HashSet.h" #include namespace vm { @@ -73,6 +73,6 @@ class CellHashTable { } private: - std::set> set_; + td::NodeHashSet set_; }; } // namespace vm diff --git a/crypto/vm/db/CellStorage.cpp b/crypto/vm/db/CellStorage.cpp index 303d46503..aad7539e5 100644 --- a/crypto/vm/db/CellStorage.cpp +++ b/crypto/vm/db/CellStorage.cpp @@ -33,6 +33,7 @@ class RefcntCellStorer { template void store(StorerT &storer) const { + TD_PERF_COUNTER(cell_store); using td::store; if (as_boc_) { td::int32 tag = -1; @@ -151,18 +152,27 @@ CellLoader::CellLoader(std::shared_ptr reader, std::function CellLoader::load(td::Slice hash, bool need_data, ExtCellCreator &ext_cell_creator) { //LOG(ERROR) << "Storage: load cell " << hash.size() << " " << td::base64_encode(hash); - LoadResult res; + TD_PERF_COUNTER(cell_load); std::string serialized; TRY_RESULT(get_status, reader_->get(hash, serialized)); if (get_status != KeyValue::GetStatus::Ok) { DCHECK(get_status == KeyValue::GetStatus::NotFound); - return res; + return LoadResult{}; + } + TRY_RESULT(res, load(hash, serialized, need_data, ext_cell_creator)); + if (on_load_callback_) { + on_load_callback_(res); } + return res; +} +td::Result CellLoader::load(td::Slice hash, td::Slice value, bool need_data, + ExtCellCreator &ext_cell_creator) { + LoadResult res; res.status = LoadResult::Ok; RefcntCellParser refcnt_cell(need_data); - td::TlParser parser(serialized); + td::TlParser parser(value); refcnt_cell.parse(parser, ext_cell_creator); TRY_STATUS(parser.get_status()); @@ -170,9 +180,6 @@ td::Result CellLoader::load(td::Slice hash, bool need_da res.cell_ = std::move(refcnt_cell.cell); res.stored_boc_ = refcnt_cell.stored_boc_; //CHECK(res.cell_->get_hash() == hash); - if (on_load_callback_) { - on_load_callback_(res); - } return res; } @@ -184,7 +191,11 @@ td::Status CellStorer::erase(td::Slice hash) { return kv_.erase(hash); } +std::string CellStorer::serialize_value(td::int32 refcnt, const td::Ref &cell, bool as_boc) { + return td::serialize(RefcntCellStorer(refcnt, cell, as_boc)); +} + td::Status CellStorer::set(td::int32 refcnt, const td::Ref &cell, bool as_boc) { - return kv_.set(cell->get_hash().as_slice(), td::serialize(RefcntCellStorer(refcnt, cell, as_boc))); + return kv_.set(cell->get_hash().as_slice(), serialize_value(refcnt, cell, as_boc)); } } // namespace vm diff --git a/crypto/vm/db/CellStorage.h b/crypto/vm/db/CellStorage.h index 3106ee16d..8c471d6ca 100644 --- a/crypto/vm/db/CellStorage.h +++ b/crypto/vm/db/CellStorage.h @@ -49,6 +49,7 @@ class CellLoader { }; CellLoader(std::shared_ptr reader, std::function on_load_callback = {}); td::Result load(td::Slice hash, bool need_data, ExtCellCreator &ext_cell_creator); + static td::Result load(td::Slice hash, td::Slice value, bool need_data, ExtCellCreator &ext_cell_creator); private: std::shared_ptr reader_; @@ -60,6 +61,7 @@ class CellStorer { CellStorer(KeyValue &kv); td::Status erase(td::Slice hash); td::Status set(td::int32 refcnt, const td::Ref &cell, bool as_boc); + static std::string serialize_value(td::int32 refcnt, const td::Ref &cell, bool as_boc); private: KeyValue &kv_; diff --git a/crypto/vm/db/DynamicBagOfCellsDb.cpp b/crypto/vm/db/DynamicBagOfCellsDb.cpp index 1aa4e0f56..f1cdb3cef 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDb.cpp @@ -60,6 +60,20 @@ struct CellInfo { bool operator<(const CellInfo &other) const { return key() < other.key(); } + + struct Eq { + using is_transparent = void; // Pred to use + bool operator()(const CellInfo &info, const CellInfo &other_info) const { return info.key() == other_info.key();} + bool operator()(const CellInfo &info, td::Slice hash) const { return info.key().as_slice() == hash;} + bool operator()(td::Slice hash, const CellInfo &info) const { return info.key().as_slice() == hash;} + + }; + struct Hash { + using is_transparent = void; // Pred to use + using transparent_key_equal = Eq; + size_t operator()(td::Slice hash) const { return cell_hash_slice_hash(hash); } + size_t operator()(const CellInfo &info) const { return cell_hash_slice_hash(info.key().as_slice());} + }; }; bool operator<(const CellInfo &a, td::Slice b) { @@ -86,6 +100,12 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat TRY_RESULT(loaded_cell, get_cell_info_force(hash).cell->load_cell()); return std::move(loaded_cell.data_cell); } + td::Result> load_root(td::Slice hash) override { + return load_cell(hash); + } + td::Result> load_root_thread_safe(td::Slice hash) const override { + return td::Status::Error("Not implemented"); + } void load_cell_async(td::Slice hash, std::shared_ptr executor, td::Promise> promise) override { auto info = hash_table_.get_if_exists(hash); diff --git a/crypto/vm/db/DynamicBagOfCellsDb.h b/crypto/vm/db/DynamicBagOfCellsDb.h index fa2b44d21..bc0b6e9ef 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.h +++ b/crypto/vm/db/DynamicBagOfCellsDb.h @@ -23,6 +23,11 @@ #include "td/utils/Status.h" #include "td/actor/PromiseFuture.h" +#include + +namespace td { +class KeyValueReader; +} namespace vm { class CellLoader; class CellStorer; @@ -45,12 +50,20 @@ class DynamicBagOfCellsDb { public: virtual ~DynamicBagOfCellsDb() = default; virtual td::Result> load_cell(td::Slice hash) = 0; + virtual td::Result> load_root(td::Slice hash) = 0; + virtual td::Result> load_root_thread_safe(td::Slice hash) const = 0; struct Stats { + td::int64 roots_total_count{0}; td::int64 cells_total_count{0}; td::int64 cells_total_size{0}; - void apply_diff(Stats diff) { + std::vector> custom_stats; + void apply_diff(const Stats &diff) { + roots_total_count += diff.roots_total_count; cells_total_count += diff.cells_total_count; cells_total_size += diff.cells_total_size; + CHECK(roots_total_count >= 0); + CHECK(cells_total_count >= 0); + CHECK(cells_total_size >= 0); } }; virtual void inc(const Ref &old_root) = 0; @@ -58,6 +71,9 @@ class DynamicBagOfCellsDb { virtual td::Status prepare_commit() = 0; virtual Stats get_stats_diff() = 0; + virtual td::Result get_stats() { + return td::Status::Error("Not implemented"); + } virtual td::Status commit(CellStorer &) = 0; virtual std::shared_ptr get_cell_db_reader() = 0; @@ -65,13 +81,24 @@ class DynamicBagOfCellsDb { virtual td::Status set_loader(std::unique_ptr loader) = 0; virtual void set_celldb_compress_depth(td::uint32 value) = 0; - virtual vm::ExtCellCreator& as_ext_cell_creator() = 0; + virtual vm::ExtCellCreator &as_ext_cell_creator() = 0; static std::unique_ptr create(); + struct CreateInMemoryOptions { + size_t extra_threads{std::thread::hardware_concurrency()}; + bool verbose{true}; + // Allocated DataCels will never be deleted + bool use_arena{false}; + // Almost no overhead in memory during creation, but will scan database twice + bool use_less_memory_during_creation{true}; + }; + static std::unique_ptr create_in_memory(td::KeyValueReader *kv, CreateInMemoryOptions options); + class AsyncExecutor { public: - virtual ~AsyncExecutor() {} + virtual ~AsyncExecutor() { + } virtual void execute_async(std::function f) = 0; virtual void execute_sync(std::function f) = 0; }; diff --git a/crypto/vm/db/InMemoryBagOfCellsDb.cpp b/crypto/vm/db/InMemoryBagOfCellsDb.cpp new file mode 100644 index 000000000..aa8f7910b --- /dev/null +++ b/crypto/vm/db/InMemoryBagOfCellsDb.cpp @@ -0,0 +1,984 @@ +#include "CellStorage.h" +#include "DynamicBagOfCellsDb.h" +#include "td/utils/Timer.h" +#include "td/utils/base64.h" +#include "td/utils/format.h" +#include "td/utils/int_types.h" +#include "td/utils/misc.h" +#include "td/utils/port/Stat.h" +#include "vm/cells/CellHash.h" +#include "vm/cells/CellSlice.h" +#include "vm/cells/DataCell.h" +#include "vm/cells/ExtCell.h" + +#include "td/utils/HashMap.h" +#include "td/utils/HashSet.h" + +#include + +#if TD_PORT_POSIX +#include +#include +#endif + +namespace vm { +namespace { +constexpr bool use_dense_hash_map = true; + +template +void parallel_run(size_t n, F &&run_task, size_t extra_threads_n) { + std::atomic next_task_id{0}; + auto loop = [&] { + while (true) { + auto task_id = next_task_id++; + if (task_id >= n) { + break; + } + run_task(task_id); + } + }; + + // NB: it could be important that td::thread is used, not std::thread + std::vector threads; + for (size_t i = 0; i < extra_threads_n; i++) { + threads.emplace_back(loop); + } + loop(); + for (auto &thread : threads) { + thread.join(); + } + threads.clear(); +} + +struct UniqueAccess { + struct Release { + void operator()(UniqueAccess *access) const { + if (access) { + access->release(); + } + } + }; + using Lock = std::unique_ptr; + Lock lock() { + CHECK(!locked_.exchange(true)); + return Lock(this); + } + + private: + std::atomic locked_{false}; + void release() { + locked_ = false; + } +}; +class DefaultPrunnedCellCreator : public ExtCellCreator { + public: + td::Result> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) override { + TRY_RESULT(cell, PrunnedCell::create(PrunnedCellInfo{level_mask, hash, depth}, td::Unit{})); + return cell; + } +}; + +class ArenaPrunnedCellCreator : public ExtCellCreator { + struct ArenaAllocator { + ArenaAllocator() { + // only one instance ever + static UniqueAccess unique_access; + [[maybe_unused]] auto ptr = unique_access.lock().release(); + } + std::mutex mutex; + struct Deleter { + static constexpr size_t batch_size = 1 << 24; +#if TD_PORT_POSIX + static std::unique_ptr alloc() { + char *ptr = reinterpret_cast( + mmap(NULL, batch_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); + CHECK(ptr != nullptr); + return std::unique_ptr(ptr); + } + void operator()(char *ptr) const { + munmap(ptr, batch_size); + } +#else + static std::unique_ptr alloc() { + auto ptr = reinterpret_cast(malloc(batch_size)); + CHECK(ptr != nullptr); + return std::unique_ptr(ptr); + } + void operator()(char *ptr) const { + free(ptr); + } +#endif + }; + std::vector> arena; + td::uint64 arena_generation{0}; + + td::MutableSlice alloc_batch() { + auto batch = Deleter::alloc(); + auto res = td::MutableSlice(batch.get(), Deleter::batch_size); + std::lock_guard guard(mutex); + arena.emplace_back(std::move(batch)); + return res; + } + + char *alloc(size_t size) { + thread_local td::MutableSlice batch; + thread_local td::uint64 batch_generation{0}; + auto aligned_size = (size + 7) / 8 * 8; + if (batch.size() < size || batch_generation != arena_generation) { + batch = alloc_batch(); + batch_generation = arena_generation; + } + auto res = batch.begin(); + batch.remove_prefix(aligned_size); + return res; + } + void clear() { + std::lock_guard guard(mutex); + arena_generation++; + td::reset_to_empty(arena); + } + }; + static ArenaAllocator arena_; + static td::ThreadSafeCounter cells_count_; + + public: + struct Counter { + Counter() { + cells_count_.add(1); + } + Counter(Counter &&other) { + cells_count_.add(1); + } + Counter(const Counter &other) { + cells_count_.add(1); + } + ~Counter() { + cells_count_.add(-1); + } + }; + + struct Allocator { + template + std::unique_ptr> make_unique(ArgsT &&...args) { + auto *ptr = arena_.alloc(sizeof(T)); + T *obj = new (ptr) T(std::forward(args)...); + return std::unique_ptr(obj); + } + }; + td::Result> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) override { + Allocator allocator; + TRY_RESULT(cell, PrunnedCell::create(allocator, PrunnedCellInfo{level_mask, hash, depth}, Counter())); + return cell; + } + static td::int64 count() { + return cells_count_.sum(); + } + static void clear_arena() { + LOG_CHECK(cells_count_.sum() == 0) << cells_count_.sum(); + arena_.clear(); + } +}; +td::ThreadSafeCounter ArenaPrunnedCellCreator::cells_count_; +ArenaPrunnedCellCreator::ArenaAllocator ArenaPrunnedCellCreator::arena_; + +struct CellInfo { + mutable td::int32 db_refcnt{0}; + Ref cell; +}; +static_assert(sizeof(CellInfo) == 16); + +CellHash as_cell_hash(const CellInfo &info) { + return info.cell->get_hash(); +} + +struct CellInfoHashTableBaseline { + td::HashSet ht_; + const CellInfo *find(CellHash hash) const { + if (auto it = ht_.find(hash); it != ht_.end()) { + return &*it; + } + return nullptr; + } + void erase(CellHash hash) { + auto it = ht_.find(hash); + CHECK(it != ht_.end()); + ht_.erase(it); + } + void insert(CellInfo info) { + ht_.insert(std::move(info)); + } + template + void init_from(Iterator begin, Iterator end) { + ht_ = td::HashSet(begin, end); + } + size_t size() const { + return ht_.size(); + } + auto begin() const { + return ht_.begin(); + } + auto end() const { + return ht_.end(); + } + size_t bucket_count() const { + return ht_.bucket_count(); + } + template + auto for_each(F &&f) { + for (auto &it : ht_) { + f(it); + } + } +}; + +struct CellInfoHashTableDense { + size_t dense_ht_size_{0}; + size_t dense_ht_buckets_{1}; + std::vector dense_ht_offsets_{1}; + std::vector dense_ht_values_; + td::HashSet new_ht_; + size_t dense_choose_bucket(const CellHash &hash) const { + return cell_hash_slice_hash(hash.as_slice()) % dense_ht_buckets_; + } + const CellInfo *dense_find(CellHash hash) const { + auto bucket_i = dense_choose_bucket(hash); + auto begin = dense_ht_values_.begin() + dense_ht_offsets_[bucket_i]; + auto end = dense_ht_values_.begin() + dense_ht_offsets_[bucket_i + 1]; + for (auto it = begin; it != end; ++it) { + if (it->cell.not_null() && it->cell->get_hash() == hash) { + return &*it; + } + } + return nullptr; + } + CellInfo *dense_find_empty(CellHash hash) { + auto bucket_i = dense_choose_bucket(hash); + auto begin = dense_ht_values_.begin() + dense_ht_offsets_[bucket_i]; + auto end = dense_ht_values_.begin() + dense_ht_offsets_[bucket_i + 1]; + for (auto it = begin; it != end; ++it) { + if (it->cell.is_null()) { + return &*it; + } + } + return nullptr; + } + const CellInfo *find(CellHash hash) const { + if (auto it = new_ht_.find(hash); it != new_ht_.end()) { + return &*it; + } + if (auto it = dense_find(hash)) { + return it; + } + return nullptr; + } + void erase(CellHash hash) { + if (auto it = new_ht_.find(hash); it != new_ht_.end()) { + new_ht_.erase(it); + return; + } + auto info = dense_find(hash); + CHECK(info && info->db_refcnt > 0); + info->db_refcnt = 0; + const_cast(info)->cell = {}; + CHECK(dense_ht_size_ > 0); + dense_ht_size_--; + } + + void insert(CellInfo info) { + if (auto dest = dense_find_empty(info.cell->get_hash())) { + *dest = std::move(info); + dense_ht_size_++; + return; + } + new_ht_.insert(std::move(info)); + } + template + void init_from(Iterator begin, Iterator end) { + auto size = td::narrow_cast(std::distance(begin, end)); + dense_ht_buckets_ = std::max(size_t(1), size_t(size / 8)); + + std::vector offsets(dense_ht_buckets_ + 2); + for (auto it = begin; it != end; ++it) { + auto bucket_i = dense_choose_bucket(it->cell->get_hash()); + offsets[bucket_i + 2]++; + } + for (size_t i = 1; i < offsets.size(); i++) { + offsets[i] += offsets[i - 1]; + } + dense_ht_values_.resize(size); + for (auto it = begin; it != end; ++it) { + auto bucket_i = dense_choose_bucket(it->cell->get_hash()); + dense_ht_values_[offsets[bucket_i + 1]++] = std::move(*it); + } + CHECK(offsets[0] == 0); + CHECK(offsets[offsets.size() - 1] == size); + CHECK(offsets[offsets.size() - 2] == size); + dense_ht_offsets_ = std::move(offsets); + dense_ht_size_ = size; + } + size_t size() const { + return dense_ht_size_ + new_ht_.size(); + } + template + auto for_each(F &&f) { + for (auto &it : dense_ht_values_) { + if (it.cell.not_null()) { + f(it); + } + } + for (auto &it : new_ht_) { + f(it); + } + } + size_t bucket_count() const { + return new_ht_.bucket_count() + dense_ht_values_.size(); + } +}; + +using CellInfoHashTable = std::conditional_t; + +class CellStorage { + struct PrivateTag {}; + struct CellBucket; + struct None { + void operator()(CellBucket *bucket) { + } + }; + struct CellBucketRef { + UniqueAccess::Lock lock; + std::unique_ptr bucket; + CellBucket &operator*() { + return *bucket; + } + CellBucket *operator->() { + return bucket.get(); + } + }; + struct CellBucket { + mutable UniqueAccess access_; + CellInfoHashTable infos_; + std::vector cells_; + std::vector> roots_; + size_t boc_count_{0}; + [[maybe_unused]] char pad3[TD_CONCURRENCY_PAD]; + + void clear() { + td::reset_to_empty(infos_); + td::reset_to_empty(cells_); + td::reset_to_empty(roots_); + } + + CellBucketRef unique_access() const { + auto lock = access_.lock(); + return CellBucketRef{.lock = std::move(lock), + .bucket = std::unique_ptr(const_cast(this))}; + } + }; + std::array buckets_{}; + bool inited_{false}; + + const CellBucket &get_bucket(size_t i) const { + return buckets_.at(i); + } + const CellBucket &get_bucket(const CellHash &hash) const { + return get_bucket(hash.as_array()[0]); + } + + mutable UniqueAccess local_access_; + td::HashSet, CellHashF, CellEqF> local_roots_; + DynamicBagOfCellsDb::Stats stats_; + + mutable std::mutex root_mutex_; + td::HashSet, CellHashF, CellEqF> roots_; + + public: + std::optional get_info(const CellHash &hash) const { + auto lock = local_access_.lock(); + auto &bucket = get_bucket(hash); + if (auto info_ptr = bucket.infos_.find(hash)) { + return *info_ptr; + } + return {}; + } + + DynamicBagOfCellsDb::Stats get_stats() { + auto unique_access = local_access_.lock(); + auto stats = stats_; + auto add_stat = [&stats](auto key, auto value) { + stats.custom_stats.emplace_back(std::move(key), PSTRING() << value); + }; + if constexpr (use_dense_hash_map) { + size_t dense_ht_capacity = 0; + size_t new_ht_capacity = 0; + size_t dense_ht_size = 0; + size_t new_ht_size = 0; + for_each_bucket(0, [&](auto bucket_id, CellBucket &bucket) { + dense_ht_capacity += bucket.infos_.dense_ht_values_.size(); + dense_ht_size += bucket.infos_.dense_ht_size_; + new_ht_capacity += bucket.infos_.new_ht_.bucket_count(); + new_ht_size += bucket.infos_.new_ht_.size(); + }); + auto size = new_ht_size + dense_ht_size; + auto capacity = new_ht_capacity + dense_ht_capacity; + add_stat("ht.capacity", capacity); + add_stat("ht.size", size); + add_stat("ht.load", double(size) / std::max(1.0, double(capacity))); + add_stat("ht.dense_ht_capacity", dense_ht_capacity); + add_stat("ht.dense_ht_size", dense_ht_size); + add_stat("ht.dense_ht_load", double(dense_ht_size) / std::max(1.0, double(dense_ht_capacity))); + add_stat("ht.new_ht_capacity", new_ht_capacity); + add_stat("ht.new_ht_size", new_ht_size); + add_stat("ht.new_ht_load", double(new_ht_size) / std::max(1.0, double(new_ht_capacity))); + } else { + size_t capacity = 0; + size_t size = 0; + for_each_bucket(0, [&](auto bucket_id, CellBucket &bucket) { + capacity += bucket.infos_.bucket_count(); + size += bucket.infos_.size(); + }); + add_stat("ht.capacity", capacity); + add_stat("ht.size", size); + add_stat("ht.load", double(size) / std::max(1.0, double(capacity))); + } + CHECK(td::narrow_cast(stats.roots_total_count) == local_roots_.size()); + return stats; + } + void apply_stats_diff(DynamicBagOfCellsDb::Stats diff) { + auto unique_access = local_access_.lock(); + stats_.apply_diff(diff); + CHECK(td::narrow_cast(stats_.roots_total_count) == local_roots_.size()); + size_t cells_count{0}; + for_each_bucket(0, [&](size_t bucket_id, auto &bucket) { cells_count += bucket.infos_.size(); }); + CHECK(td::narrow_cast(stats_.cells_total_count) == cells_count); + } + + td::Result> load_cell(const CellHash &hash) const { + auto lock = local_access_.lock(); + auto &bucket = get_bucket(hash); + if (auto info_ptr = bucket.infos_.find(hash)) { + return info_ptr->cell; + } + return td::Status::Error("not found"); + } + + td::Result> load_root_local(const CellHash &hash) const { + auto lock = local_access_.lock(); + if (auto it = local_roots_.find(hash); it != local_roots_.end()) { + return *it; + } + return td::Status::Error("not found"); + } + td::Result> load_root_shared(const CellHash &hash) const { + std::lock_guard lock(root_mutex_); + if (auto it = roots_.find(hash); it != roots_.end()) { + return *it; + } + return td::Status::Error("not found"); + } + + void erase(const CellHash &hash) { + auto lock = local_access_.lock(); + auto bucket = get_bucket(hash).unique_access(); + bucket->infos_.erase(hash); + if (auto local_it = local_roots_.find(hash); local_it != local_roots_.end()) { + local_roots_.erase(local_it); + std::lock_guard root_lock(root_mutex_); + auto shared_it = roots_.find(hash); + CHECK(shared_it != roots_.end()); + roots_.erase(shared_it); + CHECK(stats_.roots_total_count > 0); + stats_.roots_total_count--; + } + } + + void add_new_root(Ref cell) { + auto lock = local_access_.lock(); + if (local_roots_.insert(cell).second) { + std::lock_guard lock(root_mutex_); + roots_.insert(std::move(cell)); + stats_.roots_total_count++; + } + } + + void set(td::int32 refcnt, Ref cell) { + auto lock = local_access_.lock(); + //LOG(ERROR) << "setting refcnt to " << refcnt << ", cell " << td::base64_encode(cell->get_hash().as_slice()); + auto hash = cell->get_hash(); + auto bucket = get_bucket(hash).unique_access(); + if (auto info_ptr = bucket->infos_.find(hash)) { + CHECK(info_ptr->cell.get() == cell.get()); + info_ptr->db_refcnt = refcnt; + } else { + bucket->infos_.insert({.db_refcnt = refcnt, .cell = std::move(cell)}); + } + } + + template + static td::unique_ptr build(DynamicBagOfCellsDb::CreateInMemoryOptions options, + F &¶llel_scan_cells) { + auto storage = td::make_unique(PrivateTag{}); + storage->do_build(options, parallel_scan_cells); + return storage; + } + + ~CellStorage() { + clear(); + } + CellStorage() = delete; + explicit CellStorage(PrivateTag) { + } + + private: + template + void do_build(DynamicBagOfCellsDb::CreateInMemoryOptions options, F &¶llel_scan_cells) { + auto verbose = options.verbose; + td::Slice P = "loading in-memory cell database: "; + LOG_IF(WARNING, verbose) << P << "start with options use_arena=" << options.use_arena + << " use_less_memory_during_creation=" << options.use_less_memory_during_creation + << " use_dense_hash_map=" << use_dense_hash_map; + auto full_timer = td::Timer(); + auto lock = local_access_.lock(); + CHECK(ArenaPrunnedCellCreator::count() == 0); + ArenaPrunnedCellCreator arena_pc_creator; + DefaultPrunnedCellCreator default_pc_creator; + + auto timer = td::Timer(); + td::int64 cell_count{0}; + td::int64 desc_count{0}; + if (options.use_less_memory_during_creation) { + auto [new_cell_count, new_desc_count] = parallel_scan_cells( + default_pc_creator, options.use_arena, + [&](td::int32 refcnt, Ref cell) { initial_set_without_refs(refcnt, std::move(cell)); }); + cell_count = new_cell_count; + desc_count = new_desc_count; + } else { + auto [new_cell_count, new_desc_count] = + parallel_scan_cells(arena_pc_creator, options.use_arena, + [&](td::int32 refcnt, Ref cell) { initial_set(refcnt, std::move(cell)); }); + cell_count = new_cell_count; + desc_count = new_desc_count; + } + LOG_IF(WARNING, verbose) << P << "cells loaded in " << timer.elapsed() << "s, cells_count= " << cell_count + << " prunned_cells_count=" << ArenaPrunnedCellCreator::count(); + + timer = td::Timer(); + for_each_bucket(options.extra_threads, [&](size_t bucket_id, auto &bucket) { build_hashtable(bucket); }); + + size_t ht_capacity = 0; + size_t ht_size = 0; + for_each_bucket(0, [&](size_t bucket_id, auto &bucket) { + ht_size += bucket.infos_.size(); + ht_capacity += bucket.infos_.bucket_count(); + }); + double load_factor = double(ht_size) / std::max(double(ht_capacity), 1.0); + LOG_IF(WARNING, verbose) << P << "hashtable created in " << timer.elapsed() + << "s, hashtables_expected_size=" << td::format::as_size(ht_capacity * sizeof(CellInfo)) + << " load_factor=" << load_factor; + + timer = td::Timer(); + if (options.use_less_memory_during_creation) { + auto [new_cell_count, new_desc_count] = + parallel_scan_cells(default_pc_creator, false, + [&](td::int32 refcnt, Ref cell) { secondary_set(refcnt, std::move(cell)); }); + CHECK(new_cell_count == cell_count); + CHECK(new_desc_count == desc_count); + } else { + for_each_bucket(options.extra_threads, [&](size_t bucket_id, auto &bucket) { reset_refs(bucket); }); + } + LOG_IF(WARNING, verbose) << P << "refs rearranged in " << timer.elapsed() << "s"; + + timer = td::Timer(); + using Stats = DynamicBagOfCellsDb::Stats; + std::vector bucket_stats(buckets_.size()); + std::atomic boc_count{0}; + for_each_bucket(options.extra_threads, [&](size_t bucket_id, auto &bucket) { + bucket_stats[bucket_id] = validate_bucket_a(bucket, options.use_arena); + boc_count += bucket.boc_count_; + }); + for_each_bucket(options.extra_threads, [&](size_t bucket_id, auto &bucket) { validate_bucket_b(bucket); }); + stats_ = {}; + for (auto &bucket_stat : bucket_stats) { + stats_.apply_diff(bucket_stat); + } + LOG_IF(WARNING, verbose) << P << "refcnt validated in " << timer.elapsed() << "s"; + + timer = td::Timer(); + build_roots(); + LOG_IF(WARNING, verbose) << P << "roots hashtable built in " << timer.elapsed() << "s"; + ArenaPrunnedCellCreator::clear_arena(); + LOG_IF(WARNING, verbose) << P << "arena cleared in " << timer.elapsed(); + + lock.reset(); + auto r_mem_stat = td::mem_stat(); + td::MemStat mem_stat; + if (r_mem_stat.is_ok()) { + mem_stat = r_mem_stat.move_as_ok(); + } + auto stats = get_stats(); + td::StringBuilder sb; + for (auto &[key, value] : stats.custom_stats) { + sb << "\n\t" << key << "=" << value; + } + LOG_IF(ERROR, desc_count != 0 && desc_count != stats.roots_total_count + 1) + << "desc<> keys count is " << desc_count << " wich is different from roots count " << stats.roots_total_count; + LOG_IF(WARNING, verbose) + << P << "done in " << full_timer.elapsed() << "\n\troots_count=" << stats.roots_total_count << "\n\t" + << desc_count << "\n\tcells_count=" << stats.cells_total_count + << "\n\tcells_size=" << td::format::as_size(stats.cells_total_size) << "\n\tboc_count=" << boc_count.load() + << sb.as_cslice() << "\n\tdata_cells_size=" << td::format::as_size(sizeof(DataCell) * stats.cells_total_count) + << "\n\tdata_cell_size=" << sizeof(DataCell) << "\n\texpected_memory_used=" + << td::format::as_size(stats.cells_total_count * (sizeof(DataCell) + sizeof(CellInfo) * 3 / 2) + + stats.cells_total_size) + << "\n\tbest_possible_memory_used" + << td::format::as_size(stats.cells_total_count * (sizeof(DataCell) + sizeof(CellInfo)) + stats.cells_total_size) + << "\n\tmemory_used=" << td::format::as_size(mem_stat.resident_size_) + << "\n\tpeak_memory_used=" << td::format::as_size(mem_stat.resident_size_peak_); + + inited_ = true; + } + + template + void for_each_bucket(size_t extra_threads, F &&f) { + parallel_run( + buckets_.size(), [&](auto task_id) { f(task_id, *get_bucket(task_id).unique_access()); }, extra_threads); + } + + void clear() { + auto unique_access = local_access_.lock(); + for_each_bucket(td::thread::hardware_concurrency(), [&](size_t bucket_id, auto &bucket) { bucket.clear(); }); + local_roots_.clear(); + { + auto lock = std::lock_guard(root_mutex_); + roots_.clear(); + } + } + + void initial_set(td::int32 refcnt, Ref cell) { + CHECK(!inited_); + auto bucket = get_bucket(cell->get_hash()).unique_access(); + bucket->cells_.push_back({.db_refcnt = refcnt, .cell = std::move(cell)}); + } + + void initial_set_without_refs(td::int32 refcnt, Ref cell_ref) { + CHECK(!inited_); + auto bucket = get_bucket(cell_ref->get_hash()).unique_access(); + auto &cell = const_cast(*cell_ref); + for (unsigned i = 0; i < cell.size_refs(); i++) { + auto to_destroy = cell.reset_ref_unsafe(i, Ref(), false); + if (to_destroy->is_loaded()) { + bucket->boc_count_++; + } + } + bucket->cells_.push_back({.db_refcnt = refcnt, .cell = std::move(cell_ref)}); + } + + void secondary_set(td::int32 refcnt, Ref cell_copy) { + CHECK(!inited_); + auto bucket = get_bucket(cell_copy->get_hash()).unique_access(); + auto info = bucket->infos_.find(cell_copy->get_hash()); + CHECK(info); + CellSlice cs(NoVm{}, std::move(cell_copy)); + auto &cell = const_cast(*info->cell); + CHECK(cs.size_refs() == cell.size_refs()); + for (unsigned i = 0; i < cell.size_refs(); i++) { + auto prunned_cell_hash = cs.fetch_ref()->get_hash(); + auto &prunned_cell_bucket = get_bucket(prunned_cell_hash); + auto full_cell_ptr = prunned_cell_bucket.infos_.find(prunned_cell_hash); + CHECK(full_cell_ptr); + auto full_cell = full_cell_ptr->cell; + auto to_destroy = cell.reset_ref_unsafe(i, std::move(full_cell), false); + CHECK(to_destroy.is_null()); + } + } + + void build_hashtable(CellBucket &bucket) { + bucket.infos_.init_from(bucket.cells_.begin(), bucket.cells_.end()); + LOG_CHECK(bucket.infos_.size() == bucket.cells_.size()) << bucket.infos_.size() << " vs " << bucket.cells_.size(); + td::reset_to_empty(bucket.cells_); + LOG_CHECK(bucket.cells_.capacity() == 0) << bucket.cells_.capacity(); + } + + void reset_refs(CellBucket &bucket) { + bucket.infos_.for_each([&](auto &it) { + // This is generally very dangerous, but should be safe here + auto &cell = const_cast(*it.cell); + for (unsigned i = 0; i < cell.size_refs(); i++) { + auto prunned_cell = cell.get_ref_raw_ptr(i); + auto prunned_cell_hash = prunned_cell->get_hash(); + auto &prunned_cell_bucket = get_bucket(prunned_cell_hash); + auto full_cell_ptr = prunned_cell_bucket.infos_.find(prunned_cell_hash); + CHECK(full_cell_ptr); + auto full_cell = full_cell_ptr->cell; + auto to_destroy = cell.reset_ref_unsafe(i, std::move(full_cell)); + if (!to_destroy->is_loaded()) { + Ref> x(std::move(to_destroy)); + x->~PrunnedCell(); + x.release(); + } else { + bucket.boc_count_++; + } + } + }); + } + + DynamicBagOfCellsDb::Stats validate_bucket_a(CellBucket &bucket, bool use_arena) { + DynamicBagOfCellsDb::Stats stats; + bucket.infos_.for_each([&](auto &it) { + int cell_ref_cnt = it.cell->get_refcnt(); + CHECK(it.db_refcnt + 1 + use_arena >= cell_ref_cnt); + auto extra_refcnt = it.db_refcnt + 1 + use_arena - cell_ref_cnt; + if (extra_refcnt != 0) { + bucket.roots_.push_back(it.cell); + stats.roots_total_count++; + } + stats.cells_total_count++; + stats.cells_total_size += static_cast(it.cell->get_storage_size()); + }); + return stats; + } + void validate_bucket_b(CellBucket &bucket) { + // sanity check + bucket.infos_.for_each([&](auto &it) { + CellSlice cs(NoVm{}, it.cell); + while (cs.have_refs()) { + CHECK(cs.fetch_ref().not_null()); + } + }); + } + void build_roots() { + for (auto &it : buckets_) { + for (auto &root : it.roots_) { + local_roots_.insert(std::move(root)); + } + td::reset_to_empty(it.roots_); + } + auto lock = std::lock_guard(root_mutex_); + roots_ = local_roots_; + } +}; + +class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { + public: + explicit InMemoryBagOfCellsDb(td::unique_ptr storage) : storage_(std::move(storage)) { + } + + td::Result> load_cell(td::Slice hash) override { + return storage_->load_cell(CellHash::from_slice(hash)); + } + + td::Result> load_root(td::Slice hash) override { + return storage_->load_root_local(CellHash::from_slice(hash)); + } + td::Result> load_root_thread_safe(td::Slice hash) const override { + return storage_->load_root_shared(CellHash::from_slice(hash)); + } + + void inc(const Ref &cell) override { + if (cell.is_null()) { + return; + } + if (cell->get_virtualization() != 0) { + return; + } + to_inc_.push_back(cell); + } + + void dec(const Ref &cell) override { + if (cell.is_null()) { + return; + } + if (cell->get_virtualization() != 0) { + return; + } + to_dec_.push_back(cell); + } + + td::Status commit(CellStorer &cell_storer) override { + if (!to_inc_.empty() || !to_dec_.empty()) { + TRY_STATUS(prepare_commit()); + } + + Stats diff; + CHECK(to_dec_.empty()); + for (auto &it : info_) { + auto &info = it.second; + if (info.diff_refcnt == 0) { + continue; + } + auto refcnt = td::narrow_cast(static_cast(info.db_refcnt) + info.diff_refcnt); + CHECK(refcnt >= 0); + if (refcnt > 0) { + cell_storer.set(refcnt, info.cell, false); + storage_->set(refcnt, info.cell); + if (info.db_refcnt == 0) { + diff.cells_total_count++; + diff.cells_total_size += static_cast(info.cell->get_storage_size()); + } + } else { + cell_storer.erase(info.cell->get_hash().as_slice()); + storage_->erase(info.cell->get_hash()); + diff.cells_total_count--; + diff.cells_total_size -= static_cast(info.cell->get_storage_size()); + } + } + storage_->apply_stats_diff(diff); + info_ = {}; + return td::Status::OK(); + } + + td::Result get_stats() override { + return storage_->get_stats(); + } + + // Not implemented or trivial or deprecated methods + td::Status set_loader(std::unique_ptr loader) override { + return td::Status::OK(); + } + + td::Status prepare_commit() override { + CHECK(info_.empty()); + for (auto &to_inc : to_inc_) { + auto new_root = do_inc(to_inc); + storage_->add_new_root(std::move(new_root)); + } + for (auto &to_dec : to_dec_) { + do_dec(to_dec); + } + to_dec_ = {}; + to_inc_ = {}; + return td::Status::OK(); + } + Stats get_stats_diff() override { + LOG(FATAL) << "Not implemented"; + return {}; + } + std::shared_ptr get_cell_db_reader() override { + return {}; + } + void set_celldb_compress_depth(td::uint32 value) override { + LOG(FATAL) << "Not implemented"; + } + ExtCellCreator &as_ext_cell_creator() override { + UNREACHABLE(); + } + void load_cell_async(td::Slice hash, std::shared_ptr executor, + td::Promise> promise) override { + LOG(FATAL) << "Not implemented"; + } + + private: + td::unique_ptr storage_; + + struct Info { + td::int32 db_refcnt{0}; + td::int32 diff_refcnt{0}; + Ref cell; + }; + td::HashMap info_; + + std::unique_ptr loader_; + std::vector> to_inc_; + std::vector> to_dec_; + + Ref do_inc(Ref cell) { + auto cell_hash = cell->get_hash(); + if (auto it = info_.find(cell_hash); it != info_.end()) { + CHECK(it->second.diff_refcnt != std::numeric_limits::max()); + it->second.diff_refcnt++; + return it->second.cell; + } + if (auto o_info = storage_->get_info(cell_hash)) { + info_.emplace(cell_hash, Info{.db_refcnt = o_info->db_refcnt, .diff_refcnt = 1, .cell = o_info->cell}); + return std::move(o_info->cell); + } + + CellSlice cs(NoVm{}, std::move(cell)); + CellBuilder cb; + cb.store_bits(cs.data(), cs.size()); + while (cs.have_refs()) { + auto ref = do_inc(cs.fetch_ref()); + cb.store_ref(std::move(ref)); + } + auto res = cb.finalize(cs.is_special()); + CHECK(res->get_hash() == cell_hash); + info_.emplace(cell_hash, Info{.db_refcnt = 0, .diff_refcnt = 1, .cell = res}); + return res; + } + + void do_dec(Ref cell) { + auto cell_hash = cell->get_hash(); + auto it = info_.find(cell_hash); + if (it != info_.end()) { + CHECK(it->second.diff_refcnt != std::numeric_limits::min()); + --it->second.diff_refcnt; + } else { + auto info = *storage_->get_info(cell_hash); + it = info_.emplace(cell_hash, Info{.db_refcnt = info.db_refcnt, .diff_refcnt = -1, .cell = info.cell}).first; + } + if (it->second.diff_refcnt + it->second.db_refcnt != 0) { + return; + } + CellSlice cs(NoVm{}, std::move(cell)); + while (cs.have_refs()) { + do_dec(cs.fetch_ref()); + } + } +}; + +} // namespace + +std::unique_ptr DynamicBagOfCellsDb::create_in_memory(td::KeyValueReader *kv, + CreateInMemoryOptions options) { + if (kv == nullptr) { + LOG_IF(WARNING, options.verbose) << "Create empty in-memory cells database (no key value is given)"; + auto storage = CellStorage::build(options, [](auto, auto, auto) { return std::make_pair(0, 0); }); + return std::make_unique(std::move(storage)); + } + + std::vector keys; + keys.emplace_back(""); + for (td::uint32 c = 1; c <= 0xff; c++) { + keys.emplace_back(1, static_cast(c)); + } + keys.emplace_back(33, static_cast(0xff)); + + auto parallel_scan_cells = [&](ExtCellCreator &pc_creator, bool use_arena, + auto &&f) -> std::pair { + std::atomic cell_count{0}; + std::atomic desc_count{0}; + parallel_run( + keys.size() - 1, + [&](auto task_id) { + td::int64 local_cell_count = 0; + td::int64 local_desc_count = 0; + CHECK(!DataCell::use_arena); + DataCell::use_arena = use_arena; + kv->for_each_in_range(keys.at(task_id), keys.at(task_id + 1), [&](td::Slice key, td::Slice value) { + if (td::begins_with(key, "desc") && key.size() != 32) { + local_desc_count++; + return td::Status::OK(); + } + auto r_res = CellLoader::load(key, value.str(), true, pc_creator); + if (r_res.is_error()) { + LOG(ERROR) << r_res.error() << " at " << td::format::escaped(key); + return td::Status::OK(); + } + CHECK(key.size() == 32); + CHECK(key.ubegin()[0] == task_id); + auto res = r_res.move_as_ok(); + f(res.refcnt(), res.cell()); + local_cell_count++; + return td::Status::OK(); + }).ensure(); + DataCell::use_arena = false; + cell_count += local_cell_count; + desc_count += local_desc_count; + }, + options.extra_threads); + return std::make_pair(cell_count.load(), desc_count.load()); + }; + + auto storage = CellStorage::build(options, parallel_scan_cells); + return std::make_unique(std::move(storage)); +} +} // namespace vm diff --git a/crypto/vm/db/TonDb.h b/crypto/vm/db/TonDb.h index dfce31104..6cd8aa4fa 100644 --- a/crypto/vm/db/TonDb.h +++ b/crypto/vm/db/TonDb.h @@ -113,7 +113,8 @@ class TonDbTransactionImpl; using TonDbTransaction = std::unique_ptr; class TonDbTransactionImpl { public: - SmartContractDb begin_smartcontract(td::Slice hash = {}); + + SmartContractDb begin_smartcontract(td::Slice hash = std::string(32, '\0')); void commit_smartcontract(SmartContractDb txn); void commit_smartcontract(SmartContractDiff txn); @@ -142,6 +143,20 @@ class TonDbTransactionImpl { friend bool operator<(td::Slice hash, const SmartContractInfo &info) { return hash < info.hash; } + + struct Eq { + using is_transparent = void; // Pred to use + bool operator()(const SmartContractInfo &info, const SmartContractInfo &other_info) const { return info.hash == other_info.hash;} + bool operator()(const SmartContractInfo &info, td::Slice hash) const { return info.hash == hash;} + bool operator()(td::Slice hash, const SmartContractInfo &info) const { return info.hash == hash;} + + }; + struct Hash { + using is_transparent = void; // Pred to use + using transparent_key_equal = Eq; + size_t operator()(td::Slice hash) const { return cell_hash_slice_hash(hash); } + size_t operator()(const SmartContractInfo &info) const { return cell_hash_slice_hash(info.hash);} + }; }; CellHashTable contracts_; diff --git a/crypto/vm/large-boc-serializer.cpp b/crypto/vm/large-boc-serializer.cpp index a7dae1b08..d209c88ed 100644 --- a/crypto/vm/large-boc-serializer.cpp +++ b/crypto/vm/large-boc-serializer.cpp @@ -33,10 +33,12 @@ class LargeBocSerializer { public: using Hash = Cell::Hash; - explicit LargeBocSerializer(std::shared_ptr reader, td::CancellationToken cancellation_token = {}) - : reader(std::move(reader)), cancellation_token(std::move(cancellation_token)) { + explicit LargeBocSerializer(std::shared_ptr reader) : reader(std::move(reader)) { } + void set_logger(BagOfCellsLogger* logger_ptr) { + logger_ptr_ = logger_ptr; + } void add_root(Hash root); td::Status import_cells(); td::Status serialize(td::FileFd& fd, int mode); @@ -44,6 +46,7 @@ class LargeBocSerializer { private: std::shared_ptr reader; struct CellInfo { + Cell::Hash hash; std::array ref_idx; int idx; unsigned short serialized_size; @@ -67,7 +70,7 @@ class LargeBocSerializer { return 4; } }; - std::map cells; + td::NodeHashMap cells; std::vector*> cell_list; struct RootInfo { RootInfo(Hash hash, int idx) : hash(hash), idx(idx) { @@ -85,10 +88,7 @@ class LargeBocSerializer { int revisit(int cell_idx, int force = 0); td::uint64 compute_sizes(int mode, int& r_size, int& o_size); - td::CancellationToken cancellation_token; - td::Timestamp log_speed_at_; - size_t processed_cells_ = 0; - static constexpr double LOG_SPEED_PERIOD = 120.0; + BagOfCellsLogger* logger_ptr_{}; }; void LargeBocSerializer::add_root(Hash root) { @@ -96,16 +96,18 @@ void LargeBocSerializer::add_root(Hash root) { } td::Status LargeBocSerializer::import_cells() { - td::Timer timer; - log_speed_at_ = td::Timestamp::in(LOG_SPEED_PERIOD); - processed_cells_ = 0; + if (logger_ptr_) { + logger_ptr_->start_stage("import_cells"); + } for (auto& root : roots) { TRY_RESULT(idx, import_cell(root.hash)); root.idx = idx; } reorder_cells(); CHECK(!cell_list.empty()); - LOG(ERROR) << "serializer: import_cells took " << timer.elapsed() << "s, " << cell_count << " cells"; + if (logger_ptr_) { + logger_ptr_->finish_stage(PSLICE() << cell_count << " cells"); + } return td::Status::OK(); } @@ -113,14 +115,8 @@ td::Result LargeBocSerializer::import_cell(Hash hash, int depth) { if (depth > Cell::max_depth) { return td::Status::Error("error while importing a cell into a bag of cells: cell depth too large"); } - ++processed_cells_; - if (processed_cells_ % 1000 == 0) { - TRY_STATUS(cancellation_token.check()); - } - if (log_speed_at_.is_in_past()) { - log_speed_at_ += LOG_SPEED_PERIOD; - LOG(WARNING) << "serializer: import_cells " << (double)processed_cells_ / LOG_SPEED_PERIOD << " cells/s"; - processed_cells_ = 0; + if (logger_ptr_) { + TRY_STATUS(logger_ptr_->on_cell_processed()); } auto it = cells.find(hash); if (it != cells.end()) { @@ -306,7 +302,6 @@ td::uint64 LargeBocSerializer::compute_sizes(int mode, int& r_size, int& o_size) } td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { - td::Timer timer; using Mode = BagOfCells::Mode; BagOfCells::Info info; if ((mode & Mode::WithCacheBits) && !(mode & Mode::WithIndex)) { @@ -370,6 +365,9 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { DCHECK(writer.position() == info.index_offset); DCHECK((unsigned)cell_count == cell_list.size()); if (info.has_index) { + if (logger_ptr_) { + logger_ptr_->start_stage("generate_index"); + } std::size_t offs = 0; for (int i = cell_count - 1; i >= 0; --i) { const auto& dc_info = cell_list[i]->second; @@ -387,13 +385,20 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { fixed_offset = offs * 2 + dc_info.should_cache; } store_offset(fixed_offset); + if (logger_ptr_) { + TRY_STATUS(logger_ptr_->on_cell_processed()); + } } DCHECK(offs == info.data_size); + if (logger_ptr_) { + logger_ptr_->finish_stage(""); + } } DCHECK(writer.position() == info.data_offset); size_t keep_position = writer.position(); - log_speed_at_ = td::Timestamp::in(LOG_SPEED_PERIOD); - processed_cells_ = 0; + if (logger_ptr_) { + logger_ptr_->start_stage("serialize"); + } for (int i = 0; i < cell_count; ++i) { auto hash = cell_list[cell_count - 1 - i]->first; const auto& dc_info = cell_list[cell_count - 1 - i]->second; @@ -412,14 +417,8 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { DCHECK(k > i && k < cell_count); store_ref(k); } - ++processed_cells_; - if (processed_cells_ % 1000 == 0) { - TRY_STATUS(cancellation_token.check()); - } - if (log_speed_at_.is_in_past()) { - log_speed_at_ += LOG_SPEED_PERIOD; - LOG(WARNING) << "serializer: serialize " << (double)processed_cells_ / LOG_SPEED_PERIOD << " cells/s"; - processed_cells_ = 0; + if (logger_ptr_) { + TRY_STATUS(logger_ptr_->on_cell_processed()); } } DCHECK(writer.position() - keep_position == info.data_size); @@ -429,8 +428,9 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { } DCHECK(writer.empty()); TRY_STATUS(writer.finalize()); - LOG(ERROR) << "serializer: serialize took " << timer.elapsed() << "s, " << cell_count << " cells, " - << writer.position() << " bytes"; + if (logger_ptr_) { + logger_ptr_->finish_stage(PSLICE() << cell_count << " cells, " << writer.position() << " bytes"); + } return td::Status::OK(); } } // namespace @@ -439,7 +439,9 @@ td::Status std_boc_serialize_to_file_large(std::shared_ptr reader, int mode, td::CancellationToken cancellation_token) { td::Timer timer; CHECK(reader != nullptr) - LargeBocSerializer serializer(reader, std::move(cancellation_token)); + LargeBocSerializer serializer(reader); + BagOfCellsLogger logger(std::move(cancellation_token)); + serializer.set_logger(&logger); serializer.add_root(root_hash); TRY_STATUS(serializer.import_cells()); TRY_STATUS(serializer.serialize(fd, mode)); diff --git a/emulator/CMakeLists.txt b/emulator/CMakeLists.txt index 5d59bcdbf..4b6383005 100644 --- a/emulator/CMakeLists.txt +++ b/emulator/CMakeLists.txt @@ -65,4 +65,4 @@ if (USE_EMSCRIPTEN) target_compile_options(emulator-emscripten PRIVATE -fexceptions) endif() -install(TARGETS emulator LIBRARY DESTINATION lib) +install(TARGETS emulator ARCHIVE DESTINATION lib LIBRARY DESTINATION lib) diff --git a/rldp2/RldpConnection.cpp b/rldp2/RldpConnection.cpp index c6f967282..47506178e 100644 --- a/rldp2/RldpConnection.cpp +++ b/rldp2/RldpConnection.cpp @@ -170,6 +170,9 @@ td::Timestamp RldpConnection::run(ConnectionCallback &callback) { if (in_flight_count_ > congestion_window_) { bdw_stats_.on_pause(now); } + if (in_flight_count_ == 0) { + bdw_stats_.on_pause(now); + } for (auto &inbound : inbound_transfers_) { alarm_timestamp.relax(run(inbound.first, inbound.second)); diff --git a/tdactor/CMakeLists.txt b/tdactor/CMakeLists.txt index 46dd03356..98b900a1e 100644 --- a/tdactor/CMakeLists.txt +++ b/tdactor/CMakeLists.txt @@ -3,16 +3,19 @@ cmake_minimum_required(VERSION 3.5 FATAL_ERROR) #SOURCE SETS set(TDACTOR_SOURCE td/actor/core/ActorExecutor.cpp + td/actor/core/ActorTypeStat.cpp td/actor/core/CpuWorker.cpp td/actor/core/IoWorker.cpp td/actor/core/Scheduler.cpp + td/actor/ActorStats.cpp td/actor/MultiPromise.cpp td/actor/actor.h td/actor/ActorId.h td/actor/ActorOwn.h td/actor/ActorShared.h + td/actor/ActorStats.h td/actor/common.h td/actor/PromiseFuture.h td/actor/MultiPromise.h @@ -27,6 +30,7 @@ set(TDACTOR_SOURCE td/actor/core/ActorMessage.h td/actor/core/ActorSignals.h td/actor/core/ActorState.h + td/actor/core/ActorTypeStat.h td/actor/core/CpuWorker.h td/actor/core/Context.h td/actor/core/IoWorker.h diff --git a/tdactor/td/actor/ActorStats.cpp b/tdactor/td/actor/ActorStats.cpp new file mode 100644 index 000000000..ad037204b --- /dev/null +++ b/tdactor/td/actor/ActorStats.cpp @@ -0,0 +1,245 @@ +#include "ActorStats.h" + +#include "td/utils/ThreadSafeCounter.h" +namespace td { +namespace actor { +void td::actor::ActorStats::start_up() { + auto now = td::Time::now(); + for (std::size_t i = 0; i < SIZE; i++) { + stat_[i] = td::TimedStat>(DURATIONS[i], now); + stat_[i].add_event(ActorTypeStats(), now); + } + begin_ts_ = td::Timestamp::now(); + begin_ticks_ = Clocks::rdtsc(); + loop(); +} +double ActorStats::estimate_inv_ticks_per_second() { + auto now = td::Timestamp::now(); + auto elapsed_seconds = now.at() - begin_ts_.at(); + auto now_ticks = td::Clocks::rdtsc(); + auto elapsed_ticks = now_ticks - begin_ticks_; + auto estimated_inv_ticks_per_second = + elapsed_seconds > 0.1 ? elapsed_seconds / double(elapsed_ticks) : Clocks::inv_ticks_per_second(); + return estimated_inv_ticks_per_second; +} + +std::string ActorStats::prepare_stats() { + auto estimated_inv_ticks_per_second = estimate_inv_ticks_per_second(); + + auto current_stats = td::actor::ActorTypeStatManager::get_stats(estimated_inv_ticks_per_second); + auto now = td::Timestamp::now(); + auto now_ticks = Clocks::rdtsc(); + + update(now); + + // Lets look at recent stats first + auto load_stats = [&](auto &timed_stat) { + auto res = current_stats; + auto &since = timed_stat.get_stat(now.at()); + auto duration = since.get_duration(estimated_inv_ticks_per_second); + if (since.first_) { + res -= since.first_.value(); + } + res /= duration; + return res.stats; + }; + auto stats_10s = load_stats(stat_[0]); + auto stats_10m = load_stats(stat_[1]); + current_stats /= double(now_ticks - begin_ticks_) * estimated_inv_ticks_per_second; + auto stats_forever = current_stats.stats; + + std::map current_perf_map; + std::map perf_map_10s; + std::map perf_map_10m; + std::map perf_values; + td::NamedPerfCounter::get_default().for_each( + [&](td::Slice name, td::int64 value_int64) { perf_values[name.str()] = double(value_int64); }); + for (auto &value_it : perf_values) { + const auto &name = value_it.first; + auto value = value_it.second; + + auto &perf_stat = pef_stats_[name]; + auto load_perf_stats = [&](auto &timed_stat, auto &m) { + double res = double(value); + auto &since = timed_stat.get_stat(now.at()); + auto duration = since.get_duration(estimated_inv_ticks_per_second); + if (since.first_) { + res -= since.first_.value(); + } + if (td::ends_with(name, ".duration")) { + res *= estimated_inv_ticks_per_second; + } + // m[name + ".raw"] = res; + // m[name + ".range"] = duration; + res /= duration; + return res; + }; + perf_map_10s[name] = load_perf_stats(perf_stat.perf_stat_[0], perf_map_10s); + perf_map_10m[name] = load_perf_stats(perf_stat.perf_stat_[1], perf_map_10m); + + auto current_duration = (double(now_ticks - begin_ticks_) * estimated_inv_ticks_per_second); + if (td::ends_with(name, ".duration")) { + value *= estimated_inv_ticks_per_second; + } + current_perf_map[name] = double(value) / current_duration; + // current_perf_map[name + ".raw"] = double(value); + // current_perf_map[name + ".range"] = double(now_ticks - begin_ticks_) * estimated_inv_ticks_per_second; + }; + + td::StringBuilder sb; + sb << "================================= PERF COUNTERS ================================\n"; + sb << "ticks_per_second_estimate\t" << 1.0 / estimated_inv_ticks_per_second << "\n"; + for (auto &it : perf_map_10s) { + const std::string &name = it.first; + auto dot_at = name.rfind('.'); + CHECK(dot_at != std::string::npos); + auto base_name = name.substr(0, dot_at); + auto rest_name = name.substr(dot_at + 1); + td::Slice new_rest_name = rest_name; + if (rest_name == "count") { + new_rest_name = "qps"; + } + if (rest_name == "duration") { + new_rest_name = "load"; + } + auto rewrite_name = PSTRING() << base_name << "." << new_rest_name; + sb << rewrite_name << "\t" << perf_map_10s[name] << " " << perf_map_10m[name] << " " << current_perf_map[name] + << "\n"; + } + sb << "\n"; + sb << "================================= ACTORS STATS =================================\n"; + double max_delay = 0; + ActorTypeStat sum_stat_forever; + ActorTypeStat sum_stat_10m; + ActorTypeStat sum_stat_10s; + for (auto &it : stats_forever) { + sum_stat_forever += it.second; + } + for (auto &it : stats_10m) { + sum_stat_10m += it.second; + } + for (auto &it : stats_10s) { + sum_stat_10s += it.second; + } + sb << "\n"; + + auto do_describe = [&](auto &&sb, const ActorTypeStat &stat_10s, const ActorTypeStat &stat_10m, + const ActorTypeStat &stat_forever) { + sb() << "load_per_second:\t" << stat_10s.seconds << " " << stat_10m.seconds << " " << stat_forever.seconds << "\n"; + sb() << "messages_per_second:\t" << stat_10s.messages << " " << stat_10m.messages << " " << stat_forever.messages + << "\n"; + + sb() << "max_execute_messages:\t" << stat_forever.max_execute_messages.value_10s << " " + << stat_forever.max_execute_messages.value_10m << " " << stat_forever.max_execute_messages.value_forever + << "\n"; + + sb() << "max_execute_seconds:\t" << stat_forever.max_execute_seconds.value_10s << "s" + << " " << stat_forever.max_execute_seconds.value_10m << "s" + << " " << stat_forever.max_execute_seconds.value_forever << "s\n"; + sb() << "max_message_seconds:\t" << stat_forever.max_message_seconds.value_10s << " " + << stat_forever.max_message_seconds.value_10m << " " << stat_forever.max_message_seconds.value_forever << "\n"; + sb() << "created_per_second:\t" << stat_10s.created << " " << stat_10m.created << " " << stat_forever.created + << "\n"; + + auto executing_for = + stat_forever.executing_start > 1e15 + ? 0 + : double(td::Clocks::rdtsc()) * estimated_inv_ticks_per_second - stat_forever.executing_start; + sb() << "max_delay:\t" << stat_forever.max_delay_seconds.value_10s << "s " + << stat_forever.max_delay_seconds.value_10m << "s " << stat_forever.max_delay_seconds.value_forever << "s\n"; + sb() << "" + << "alive: " << stat_forever.alive << " executing: " << stat_forever.executing + << " max_executing_for: " << executing_for << "s\n"; + }; + + auto describe = [&](td::StringBuilder &sb, std::type_index actor_type_index) { + auto stat_10s = stats_10s[actor_type_index]; + auto stat_10m = stats_10m[actor_type_index]; + auto stat_forever = stats_forever[actor_type_index]; + do_describe([&sb]() -> td::StringBuilder & { return sb << "\t\t"; }, stat_10s, stat_10m, stat_forever); + }; + + sb << "Cummulative stats:\n"; + do_describe([&sb]() -> td::StringBuilder & { return sb << "\t"; }, sum_stat_10s, sum_stat_10m, sum_stat_forever); + sb << "\n"; + + auto top_k_by = [&](auto &stats_map, size_t k, std::string description, auto by) { + auto stats = td::transform(stats_map, [](auto &it) { return std::make_pair(it.first, it.second); }); + k = std::min(k, stats.size()); + std::partial_sort(stats.begin(), stats.begin() + k, stats.end(), [&](auto &a, auto &b) { return by(a) > by(b); }); + bool is_first = true; + for (size_t i = 0; i < k; i++) { + auto value = by(stats[i]); + if (value < 1e-9) { + break; + } + if (is_first) { + sb << "top actors by " << description << "\n"; + is_first = false; + } + sb << "\t#" << i << ": " << ActorTypeStatManager::get_class_name(stats[i].first.name()) << "\t" << value << "\n"; + } + sb << "\n"; + }; + using Entry = std::pair; + static auto cutoff = [](auto x, auto min_value) { return x < min_value ? decltype(x){} : x; }; + top_k_by(stats_10s, 10, "load_10s", [](auto &x) { return cutoff(x.second.seconds, 0.005); }); + + top_k_by(stats_10m, 10, "load_10m", [](auto &x) { return cutoff(x.second.seconds, 0.005); }); + top_k_by(stats_forever, 10, "max_execute_seconds_10m", + [](Entry &x) { return cutoff(x.second.max_execute_seconds.value_10m, 0.5); }); + auto rdtsc_seconds = double(now_ticks) * estimated_inv_ticks_per_second; + top_k_by(stats_forever, 10, "executing_for", [&](Entry &x) { + if (x.second.executing_start > 1e15) { + return 0.0; + } + return rdtsc_seconds - x.second.executing_start; + }); + top_k_by(stats_forever, 10, "max_execute_messages_10m", + [](Entry &x) { return cutoff(x.second.max_execute_messages.value_10m, 10u); }); + + auto stats = td::transform(stats_forever, [](auto &it) { return std::make_pair(it.first, it.second); }); + + auto main_key = [&](std::type_index actor_type_index) { + auto stat_10s = stats_10s[actor_type_index]; + auto stat_10m = stats_10m[actor_type_index]; + auto stat_forever = stats_forever[actor_type_index]; + return std::make_tuple(cutoff(std::max(stat_10s.seconds, stat_10m.seconds), 0.1), + cutoff(stat_forever.max_execute_seconds.value_10m, 0.5), stat_forever.seconds); + }; + std::sort(stats.begin(), stats.end(), + [&](auto &left, auto &right) { return main_key(left.first) > main_key(right.first); }); + auto debug = Debug(SchedulerContext::get()->scheduler_group()); + debug.dump(sb); + sb << "All actors:\n"; + for (auto &it : stats) { + sb << "\t" << ActorTypeStatManager::get_class_name(it.first.name()) << "\n"; + auto key = main_key(it.first); + describe(sb, it.first); + } + sb << "\n"; + return sb.as_cslice().str(); +} +ActorStats::PefStat::PefStat() { + for (std::size_t i = 0; i < SIZE; i++) { + perf_stat_[i] = td::TimedStat>(DURATIONS[i], td::Time::now()); + perf_stat_[i].add_event(0, td::Time::now()); + } +} + +void ActorStats::update(td::Timestamp now) { + auto stat = td::actor::ActorTypeStatManager::get_stats(estimate_inv_ticks_per_second()); + for (auto &timed_stat : stat_) { + timed_stat.add_event(stat, now.at()); + } + NamedPerfCounter::get_default().for_each([&](td::Slice name, td::int64 value) { + auto &stat = pef_stats_[name.str()].perf_stat_; + for (auto &timed_stat : stat) { + timed_stat.add_event(value, now.at()); + } + }); +} +constexpr int ActorStats::DURATIONS[SIZE]; +constexpr const char *ActorStats::DESCR[SIZE]; +} // namespace actor +} // namespace td diff --git a/tdactor/td/actor/ActorStats.h b/tdactor/td/actor/ActorStats.h new file mode 100644 index 000000000..00f7ebeb7 --- /dev/null +++ b/tdactor/td/actor/ActorStats.h @@ -0,0 +1,52 @@ +#pragma once +#include "td/actor/actor.h" +#include "td/utils/TimedStat.h" +namespace td { +namespace actor { + +class ActorStats : public td::actor::Actor { + public: + ActorStats() { + } + void start_up() override; + double estimate_inv_ticks_per_second(); + std::string prepare_stats(); + + private: + template + struct StatStorer { + void on_event(const T &event) { + if (!first_) { + first_ = event; + first_ts_ = Clocks::rdtsc(); + } + } + double get_duration(double inv_ticks_per_second) const { + if (first_) { + return std::max(1.0, (Clocks::rdtsc() - first_ts_) * inv_ticks_per_second); + } + return 1.0; + } + td::optional first_; + td::uint64 first_ts_; + }; + static constexpr std::size_t SIZE = 2; + static constexpr const char *DESCR[SIZE] = {"10sec", "10min"}; + static constexpr int DURATIONS[SIZE] = {10, 10 * 60}; + td::TimedStat> stat_[SIZE]; + struct PefStat { + PefStat(); + td::TimedStat> perf_stat_[SIZE]; + }; + std::map pef_stats_; + td::Timestamp begin_ts_; + td::uint64 begin_ticks_{}; + void loop() override { + alarm_timestamp() = td::Timestamp::in(5.0); + update(td::Timestamp::now()); + } + void update(td::Timestamp now); +}; + +} // namespace actor +} // namespace td diff --git a/tdactor/td/actor/common.h b/tdactor/td/actor/common.h index d38589747..222db7436 100644 --- a/tdactor/td/actor/common.h +++ b/tdactor/td/actor/common.h @@ -19,6 +19,7 @@ #pragma once #include "td/actor/core/Actor.h" #include "td/actor/core/ActorSignals.h" +#include "td/actor/core/ActorTypeStat.h" #include "td/actor/core/SchedulerId.h" #include "td/actor/core/SchedulerContext.h" #include "td/actor/core/Scheduler.h" @@ -68,7 +69,7 @@ using core::set_debug; struct Debug { public: Debug() = default; - Debug(std::shared_ptr group_info) : group_info_(std::move(group_info)) { + Debug(core::SchedulerGroupInfo *group_info) : group_info_(group_info) { } template void for_each(F &&f) { @@ -80,18 +81,29 @@ struct Debug { } } - void dump() { - for_each([](core::Debug &debug) { + void dump(td::StringBuilder &sb) { + sb << "list of active actors with names:\n"; + for_each([&](core::Debug &debug) { core::DebugInfo info; debug.read(info); if (info.is_active) { - LOG(ERROR) << info.name << " " << td::format::as_time(Time::now() - info.start_at); + sb << "\t\"" << info.name << "\" is active for " << Time::now() - info.start_at << "s\n"; } }); + sb << "\nsizes of cpu local queues:\n"; + for (auto &scheduler : group_info_->schedulers) { + for (size_t i = 0; i < scheduler.cpu_threads_count; i++) { + auto size = scheduler.cpu_local_queue[i].size(); + if (size != 0) { + sb << "\tcpu#" << i << " queue.size() = " << size << "\n"; + } + } + } + sb << "\n"; } private: - std::shared_ptr group_info_; + core::SchedulerGroupInfo *group_info_; }; class Scheduler { @@ -142,7 +154,7 @@ class Scheduler { } Debug get_debug() { - return Debug{group_info_}; + return Debug{group_info_.get()}; } bool run() { @@ -200,6 +212,10 @@ class Scheduler { } }; +using core::ActorTypeStat; +using core::ActorTypeStatManager; +using core::ActorTypeStats; + // Some helper functions. Not part of public interface and not part // of namespace core namespace detail { @@ -324,7 +340,7 @@ void send_closure_impl(ActorRef actor_ref, ClosureT &&closure) { } template -void send_closure(ActorRef actor_ref, ArgsT &&... args) { +void send_closure(ActorRef actor_ref, ArgsT &&...args) { send_closure_impl(actor_ref, create_immediate_closure(std::forward(args)...)); } @@ -365,7 +381,7 @@ void send_closure_with_promise_later(ActorRef actor_ref, ClosureT &&closure, Pro } template -void send_closure_later(ActorRef actor_ref, ArgsT &&... args) { +void send_closure_later(ActorRef actor_ref, ArgsT &&...args) { send_closure_later_impl(actor_ref, create_delayed_closure(std::forward(args)...)); } @@ -396,15 +412,17 @@ inline void send_signals_later(ActorRef actor_ref, ActorSignals signals) { inline void register_actor_info_ptr(core::ActorInfoPtr actor_info_ptr) { auto state = actor_info_ptr->state().get_flags_unsafe(); + actor_info_ptr->on_add_to_queue(); core::SchedulerContext::get()->add_to_queue(std::move(actor_info_ptr), state.get_scheduler_id(), !state.is_shared()); } template -core::ActorInfoPtr create_actor(core::ActorOptions &options, ArgsT &&... args) noexcept { +core::ActorInfoPtr create_actor(core::ActorOptions &options, ArgsT &&...args) noexcept { auto *scheduler_context = core::SchedulerContext::get(); if (!options.has_scheduler()) { options.on_scheduler(scheduler_context->get_scheduler_id()); } + options.with_actor_stat_id(core::ActorTypeStatImpl::get_unique_id()); auto res = scheduler_context->get_actor_info_creator().create(std::make_unique(std::forward(args)...), options); register_actor_info_ptr(res); diff --git a/tdactor/td/actor/core/ActorExecutor.cpp b/tdactor/td/actor/core/ActorExecutor.cpp index 267758d5e..d15422424 100644 --- a/tdactor/td/actor/core/ActorExecutor.cpp +++ b/tdactor/td/actor/core/ActorExecutor.cpp @@ -114,6 +114,8 @@ void ActorExecutor::start() noexcept { actor_execute_context_.set_actor(&actor_info_.actor()); + actor_stats_ = actor_info_.actor_type_stat(); + auto execute_timer = actor_stats_.create_execute_timer(); while (flush_one_signal(signals)) { if (actor_execute_context_.has_immediate_flags()) { return; @@ -175,6 +177,11 @@ void ActorExecutor::finish() noexcept { if (add_to_queue) { actor_info_ptr = actor_info_.actor().get_actor_info_ptr(); } + if (!flags().is_closed() && flags().is_in_queue()) { + // Must do it while we are locked, so to it optimistically + // we will add actor to queue after unlock OR we are already in a queue OR we will be closed + actor_info_.on_add_to_queue(); + } if (actor_locker_.try_unlock(flags())) { if (add_to_queue) { dispatcher_.add_to_queue(std::move(actor_info_ptr), flags().get_scheduler_id(), !flags().is_shared()); @@ -193,23 +200,31 @@ bool ActorExecutor::flush_one_signal(ActorSignals &signals) { } switch (signal) { //NB: Signals will be handled in order of their value. - // For clarity it conincides with order in this switch + // For clarity, it coincides with order in this switch case ActorSignals::Pause: actor_execute_context_.set_pause(); break; - case ActorSignals::Kill: + case ActorSignals::Kill: { + auto message_timer = actor_stats_.create_message_timer(); actor_execute_context_.set_stop(); break; - case ActorSignals::StartUp: + } + case ActorSignals::StartUp: { + auto message_timer = actor_stats_.create_message_timer(); + actor_stats_.created(); actor_info_.actor().start_up(); break; - case ActorSignals::Wakeup: + } + case ActorSignals::Wakeup: { + auto message_timer = actor_stats_.create_message_timer(); actor_info_.actor().wake_up(); break; + } case ActorSignals::Alarm: if (actor_execute_context_.get_alarm_timestamp() && actor_execute_context_.get_alarm_timestamp().is_in_past()) { actor_execute_context_.alarm_timestamp() = Timestamp::never(); actor_info_.set_alarm_timestamp(Timestamp::never()); + auto message_timer = actor_stats_.create_message_timer(); actor_info_.actor().alarm(); } break; @@ -245,6 +260,7 @@ bool ActorExecutor::flush_one_message() { } actor_execute_context_.set_link_token(message.get_link_token()); + auto message_timer = actor_stats_.create_message_timer(); message.run(); return true; } @@ -257,7 +273,9 @@ void ActorExecutor::flush_context_flags() { } flags_.set_closed(true); if (!flags_.get_signals().has_signal(ActorSignals::Signal::StartUp)) { + auto message_timer = actor_stats_.create_message_timer(); actor_info_.actor().tear_down(); + actor_stats_.destroyed(); } actor_info_.destroy_actor(); } else { diff --git a/tdactor/td/actor/core/ActorExecutor.h b/tdactor/td/actor/core/ActorExecutor.h index 366cb754d..dd86b5ae2 100644 --- a/tdactor/td/actor/core/ActorExecutor.h +++ b/tdactor/td/actor/core/ActorExecutor.h @@ -24,6 +24,7 @@ #include "td/actor/core/ActorMessage.h" #include "td/actor/core/ActorSignals.h" #include "td/actor/core/ActorState.h" +#include "td/actor/core/ActorTypeStat.h" #include "td/actor/core/SchedulerContext.h" #include "td/utils/format.h" @@ -95,6 +96,7 @@ class ActorExecutor { ActorInfo &actor_info_; SchedulerDispatcher &dispatcher_; Options options_; + ActorTypeStatRef actor_stats_; ActorLocker actor_locker_{&actor_info_.state(), ActorLocker::Options() .with_can_execute_paused(options_.from_queue) .with_is_shared(!options_.has_poll) diff --git a/tdactor/td/actor/core/ActorInfo.h b/tdactor/td/actor/core/ActorInfo.h index 974192938..690b06d9f 100644 --- a/tdactor/td/actor/core/ActorInfo.h +++ b/tdactor/td/actor/core/ActorInfo.h @@ -19,6 +19,7 @@ #pragma once #include "td/actor/core/ActorState.h" +#include "td/actor/core/ActorTypeStat.h" #include "td/actor/core/ActorMailbox.h" #include "td/utils/Heap.h" @@ -34,8 +35,8 @@ class ActorInfo; using ActorInfoPtr = SharedObjectPool::Ptr; class ActorInfo : private HeapNode, private ListNode { public: - ActorInfo(std::unique_ptr actor, ActorState::Flags state_flags, Slice name) - : actor_(std::move(actor)), name_(name.begin(), name.size()) { + ActorInfo(std::unique_ptr actor, ActorState::Flags state_flags, Slice name, td::uint32 actor_stat_id) + : actor_(std::move(actor)), name_(name.begin(), name.size()), actor_stat_id_(actor_stat_id) { state_.set_flags_unsafe(state_flags); VLOG(actor) << "Create actor [" << name_ << "]"; } @@ -58,6 +59,18 @@ class ActorInfo : private HeapNode, private ListNode { Actor *actor_ptr() const { return actor_.get(); } + // NB: must be called only when actor is locked + ActorTypeStatRef actor_type_stat() { + auto res = ActorTypeStatManager::get_actor_type_stat(actor_stat_id_, actor_.get()); + if (in_queue_since_) { + res.pop_from_queue(in_queue_since_); + in_queue_since_ = 0; + } + return res; + } + void on_add_to_queue() { + in_queue_since_ = td::Clocks::rdtsc(); + } void destroy_actor() { actor_.reset(); } @@ -103,6 +116,8 @@ class ActorInfo : private HeapNode, private ListNode { std::atomic alarm_timestamp_at_{0}; ActorInfoPtr pin_; + td::uint64 in_queue_since_{0}; + td::uint32 actor_stat_id_{0}; }; } // namespace core diff --git a/tdactor/td/actor/core/ActorInfoCreator.h b/tdactor/td/actor/core/ActorInfoCreator.h index 49c7611ae..d818dc631 100644 --- a/tdactor/td/actor/core/ActorInfoCreator.h +++ b/tdactor/td/actor/core/ActorInfoCreator.h @@ -46,10 +46,16 @@ class ActorInfoCreator { return *this; } + Options& with_actor_stat_id(td::uint32 new_id) { + actor_stat_id = new_id; + return *this; + } + private: friend class ActorInfoCreator; Slice name; SchedulerId scheduler_id; + td::uint32 actor_stat_id{0}; bool is_shared{true}; bool in_queue{true}; //TODO: rename @@ -65,7 +71,7 @@ class ActorInfoCreator { flags.set_in_queue(args.in_queue); flags.set_signals(ActorSignals::one(ActorSignals::StartUp)); - auto actor_info_ptr = pool_.alloc(std::move(actor), flags, args.name); + auto actor_info_ptr = pool_.alloc(std::move(actor), flags, args.name, args.actor_stat_id); actor_info_ptr->actor().set_actor_info_ptr(actor_info_ptr); return actor_info_ptr; } diff --git a/tdactor/td/actor/core/ActorTypeStat.cpp b/tdactor/td/actor/core/ActorTypeStat.cpp new file mode 100644 index 000000000..4de6f1054 --- /dev/null +++ b/tdactor/td/actor/core/ActorTypeStat.cpp @@ -0,0 +1,124 @@ +#include "td/actor/core/Actor.h" +#include "td/actor/core/ActorTypeStat.h" +#include "td/actor/core/Scheduler.h" +#include "td/utils/port/thread_local.h" +#include +#include +#include +#include +#include +#include + +#ifdef __has_include +#if __has_include() +#include +#define CXXABI_AVAILABLE 1 +#else +#define CXXABI_AVAILABLE 0 +#endif +#else +#define CXXABI_AVAILABLE 0 +#endif + +namespace td { +namespace actor { +namespace core { + +class ActorTypeStatRef; +struct ActorTypeStatsTlsEntry { + struct Entry { + std::unique_ptr stat; + std::optional o_type_index; + }; + std::vector by_id; + std::mutex mutex; + + template + void foreach_entry(F &&f) { + std::lock_guard guard(mutex); + for (auto &entry : by_id) { + f(entry); + } + } + ActorTypeStatRef get_actor_type_stat(td::uint32 id, Actor &actor) { + if (id >= by_id.size()) { + std::lock_guard guard(mutex); + by_id.resize(id + 1); + } + auto &entry = by_id.at(id); + if (!entry.o_type_index) { + std::lock_guard guard(mutex); + entry.o_type_index = std::type_index(typeid(actor)); + entry.stat = std::make_unique(); + } + return ActorTypeStatRef{entry.stat.get()}; + } +}; + +struct ActorTypeStatsRegistry { + std::mutex mutex; + std::vector> entries; + void registry_entry(std::shared_ptr entry) { + std::lock_guard guard(mutex); + entries.push_back(std::move(entry)); + } + template + void foreach_entry(F &&f) { + std::lock_guard guard(mutex); + for (auto &entry : entries) { + f(*entry); + } + } +}; + +ActorTypeStatsRegistry registry; + +struct ActorTypeStatsTlsEntryRef { + ActorTypeStatsTlsEntryRef() { + entry_ = std::make_shared(); + registry.registry_entry(entry_); + } + std::shared_ptr entry_; +}; + +static TD_THREAD_LOCAL ActorTypeStatsTlsEntryRef *actor_type_stats_tls_entry = nullptr; + +ActorTypeStatRef ActorTypeStatManager::get_actor_type_stat(td::uint32 id, Actor *actor) { + if (!actor || !need_debug()) { + return ActorTypeStatRef{nullptr}; + } + td::init_thread_local(actor_type_stats_tls_entry); + ActorTypeStatsTlsEntry &tls_entry = *actor_type_stats_tls_entry->entry_; + return tls_entry.get_actor_type_stat(id, *actor); +} + +std::string ActorTypeStatManager::get_class_name(const char *name) { +#if CXXABI_AVAILABLE + int status; + char *real_name = abi::__cxa_demangle(name, nullptr, nullptr, &status); + if (status < 0) { + return name; + } + + std::string result = real_name; + std::free(real_name); + return result; +#else + return name; +#endif +} + +ActorTypeStats ActorTypeStatManager::get_stats(double inv_ticks_per_second) { + std::map stats; + registry.foreach_entry([&](ActorTypeStatsTlsEntry &tls_entry) { + tls_entry.foreach_entry([&](ActorTypeStatsTlsEntry::Entry &entry) { + if (entry.o_type_index) { + stats[entry.o_type_index.value()] += entry.stat->to_stat(inv_ticks_per_second); + } + }); + }); + return ActorTypeStats{.stats = std::move(stats)}; +} +} // namespace core +} // namespace actor +} // namespace td diff --git a/tdactor/td/actor/core/ActorTypeStat.h b/tdactor/td/actor/core/ActorTypeStat.h new file mode 100644 index 000000000..5b0bd18dc --- /dev/null +++ b/tdactor/td/actor/core/ActorTypeStat.h @@ -0,0 +1,395 @@ +#pragma once +#include "td/utils/int_types.h" +#include "td/utils/port/Clocks.h" +#include +#include +#include + +namespace td { +namespace actor { +namespace core { +class Actor; + +struct ActorTypeStat { + // diff (speed) + double created{0}; + double executions{0}; + double messages{0}; + double seconds{0}; + + // current statistics + td::int64 alive{0}; + td::int32 executing{0}; + double executing_start{1e20}; + + // max statistics (TODO: recent_max) + template + struct MaxStatGroup { + T value_forever{}; + T value_10s{}; + T value_10m{}; + MaxStatGroup &operator+=(const MaxStatGroup &other) { + value_forever = std::max(value_forever, other.value_forever); + value_10s = std::max(value_10s, other.value_10s); + value_10m = std::max(value_10m, other.value_10m); + return *this; + } + }; + MaxStatGroup max_execute_messages; + MaxStatGroup max_message_seconds; + MaxStatGroup max_execute_seconds; + MaxStatGroup max_delay_seconds; + + ActorTypeStat &operator+=(const ActorTypeStat &other) { + created += other.created; + executions += other.executions; + messages += other.messages; + seconds += other.seconds; + + alive += other.alive; + executing += other.executing; + executing_start = std::min(other.executing_start, executing_start); + + max_execute_messages += other.max_execute_messages; + max_message_seconds += other.max_message_seconds; + max_execute_seconds += other.max_execute_seconds; + max_delay_seconds += other.max_delay_seconds; + return *this; + } + + ActorTypeStat &operator-=(const ActorTypeStat &other) { + created -= other.created; + executions -= other.executions; + messages -= other.messages; + seconds -= other.seconds; + return *this; + } + ActorTypeStat &operator/=(double t) { + if (t > 1e-2) { + created /= t; + executions /= t; + messages /= t; + seconds /= t; + } else { + created = 0; + executions = 0; + messages = 0; + seconds = 0; + } + return *this; + } +}; + +struct ActorTypeStatImpl { + public: + ActorTypeStatImpl() { + } + + class MessageTimer { + public: + MessageTimer(ActorTypeStatImpl *stat, td::uint64 started_at = Clocks::rdtsc()) + : stat_(stat), started_at_(started_at) { + } + MessageTimer(const MessageTimer &) = delete; + MessageTimer(MessageTimer &&) = delete; + MessageTimer &operator=(const MessageTimer &) = delete; + MessageTimer &operator=(MessageTimer &&) = delete; + ~MessageTimer() { + if (stat_) { + auto ts = td::Clocks::rdtsc(); + stat_->message_finish(ts, ts - started_at_); + } + } + + private: + ActorTypeStatImpl *stat_; + td::uint64 started_at_; + }; + void created() { + inc(total_created_); + inc(alive_); + } + void destroyed() { + dec(alive_); + } + MessageTimer create_run_timer() { + return MessageTimer{this}; + } + + void message_finish(td::uint64 ts, td::uint64 ticks) { + inc(total_messages_); + inc(execute_messages_); + add(total_ticks_, ticks); + max_message_ticks_.update(ts, ticks); + } + void on_delay(td::uint64 ts, td::uint64 ticks) { + max_delay_ticks_.update(ts, ticks); + } + + void execute_start(td::uint64 ts) { + // TODO: this is mostly protection for recursive actor calls, which curretly should be almost impossible + // But too full handle it, one would use one executing_cnt per thread, so only upper level execution is counted + if (inc(executing_) == 1) { + store(execute_start_, ts); + store(execute_messages_, 0); + } + } + void execute_finish(td::uint64 ts) { + CHECK(executing_ > 0); + if (dec(executing_) == 0) { + max_execute_messages_.update(ts, load(execute_messages_)); + max_execute_ticks_.update(ts, ts - load(execute_start_)); + + inc(total_executions_); + store(execute_start_, 0); + store(execute_messages_, 0); + } + } + + template + static td::uint32 get_unique_id() { + static td::uint32 value = get_next_unique_id(); + return value; + } + + static td::uint32 get_next_unique_id() { + static std::atomic next_id_{}; + return ++next_id_; + } + ActorTypeStat to_stat(double inv_ticks_per_second) const { + auto execute_start_copy = load(execute_start_); + auto actual_total_ticks = load(total_ticks_); + auto ts = Clocks::rdtsc(); + if (execute_start_copy != 0) { + actual_total_ticks += ts - execute_start_copy; + } + auto execute_start = ticks_to_seconds(load(execute_start_), inv_ticks_per_second); + return ActorTypeStat{.created = double(load(total_created_)), + .executions = double(load(total_executions_)), + .messages = double(load(total_messages_)), + .seconds = ticks_to_seconds(actual_total_ticks, inv_ticks_per_second), + + .alive = load(alive_), + .executing = load(executing_), + .executing_start = execute_start < 1e-9 ? 1e20 : execute_start, + .max_execute_messages = load(max_execute_messages_), + .max_message_seconds = load_seconds(max_message_ticks_, inv_ticks_per_second), + .max_execute_seconds = load_seconds(max_execute_ticks_, inv_ticks_per_second), + .max_delay_seconds = load_seconds(max_delay_ticks_, inv_ticks_per_second)}; + } + + private: + static double ticks_to_seconds(td::uint64 ticks, double inv_tick_per_second) { + return double(ticks) * inv_tick_per_second; + } + + template + static T load(const std::atomic &a) { + return a.load(std::memory_order_relaxed); + } + template + static void store(std::atomic &a, S value) { + a.store(value, std::memory_order_relaxed); + } + template + static T add(std::atomic &a, S value) { + T new_value = load(a) + value; + store(a, new_value); + return new_value; + } + template + static T inc(std::atomic &a) { + return add(a, 1); + } + + template + static T dec(std::atomic &a) { + return add(a, -1); + } + template + static void relax_max(std::atomic &a, T value) { + auto old_value = load(a); + if (value > old_value) { + store(a, value); + } + } + + template + class MaxCounter { + alignas(64) std::atomic max_values[2] = {0}; + std::atomic last_update_segment_time = 0; + + void update_current_segment(uint64 current_segment_time, uint64 segment_difference) { + if (segment_difference >= 2) { + store(max_values[0], 0); + store(max_values[1], 0); + } else if (segment_difference == 1) { + store(max_values[1 - (current_segment_time & 1)], 0); + } + store(last_update_segment_time, current_segment_time); + } + + public: + inline void update(td::uint64 rdtsc, ValueT value) { + auto current_segment_time = rdtsc / (Clocks::rdtsc_frequency() * Interval); + + auto segment_difference = current_segment_time - last_update_segment_time; + + if (unlikely(segment_difference != 0)) { + update_current_segment(current_segment_time, segment_difference); + } + + relax_max(max_values[current_segment_time & 1], value); + } + + inline ValueT get_max(uint64_t rdtsc) const { + uint64_t current_segment_time = rdtsc / (Clocks::rdtsc_frequency() * Interval); + uint64_t segment_difference = current_segment_time - load(last_update_segment_time); + + if (segment_difference >= 2) { + return 0; + } else if (segment_difference == 1) { + return load(max_values[current_segment_time & 1]); + } else { + return std::max(load(max_values[0]), load(max_values[1])); + } + } + }; + + template + struct MaxCounterGroup { + std::atomic max_forever{}; + MaxCounter max_10m; + MaxCounter max_10s; + + inline void update(td::uint64 rdtsc, T value) { + relax_max(max_forever, value); + max_10m.update(rdtsc, value); + max_10s.update(rdtsc, value); + } + }; + template + static ActorTypeStat::MaxStatGroup load(const MaxCounterGroup &src) { + auto ts = Clocks::rdtsc(); + return {.value_forever = load(src.max_forever), + .value_10s = src.max_10s.get_max(ts), + .value_10m = src.max_10m.get_max(ts)}; + } + template + static ActorTypeStat::MaxStatGroup load_seconds(const MaxCounterGroup &src, double inv_ticks_per_second) { + auto ts = Clocks::rdtsc(); + return {.value_forever = ticks_to_seconds(load(src.max_forever), inv_ticks_per_second), + .value_10s = ticks_to_seconds(src.max_10s.get_max(ts), inv_ticks_per_second), + .value_10m = ticks_to_seconds(src.max_10m.get_max(ts), inv_ticks_per_second)}; + } + + // total (increment only statistics) + std::atomic total_created_{0}; + std::atomic total_executions_{0}; + std::atomic total_messages_{0}; + std::atomic total_ticks_{0}; + + // current statistics + std::atomic alive_{0}; + std::atomic executing_{0}; + + // max statistics (TODO: recent_max) + MaxCounterGroup max_execute_messages_; + MaxCounterGroup max_message_ticks_; + MaxCounterGroup max_execute_ticks_; + MaxCounterGroup max_delay_ticks_; + + // execute state + std::atomic execute_start_{0}; + std::atomic execute_messages_{0}; +}; + +class ActorTypeStatRef { + public: + ActorTypeStatImpl *ref_{nullptr}; + + void created() { + if (!ref_) { + return; + } + ref_->created(); + } + void destroyed() { + if (!ref_) { + return; + } + ref_->destroyed(); + } + void pop_from_queue(td::uint64 in_queue_since) { + if (!ref_) { + return; + } + CHECK(in_queue_since); + auto ts = td::Clocks::rdtsc(); + ref_->on_delay(ts, ts - in_queue_since); + } + void start_execute() { + if (!ref_) { + return; + } + ref_->execute_start(td::Clocks::rdtsc()); + } + void finish_execute() { + if (!ref_) { + return; + } + ref_->execute_finish(td::Clocks::rdtsc()); + } + ActorTypeStatImpl::MessageTimer create_message_timer() { + if (!ref_) { + return ActorTypeStatImpl::MessageTimer{nullptr, 0}; + } + return ActorTypeStatImpl::MessageTimer{ref_}; + } + + struct ExecuteTimer { + ExecuteTimer() = delete; + ExecuteTimer(const ExecuteTimer &) = delete; + ExecuteTimer(ExecuteTimer &&) = delete; + ExecuteTimer &operator=(const ExecuteTimer &) = delete; + ExecuteTimer &operator=(ExecuteTimer &&) = delete; + + ExecuteTimer(ActorTypeStatRef *stat) : stat(stat) { + stat->start_execute(); + } + ActorTypeStatRef *stat{}; + ~ExecuteTimer() { + stat->finish_execute(); + } + }; + ExecuteTimer create_execute_timer() { + return ExecuteTimer(this); + } +}; + +// TODO: currently it is implemented via TD_THREAD_LOCAL, so the statistics is global across different schedulers +struct ActorTypeStats { + std::map stats; + ActorTypeStats &operator-=(const ActorTypeStats &other) { + for (auto &it : other.stats) { + stats.at(it.first) -= it.second; + } + return *this; + } + ActorTypeStats &operator/=(double x) { + for (auto &it : stats) { + it.second /= x; + } + return *this; + } +}; +class ActorTypeStatManager { + public: + static ActorTypeStatRef get_actor_type_stat(td::uint32 id, Actor *actor); + static ActorTypeStats get_stats(double inv_ticks_per_second); + static std::string get_class_name(const char *name); +}; + +} // namespace core +} // namespace actor +} // namespace td \ No newline at end of file diff --git a/tdactor/td/actor/core/Scheduler.h b/tdactor/td/actor/core/Scheduler.h index 3de519e09..e76b919e1 100644 --- a/tdactor/td/actor/core/Scheduler.h +++ b/tdactor/td/actor/core/Scheduler.h @@ -130,27 +130,20 @@ struct LocalQueue { public: template bool push(T value, F &&overflow_f) { - auto res = std::move(next_); - next_ = std::move(value); - if (res) { - queue_.local_push(res.unwrap(), overflow_f); - return true; - } - return false; + queue_.local_push(std::move(value), overflow_f); + return true; } bool try_pop(T &message) { - if (!next_) { - return queue_.local_pop(message); - } - message = next_.unwrap(); - return true; + return queue_.local_pop(message); } - bool steal(T &message, LocalQueue &other) { + bool steal(T &message, LocalQueue &other) { return queue_.steal(message, other.queue_); } + size_t size() const { + return queue_.size(); + } private: - td::optional next_; StealingQueue queue_; char pad[TD_CONCURRENCY_PAD - sizeof(optional)]; }; @@ -267,11 +260,12 @@ class Scheduler { bool is_stop_requested() override; void stop() override; - private: - SchedulerGroupInfo *scheduler_group() const { + SchedulerGroupInfo *scheduler_group() const override { return scheduler_group_; } + private: + ActorInfoCreator *creator_; SchedulerId scheduler_id_; CpuWorkerId cpu_worker_id_; diff --git a/tdactor/td/actor/core/SchedulerContext.h b/tdactor/td/actor/core/SchedulerContext.h index e46aef7d5..99b1922f5 100644 --- a/tdactor/td/actor/core/SchedulerContext.h +++ b/tdactor/td/actor/core/SchedulerContext.h @@ -38,6 +38,7 @@ class SchedulerDispatcher { }; struct Debug; +struct SchedulerGroupInfo; class SchedulerContext : public Context, public SchedulerDispatcher { public: virtual ~SchedulerContext() = default; @@ -59,6 +60,7 @@ class SchedulerContext : public Context, public SchedulerDispa // Debug virtual Debug &get_debug() = 0; + virtual SchedulerGroupInfo *scheduler_group() const = 0; }; } // namespace core } // namespace actor diff --git a/tdactor/test/actors_core.cpp b/tdactor/test/actors_core.cpp index 96cd6239e..1c56d1e5b 100644 --- a/tdactor/test/actors_core.cpp +++ b/tdactor/test/actors_core.cpp @@ -19,15 +19,20 @@ #include "td/actor/core/ActorLocker.h" #include "td/actor/actor.h" #include "td/actor/PromiseFuture.h" +#include "td/actor/ActorStats.h" #include "td/utils/format.h" #include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/port/thread.h" #include "td/utils/port/thread.h" #include "td/utils/Random.h" #include "td/utils/Slice.h" #include "td/utils/StringBuilder.h" #include "td/utils/tests.h" #include "td/utils/Time.h" +#include "td/utils/TimedStat.h" +#include "td/utils/port/sleep.h" #include #include @@ -1102,4 +1107,59 @@ TEST(Actor2, send_vs_close2) { scheduler.run(); } } + +TEST(Actor2, test_stats) { + Scheduler scheduler({8}); + td::actor::set_debug(true); + + auto watcher = td::create_shared_destructor([] { SchedulerContext::get()->stop(); }); + scheduler.run_in_context([watcher = std::move(watcher)] { + class SleepWorker : public Actor { + void loop() override { + // 0.8 load + td::usleep_for(800000); + alarm_timestamp() = td::Timestamp::in(0.2); + } + }; + class QueueWorker : public Actor { + void loop() override { + for (int i = 0; i < 20; i++) { + send_closure(actor_id(this), &QueueWorker::ping); + } + alarm_timestamp() = td::Timestamp::in(1.0); + } + void ping() { + } + }; + class Master : public Actor { + public: + Master(std::shared_ptr watcher) : watcher_(std::move(watcher)) { + } + void start_up() override { + alarm_timestamp() = td::Timestamp::in(1); + stats_ = td::actor::create_actor("actor_stats"); + td::actor::create_actor("sleep_worker").release(); + td::actor::create_actor("queue_worker").release(); + } + void alarm() override { + td::actor::send_closure(stats_, &ActorStats::prepare_stats, td::promise_send_closure(actor_id(this), &Master::on_stats)); + alarm_timestamp() = td::Timestamp::in(5); + } + void on_stats(td::Result r_stats) { + LOG(ERROR) << "\n" << r_stats.ok(); + if (--cnt_ == 0) { + stop(); + } + } + + private: + std::shared_ptr watcher_; + td::actor::ActorOwn stats_; + int cnt_={2}; + }; + td::actor::create_actor("Master", watcher).release(); + }); + + scheduler.run(); +} #endif //!TD_THREAD_UNSUPPORTED diff --git a/tddb/CMakeLists.txt b/tddb/CMakeLists.txt index 89730b954..55476e648 100644 --- a/tddb/CMakeLists.txt +++ b/tddb/CMakeLists.txt @@ -59,7 +59,7 @@ target_link_libraries(tddb PUBLIC tdutils tdactor tddb_utils) if (TDDB_USE_ROCKSDB) target_sources(tddb PRIVATE ${TDDB_ROCKSDB_SOURCE}) target_compile_definitions(tddb PUBLIC -DTDDB_USE_ROCKSDB) - target_link_libraries(tddb PRIVATE rocksdb) + target_link_libraries(tddb PUBLIC rocksdb) target_include_directories(tddb PRIVATE $) endif() diff --git a/tddb/td/db/KeyValue.h b/tddb/td/db/KeyValue.h index 980e367c6..12c3a4f8d 100644 --- a/tddb/td/db/KeyValue.h +++ b/tddb/td/db/KeyValue.h @@ -32,6 +32,9 @@ class KeyValueReader { virtual Status for_each(std::function f) { return Status::Error("for_each is not supported"); } + virtual Status for_each_in_range (Slice begin, Slice end, std::function f) { + return td::Status::Error("foreach_range is not supported"); + } }; class PrefixedKeyValueReader : public KeyValueReader { diff --git a/tddb/td/db/MemoryKeyValue.cpp b/tddb/td/db/MemoryKeyValue.cpp index aaf5472d7..080133602 100644 --- a/tddb/td/db/MemoryKeyValue.cpp +++ b/tddb/td/db/MemoryKeyValue.cpp @@ -29,6 +29,24 @@ Result MemoryKeyValue::get(Slice key, std::string &va value = it->second; return GetStatus::Ok; } + +Status MemoryKeyValue::for_each(std::function f) { + for (auto &it : map_) { + TRY_STATUS(f(it.first, it.second)); + } + return Status::OK(); +} + +Status MemoryKeyValue::for_each_in_range(Slice begin, Slice end, std::function f) { + for (auto it = map_.lower_bound(begin); it != map_.end(); it++) { + if (it->first < end) { + TRY_STATUS(f(it->first, it->second)); + } else { + break; + } + } + return Status::OK(); +} Status MemoryKeyValue::set(Slice key, Slice value) { map_[key.str()] = value.str(); return Status::OK(); diff --git a/tddb/td/db/MemoryKeyValue.h b/tddb/td/db/MemoryKeyValue.h index c9d584bcf..f0b5faa08 100644 --- a/tddb/td/db/MemoryKeyValue.h +++ b/tddb/td/db/MemoryKeyValue.h @@ -25,6 +25,8 @@ namespace td { class MemoryKeyValue : public KeyValue { public: Result get(Slice key, std::string &value) override; + Status for_each(std::function f) override; + Status for_each_in_range(Slice begin, Slice end, std::function f) override; Status set(Slice key, Slice value) override; Status erase(Slice key) override; Result count(Slice prefix) override; diff --git a/tddb/td/db/RocksDb.cpp b/tddb/td/db/RocksDb.cpp index f1af0cf79..0be79cb31 100644 --- a/tddb/td/db/RocksDb.cpp +++ b/tddb/td/db/RocksDb.cpp @@ -65,12 +65,16 @@ Result RocksDb::open(std::string path, RocksDbOptions options) { rocksdb::Options db_options; static auto default_cache = rocksdb::NewLRUCache(1 << 30); - if (options.block_cache == nullptr) { + if (!options.no_block_cache && options.block_cache == nullptr) { options.block_cache = default_cache; } rocksdb::BlockBasedTableOptions table_options; - table_options.block_cache = options.block_cache; + if (options.no_block_cache) { + table_options.no_block_cache = true; + } else { + table_options.block_cache = options.block_cache; + } db_options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options)); db_options.use_direct_reads = options.use_direct_reads; @@ -212,6 +216,32 @@ Status RocksDb::for_each(std::function f) { return Status::OK(); } +Status RocksDb::for_each_in_range(Slice begin, Slice end, std::function f) { + rocksdb::ReadOptions options; + options.snapshot = snapshot_.get(); + std::unique_ptr iterator; + if (snapshot_ || !transaction_) { + iterator.reset(db_->NewIterator(options)); + } else { + iterator.reset(transaction_->GetIterator(options)); + } + + auto comparator = rocksdb::BytewiseComparator(); + iterator->Seek(to_rocksdb(begin)); + for (; iterator->Valid(); iterator->Next()) { + auto key = from_rocksdb(iterator->key()); + if (comparator->Compare(to_rocksdb(key), to_rocksdb(end)) >= 0) { + break; + } + auto value = from_rocksdb(iterator->value()); + TRY_STATUS(f(key, value)); + } + if (!iterator->status().ok()) { + return from_rocksdb(iterator->status()); + } + return td::Status::OK(); +} + Status RocksDb::begin_write_batch() { CHECK(!transaction_); write_batch_ = std::make_unique(); diff --git a/tddb/td/db/RocksDb.h b/tddb/td/db/RocksDb.h index 32c53a529..499a33281 100644 --- a/tddb/td/db/RocksDb.h +++ b/tddb/td/db/RocksDb.h @@ -23,6 +23,7 @@ #endif #include "td/db/KeyValue.h" +#include "td/utils/Span.h" #include "td/utils/Status.h" #include "td/utils/optional.h" @@ -32,6 +33,8 @@ #include #include +#include + namespace rocksdb { class Cache; class OptimisticTransactionDB; @@ -59,6 +62,7 @@ struct RocksDbOptions { std::shared_ptr block_cache; // Default - one 1G cache for all RocksDb std::shared_ptr snapshot_statistics = nullptr; bool use_direct_reads = false; + bool no_block_cache = false; }; class RocksDb : public KeyValue { @@ -72,6 +76,7 @@ class RocksDb : public KeyValue { Status erase(Slice key) override; Result count(Slice prefix) override; Status for_each(std::function f) override; + Status for_each_in_range (Slice begin, Slice end, std::function f) override; Status begin_write_batch() override; Status commit_write_batch() override; diff --git a/tddb/td/db/utils/StreamToFileActor.cpp b/tddb/td/db/utils/StreamToFileActor.cpp index 5a3427d38..fb733d02f 100644 --- a/tddb/td/db/utils/StreamToFileActor.cpp +++ b/tddb/td/db/utils/StreamToFileActor.cpp @@ -81,7 +81,7 @@ Result StreamToFileActor::do_loop() { sync_at_ = {}; } - bool need_update = sync_state_.set_synced_size(synced_size_) | sync_state_.set_flushed_size(flushed_size_); + bool need_update = int(sync_state_.set_synced_size(synced_size_)) | int(sync_state_.set_flushed_size(flushed_size_)); if (need_update && callback_) { callback_->on_sync_state_changed(); } diff --git a/tdfec/td/fec/raptorq/Solver.cpp b/tdfec/td/fec/raptorq/Solver.cpp index 02d8102fe..772271fa7 100644 --- a/tdfec/td/fec/raptorq/Solver.cpp +++ b/tdfec/td/fec/raptorq/Solver.cpp @@ -19,6 +19,7 @@ #include "td/fec/raptorq/Solver.h" #include "td/fec/algebra/GaussianElimination.h" #include "td/fec/algebra/InactivationDecoding.h" +#include "td/utils/ThreadSafeCounter.h" #include "td/utils/Timer.h" #include @@ -70,6 +71,7 @@ Result Solver::run(const Rfc::Parameters &p, Span symbol auto C = GaussianElimination::run(std::move(A), std::move(D)); return C; } + TD_PERF_COUNTER(raptor_solve); PerfWarningTimer x("solve"); Timer timer; auto perf_log = [&](Slice message) { diff --git a/tdtl/td/tl/tl_writer.cpp b/tdtl/td/tl/tl_writer.cpp index 5c672f264..f4f5f27cc 100644 --- a/tdtl/td/tl/tl_writer.cpp +++ b/tdtl/td/tl/tl_writer.cpp @@ -28,7 +28,7 @@ namespace tl { std::string TL_writer::int_to_string(int x) { char buf[15]; - std::sprintf(buf, "%d", x); + std::snprintf(buf, sizeof(buf), "%d", x); return buf; } diff --git a/tdutils/CMakeLists.txt b/tdutils/CMakeLists.txt index 4c7132470..f8c191a14 100644 --- a/tdutils/CMakeLists.txt +++ b/tdutils/CMakeLists.txt @@ -356,6 +356,7 @@ if (LZ4_FOUND) message(STATUS "Found LZ4 ${LZ4_LIBRARIES} ${LZ4_INCLUDE_DIRS}") target_link_libraries(tdutils PRIVATE ${LZ4_LIBRARIES}) target_include_directories(tdutils SYSTEM PRIVATE ${LZ4_INCLUDE_DIRS}) + target_link_directories(tdutils PUBLIC ${LZ4_LIBRARY_DIRS}) endif() if (ABSL_FOUND) diff --git a/tdutils/td/utils/HashMap.h b/tdutils/td/utils/HashMap.h index 9646bef90..1380f0291 100644 --- a/tdutils/td/utils/HashMap.h +++ b/tdutils/td/utils/HashMap.h @@ -22,6 +22,7 @@ #if TD_HAVE_ABSL #include +#include #else #include #endif @@ -31,9 +32,13 @@ namespace td { #if TD_HAVE_ABSL template > using HashMap = absl::flat_hash_map; +template , class E = std::equal_to<>> +using NodeHashMap = absl::node_hash_map; #else template > using HashMap = std::unordered_map; +template > +using NodeHashMap = std::unordered_map; #endif } // namespace td diff --git a/tdutils/td/utils/HashSet.h b/tdutils/td/utils/HashSet.h index b00d5a922..85254b325 100644 --- a/tdutils/td/utils/HashSet.h +++ b/tdutils/td/utils/HashSet.h @@ -22,6 +22,7 @@ #if TD_HAVE_ABSL #include +#include #else #include #endif @@ -29,11 +30,15 @@ namespace td { #if TD_HAVE_ABSL -template > -using HashSet = absl::flat_hash_set; +template , class E = std::equal_to<>> +using HashSet = absl::flat_hash_set; +template , class E = std::equal_to<>> +using NodeHashSet = absl::node_hash_set; #else -template > -using HashSet = std::unordered_set; +template , class E = std::equal_to<>> +using HashSet = std::unordered_set; +template , class E = std::equal_to<>> +using NodeHashSet = HashSet; #endif } // namespace td diff --git a/tdutils/td/utils/HazardPointers.h b/tdutils/td/utils/HazardPointers.h index 954d6efdc..7465140a5 100644 --- a/tdutils/td/utils/HazardPointers.h +++ b/tdutils/td/utils/HazardPointers.h @@ -32,12 +32,7 @@ class HazardPointers { explicit HazardPointers(size_t threads_n) : threads_(threads_n) { for (auto &data : threads_) { for (auto &ptr : data.hazard_) { -// workaround for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=64658 -#if TD_GCC && GCC_VERSION <= 40902 ptr = nullptr; -#else - std::atomic_init(&ptr, static_cast(nullptr)); -#endif } } } diff --git a/tdutils/td/utils/StealingQueue.h b/tdutils/td/utils/StealingQueue.h index 20a39dc06..86c2f0ef6 100644 --- a/tdutils/td/utils/StealingQueue.h +++ b/tdutils/td/utils/StealingQueue.h @@ -115,6 +115,22 @@ class StealingQueue { std::atomic_thread_fence(std::memory_order_seq_cst); } + size_t size() const { + while (true) { + auto head = head_.load(); + auto tail = tail_.load(std::memory_order_acquire); + + if (tail < head) { + continue; + } + size_t n = tail - head; + if (n > N) { + continue; + } + return n; + } + } + private: std::atomic head_{0}; std::atomic tail_{0}; diff --git a/tdutils/td/utils/ThreadSafeCounter.h b/tdutils/td/utils/ThreadSafeCounter.h index 55bf94b5a..aa976b2fb 100644 --- a/tdutils/td/utils/ThreadSafeCounter.h +++ b/tdutils/td/utils/ThreadSafeCounter.h @@ -137,4 +137,55 @@ class NamedThreadSafeCounter { Counter counter_; }; +// another class for simplicity, it +struct NamedPerfCounter { + public: + static NamedPerfCounter &get_default() { + static NamedPerfCounter res; + return res; + } + struct PerfCounterRef { + NamedThreadSafeCounter::CounterRef count; + NamedThreadSafeCounter::CounterRef duration; + }; + PerfCounterRef get_counter(Slice name) { + return {.count = counter_.get_counter(PSLICE() << name << ".count"), + .duration = counter_.get_counter(PSLICE() << name << ".duration")}; + } + + struct ScopedPerfCounterRef : public NoCopyOrMove { + PerfCounterRef perf_counter; + uint64 started_at_ticks{td::Clocks::rdtsc()}; + + ~ScopedPerfCounterRef() { + perf_counter.count.add(1); + perf_counter.duration.add(td::Clocks::rdtsc() - started_at_ticks); + } + }; + + template + void for_each(F &&f) const { + counter_.for_each(f); + } + + void clear() { + counter_.clear(); + } + + friend StringBuilder &operator<<(StringBuilder &sb, const NamedPerfCounter &counter) { + return sb << counter.counter_; + } + private: + NamedThreadSafeCounter counter_; +}; + } // namespace td + +#define TD_PERF_COUNTER(name) \ + static auto perf_##name = td::NamedPerfCounter::get_default().get_counter(td::Slice(#name)); \ + auto scoped_perf_##name = td::NamedPerfCounter::ScopedPerfCounterRef{.perf_counter = perf_##name}; + +#define TD_PERF_COUNTER_SINCE(name, since) \ + static auto perf_##name = td::NamedPerfCounter::get_default().get_counter(td::Slice(#name)); \ + auto scoped_perf_##name = \ + td::NamedPerfCounter::ScopedPerfCounterRef{.perf_counter = perf_##name, .started_at_ticks = since}; diff --git a/tdutils/td/utils/common.h b/tdutils/td/utils/common.h index 79d3dc52a..f9f86c5c1 100644 --- a/tdutils/td/utils/common.h +++ b/tdutils/td/utils/common.h @@ -127,4 +127,12 @@ struct Auto { } }; +struct NoCopyOrMove { + NoCopyOrMove() = default; + NoCopyOrMove(NoCopyOrMove &&) = delete; + NoCopyOrMove(const NoCopyOrMove &) = delete; + NoCopyOrMove &operator=(NoCopyOrMove &&) = delete; + NoCopyOrMove &operator=(const NoCopyOrMove &) = delete; +}; + } // namespace td diff --git a/tdutils/td/utils/logging.cpp b/tdutils/td/utils/logging.cpp index 477823b89..345615f12 100644 --- a/tdutils/td/utils/logging.cpp +++ b/tdutils/td/utils/logging.cpp @@ -18,6 +18,7 @@ */ #include "td/utils/logging.h" +#include "ThreadSafeCounter.h" #include "td/utils/port/Clocks.h" #include "td/utils/port/StdStreams.h" #include "td/utils/port/thread_local.h" @@ -127,6 +128,9 @@ Logger::~Logger() { slice = MutableCSlice(slice.begin(), slice.begin() + slice.size() - 1); } log_.append(slice, log_level_); + + // put stats here to avoid conflict with PSTRING and PSLICE + TD_PERF_COUNTER_SINCE(logger, start_at_); } else { log_.append(as_cslice(), log_level_); } @@ -301,5 +305,4 @@ ScopedDisableLog::~ScopedDisableLog() { set_verbosity_level(sdl_verbosity); } } - } // namespace td diff --git a/tdutils/td/utils/logging.h b/tdutils/td/utils/logging.h index e5278b4bc..d00fba154 100644 --- a/tdutils/td/utils/logging.h +++ b/tdutils/td/utils/logging.h @@ -40,6 +40,7 @@ #include "td/utils/Slice.h" #include "td/utils/StackAllocator.h" #include "td/utils/StringBuilder.h" +#include "td/utils/port/Clocks.h" #include #include @@ -251,7 +252,8 @@ class Logger { , log_(log) , sb_(buffer_.as_slice()) , options_(options) - , log_level_(log_level) { + , log_level_(log_level) + , start_at_(Clocks::rdtsc()) { } Logger(LogInterface &log, const LogOptions &options, int log_level, Slice file_name, int line_num, Slice comment); @@ -283,6 +285,7 @@ class Logger { StringBuilder sb_; const LogOptions &options_; int log_level_; + td::uint64 start_at_; }; namespace detail { @@ -346,5 +349,4 @@ class TsLog : public LogInterface { lock_.clear(std::memory_order_release); } }; - } // namespace td diff --git a/tdutils/td/utils/port/Clocks.cpp b/tdutils/td/utils/port/Clocks.cpp index 59e80f619..7e27d51a9 100644 --- a/tdutils/td/utils/port/Clocks.cpp +++ b/tdutils/td/utils/port/Clocks.cpp @@ -22,6 +22,10 @@ #include namespace td { +int64 Clocks::monotonic_nano() { + auto duration = std::chrono::steady_clock::now().time_since_epoch(); + return std::chrono::duration_cast(duration).count(); +} double Clocks::monotonic() { // TODO write system specific functions, because std::chrono::steady_clock is steady only under Windows diff --git a/tdutils/td/utils/port/Clocks.h b/tdutils/td/utils/port/Clocks.h index 2d7d9e0fd..b20543650 100644 --- a/tdutils/td/utils/port/Clocks.h +++ b/tdutils/td/utils/port/Clocks.h @@ -17,15 +17,89 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "td/utils/int_types.h" namespace td { struct Clocks { + static int64 monotonic_nano(); + static double monotonic(); static double system(); static int tz_offset(); + +#if defined(__i386__) or defined(__M_IX86) + static uint64 rdtsc() { + unsigned long long int x; + __asm__ volatile("rdtsc" : "=A"(x)); + return x; + } + + static constexpr uint64 rdtsc_frequency() { + return 2000'000'000; + } + + static constexpr double ticks_per_second() { + return 2e9; + } + + static constexpr double inv_ticks_per_second() { + return 0.5e-9; + } +#elif defined(__x86_64__) or defined(__M_X64) + static uint64 rdtsc() { + unsigned hi, lo; + __asm__ __volatile__("rdtsc" : "=a"(lo), "=d"(hi)); + return ((unsigned long long)lo) | (((unsigned long long)hi) << 32); + } + static constexpr uint64 rdtsc_frequency() { + return 2000'000'000; + } + + static constexpr double ticks_per_second() { + return 2e9; + } + + static constexpr double inv_ticks_per_second() { + return 0.5e-9; + } +#elif defined(__aarch64__) or defined(_M_ARM64) + static uint64 rdtsc() { + unsigned long long val; + asm volatile("mrs %0, cntvct_el0" : "=r"(val)); + return val; + } + static uint64 rdtsc_frequency() { + unsigned long long val; + asm volatile("mrs %0, cntfrq_el0" : "=r"(val)); + return val; + } + + static double ticks_per_second() { + return static_cast(rdtsc_frequency()); + } + + static double inv_ticks_per_second() { + return 1.0 / static_cast(rdtsc_frequency()); + } +#else + static uint64 rdtsc() { + return monotonic_nano(); + } + static uint64 rdtsc_frequency() { + return 1000'000'000; + } + + static double ticks_per_second() { + return static_cast(rdtsc_frequency()); + } + + static double inv_ticks_per_second() { + return 1.0 / static_cast(rdtsc_frequency()); + } +#endif }; } // namespace td diff --git a/tdutils/td/utils/port/Stat.cpp b/tdutils/td/utils/port/Stat.cpp index d7ec50f75..816d622e4 100644 --- a/tdutils/td/utils/port/Stat.cpp +++ b/tdutils/td/utils/port/Stat.cpp @@ -58,8 +58,7 @@ #define PSAPI_VERSION 1 #endif #include -#pragma comment( lib, "psapi.lib" ) - +#pragma comment(lib, "psapi.lib") #endif @@ -413,7 +412,7 @@ Result cpu_stat() { #endif } -Result get_total_ram() { +Result get_total_mem_stat() { #if TD_LINUX TRY_RESULT(fd, FileFd::open("/proc/meminfo", FileFd::Read)); SCOPE_EXIT { @@ -425,8 +424,10 @@ Result get_total_ram() { if (size >= TMEM_SIZE - 1) { return Status::Error("Failed for read /proc/meminfo"); } + TotalMemStat stat; mem[size] = 0; - const char* s = mem; + const char *s = mem; + size_t got = 0; while (*s) { const char *name_begin = s; while (*s != 0 && *s != '\n') { @@ -437,18 +438,28 @@ Result get_total_ram() { name_end++; } Slice name(name_begin, name_end); + td::uint64 *dest = nullptr; if (name == "MemTotal") { + dest = &stat.total_ram; + } else if (name == "MemAvailable") { + dest = &stat.available_ram; + } + if (dest != nullptr) { Slice value(name_end, s); if (!value.empty() && value[0] == ':') { value.remove_prefix(1); } value = trim(value); value = split(value).first; - TRY_RESULT_PREFIX(mem, to_integer_safe(value), "Invalid value of MemTotal"); + TRY_RESULT_PREFIX(mem, to_integer_safe(value), PSLICE() << "Invalid value of " << name); if (mem >= 1ULL << (64 - 10)) { return Status::Error("Invalid value of MemTotal"); } - return mem * 1024; + *dest = mem * 1024; + got++; + if (got == 2) { + return stat; + } } if (*s == 0) { break; diff --git a/tdutils/td/utils/port/Stat.h b/tdutils/td/utils/port/Stat.h index cd16ebb36..ab97be0f1 100644 --- a/tdutils/td/utils/port/Stat.h +++ b/tdutils/td/utils/port/Stat.h @@ -64,6 +64,10 @@ Status update_atime(CSlice path) TD_WARN_UNUSED_RESULT; #endif -Result get_total_ram() TD_WARN_UNUSED_RESULT; +struct TotalMemStat { + uint64 total_ram; + uint64 available_ram; +}; +Result get_total_mem_stat() TD_WARN_UNUSED_RESULT; } // namespace td diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 89b5763e8..65bdf71a8 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -668,6 +668,7 @@ engine.validator.signature signature:bytes = engine.validator.Signature; engine.validator.oneStat key:string value:string = engine.validator.OneStat; engine.validator.stats stats:(vector engine.validator.oneStat) = engine.validator.Stats; +engine.validator.textStats data:string = engine.validator.TextStats; engine.validator.controlQueryError code:int message:string = engine.validator.ControlQueryError; @@ -758,6 +759,7 @@ engine.validator.setCollatorOptionsJson json:string = engine.validator.Success; engine.validator.getCollatorOptionsJson = engine.validator.JsonConfig; engine.validator.getAdnlStats = adnl.Stats; +engine.validator.getActorTextStats = engine.validator.TextStats; ---types--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 9efac72a4..790d2df2f 100644 Binary files a/tl/generate/scheme/ton_api.tlo and b/tl/generate/scheme/ton_api.tlo differ diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index f62b9ae47..5b799b304 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -1768,7 +1768,7 @@ class RunEmulator : public TonlibQueryActor { void get_block_id(td::Promise&& promise) { auto shard_id = ton::shard_prefix(request_.address.addr, 60); auto query = ton::lite_api::liteServer_lookupBlock(0b111111010, ton::create_tl_lite_block_id_simple({request_.address.workchain, shard_id, 0}), request_.lt, 0); - client_.send_query(std::move(query), promise.wrap([self = this, shard_id](td::Result> header_r) -> td::Result { + client_.send_query(std::move(query), promise.wrap([shard_id](td::Result> header_r) -> td::Result { TRY_RESULT(header, std::move(header_r)); ton::BlockIdExt block_id = ton::create_block_id(header->id_); @@ -1817,7 +1817,7 @@ class RunEmulator : public TonlibQueryActor { void get_mc_state_root(td::Promise>&& promise) { TRY_RESULT_PROMISE(promise, lite_block, to_lite_api(*to_tonlib_api(block_id_.mc))); auto block = ton::create_block_id(lite_block); - client_.send_query(ton::lite_api::liteServer_getConfigAll(0b11'11111111, std::move(lite_block)), promise.wrap([self = this, block](auto r_config) -> td::Result> { + client_.send_query(ton::lite_api::liteServer_getConfigAll(0b11'11111111, std::move(lite_block)), promise.wrap([block](auto r_config) -> td::Result> { TRY_RESULT(state, block::check_extract_state_proof(block, r_config->state_proof_.as_slice(), r_config->config_proof_.as_slice())); @@ -1842,7 +1842,7 @@ class RunEmulator : public TonlibQueryActor { constexpr int req_count = 256; auto query = ton::lite_api::liteServer_listBlockTransactions(std::move(lite_block), mode, req_count, std::move(after), false, true); - client_.send_query(std::move(query), [self = this, mode, lt, root_hash = block_id_.id.root_hash, req_count](lite_api_ptr&& bTxes) { + client_.send_query(std::move(query), [self = this, mode, lt, root_hash = block_id_.id.root_hash](lite_api_ptr&& bTxes) { if (!bTxes) { self->check(td::Status::Error("liteServer.blockTransactions is null")); return; @@ -5867,7 +5867,7 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getTransactions& re std::move(after), reverse_mode, check_proof), - promise.wrap([check_proof, reverse_mode, root_hash, req_count = request.count_, start_addr, start_lt, mode = request.mode_] + promise.wrap([root_hash, req_count = request.count_, start_addr, start_lt, mode = request.mode_] (lite_api_ptr&& bTxes) -> td::Result> { TRY_STATUS(check_block_transactions_proof(bTxes, mode, start_lt, start_addr, root_hash, req_count)); diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 372fa8121..baa39f8ee 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1008,6 +1008,32 @@ td::Status ImportShardOverlayCertificateQuery::receive(td::BufferSlice data) { td::TerminalIO::out() << "successfully sent certificate to overlay manager\n"; return td::Status::OK(); } +td::Status GetActorStatsQuery::run() { + auto r_file_name = tokenizer_.get_token(); + if (r_file_name.is_ok()) { + file_name_ = r_file_name.move_as_ok(); + } + return td::Status::OK(); +} +td::Status GetActorStatsQuery::send() { + auto b = ton::create_serialize_tl_object(); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status GetActorStatsQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + if (file_name_.empty()) { + td::TerminalIO::out() << f->data_; + } else { + std::ofstream sb(file_name_); + sb << f->data_; + sb << std::flush; + td::TerminalIO::output(std::string("wrote stats to " + file_name_ + "\n")); + } + return td::Status::OK(); +} td::Status GetPerfTimerStatsJsonQuery::run() { TRY_RESULT_ASSIGN(file_name_, tokenizer_.get_token()); diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index 6314d6199..c3c021509 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -1076,6 +1076,28 @@ class ImportShardOverlayCertificateQuery : public Query { std::string in_file_; }; +class GetActorStatsQuery : public Query { + public: + GetActorStatsQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "getactorstats"; + } + static std::string get_help() { + return "getactorstats []\tget actor stats and print it either in stdout or in "; + } + std::string name() const override { + return get_name(); + } + + private: + std::string file_name_; +}; + class GetPerfTimerStatsJsonQuery : public Query { public: GetPerfTimerStatsJsonQuery(td::actor::ActorId console, Tokenizer tokenizer) diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index 1ec0f3803..59e2f2e8a 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -140,6 +140,7 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index bb7574105..a2ca6a241 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1432,7 +1432,11 @@ td::Status ValidatorEngine::load_global_config() { if (!session_logs_file_.empty()) { validator_options_.write().set_session_logs_file(session_logs_file_); } + if (celldb_in_memory_) { + celldb_compress_depth_ = 0; + } validator_options_.write().set_celldb_compress_depth(celldb_compress_depth_); + validator_options_.write().set_celldb_in_memory(celldb_in_memory_); validator_options_.write().set_max_open_archive_files(max_open_archive_files_); validator_options_.write().set_archive_preload_period(archive_preload_period_); validator_options_.write().set_disable_rocksdb_stats(disable_rocksdb_stats_); @@ -1471,11 +1475,11 @@ td::Status ValidatorEngine::load_global_config() { } validator_options_.write().set_hardforks(std::move(h)); - auto r_total_ram = td::get_total_ram(); - if (r_total_ram.is_error()) { - LOG(ERROR) << "Failed to get total RAM size: " << r_total_ram.move_as_error(); + auto r_total_mem_stat = td::get_total_mem_stat(); + if (r_total_mem_stat.is_error()) { + LOG(ERROR) << "Failed to get total RAM size: " << r_total_mem_stat.move_as_error(); } else { - td::uint64 total_ram = r_total_ram.move_as_ok(); + td::uint64 total_ram = r_total_mem_stat.ok().total_ram; LOG(WARNING) << "Total RAM = " << td::format::as_size(total_ram); if (total_ram >= (90ULL << 30)) { fast_state_serializer_enabled_ = true; @@ -3540,6 +3544,31 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getOverla }); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getActorTextStats &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + + if (validator_manager_.empty()) { + promise.set_value( + create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "validator manager not started"))); + return; + } + + auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + auto r = R.move_as_ok(); + promise.set_value(ton::create_serialize_tl_object(std::move(r))); + } + }); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::prepare_actor_stats, + std::move(P)); +} + void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getPerfTimerStats &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_default)) { @@ -3753,9 +3782,8 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCustom }); } -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delCustomOverlay &query, - td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, - td::Promise promise) { +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delCustomOverlay &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_modify)) { promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); return; @@ -3795,8 +3823,8 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showCusto return; } - promise.set_value(ton::serialize_tl_object( - custom_overlays_config_, true)); + promise.set_value( + ton::serialize_tl_object(custom_overlays_config_, true)); } void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setStateSerializerEnabled &query, @@ -4223,7 +4251,8 @@ int main(int argc, char *argv[]) { return td::Status::OK(); }); p.add_checked_option( - '\0', "max-archive-fd", "limit for a number of open file descriptirs in archive manager. 0 is unlimited (default)", + '\0', "max-archive-fd", + "limit for a number of open file descriptirs in archive manager. 0 is unlimited (default)", [&](td::Slice s) -> td::Status { TRY_RESULT(v, td::to_integer_safe(s)); acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_max_open_archive_files, v); }); @@ -4258,13 +4287,23 @@ int main(int argc, char *argv[]) { acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_cache_size, v); }); return td::Status::OK(); }); + p.add_option('\0', "celldb-direct-io", + "enable direct I/O mode for RocksDb in CellDb (doesn't apply when celldb cache is < 30G)", [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_direct_io, true); }); + }); + p.add_option('\0', "celldb-preload-all", + "preload all cells from CellDb on startup (recommended to use with big enough celldb-cache-size and " + "celldb-direct-io)", + [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_preload_all, true); }); + }); + p.add_option( - '\0', "celldb-direct-io", "enable direct I/O mode for RocksDb in CellDb (doesn't apply when celldb cache is < 30G)", - [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_direct_io, true); }); }); - p.add_option( - '\0', "celldb-preload-all", - "preload all cells from CellDb on startup (recommended to use with big enough celldb-cache-size and celldb-direct-io)", - [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_preload_all, true); }); }); + '\0', "celldb-in-memory", + "store all cells in-memory, much faster but requires a lot of RAM. RocksDb is still used as persistent storage", + [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_in_memory, true); }); + }); p.add_checked_option( '\0', "catchain-max-block-delay", "delay before creating a new catchain block, in seconds (default: 0.4)", [&](td::Slice s) -> td::Status { @@ -4319,7 +4358,9 @@ int main(int argc, char *argv[]) { } if (need_scheduler_status_flag.exchange(false)) { LOG(ERROR) << "DUMPING SCHEDULER STATISTICS"; - scheduler.get_debug().dump(); + td::StringBuilder sb; + scheduler.get_debug().dump(sb); + LOG(ERROR) << "GOT SCHEDULER STATISTICS\n" << sb.as_cslice(); } if (rotate_logs_flags.exchange(false)) { if (td::log_interface) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 2e94dd1ef..50cd5a323 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -214,6 +214,7 @@ class ValidatorEngine : public td::actor::Actor { td::optional celldb_cache_size_ = 1LL << 30; bool celldb_direct_io_ = false; bool celldb_preload_all_ = false; + bool celldb_in_memory_ = false; td::optional catchain_max_block_delay_, catchain_max_block_delay_slow_; bool read_config_ = false; bool started_keyring_ = false; @@ -297,6 +298,9 @@ class ValidatorEngine : public td::actor::Actor { void set_celldb_preload_all(bool value) { celldb_preload_all_ = value; } + void set_celldb_in_memory(bool value) { + celldb_in_memory_ = value; + } void set_catchain_max_block_delay(double value) { catchain_max_block_delay_ = value; } @@ -474,6 +478,8 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_getOverlaysStats &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_getActorTextStats &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_getPerfTimerStats &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_getShardOutQueueSize &query, td::BufferSlice data, diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 38e00cdea..fe265b28a 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -42,7 +42,7 @@ void ValidatorSessionImpl::process_blocks(std::vector on_new_round(real_state_->cur_round_seqno()); } - td::uint32 cnt = 0; + [[maybe_unused]] td::uint32 cnt = 0; auto ts = description().get_ts(); auto att = description().get_attempt_seqno(ts); std::vector> msgs; diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index 14d3ec469..71b3b73de 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -32,11 +32,11 @@ std::string PackageId::path() const { return "/files/packages/"; } else if (key) { char s[24]; - sprintf(s, "key%03d", id / 1000000); + snprintf(s, sizeof(s), "key%03d", id / 1000000); return PSTRING() << "/archive/packages/" << s << "/"; } else { char s[20]; - sprintf(s, "arch%04d", id / 100000); + snprintf(s, sizeof(s), "arch%04d", id / 100000); return PSTRING() << "/archive/packages/" << s << "/"; } } @@ -46,11 +46,11 @@ std::string PackageId::name() const { return PSTRING() << "temp.archive." << id; } else if (key) { char s[20]; - sprintf(s, "%06d", id); + snprintf(s, sizeof(s), "%06d", id); return PSTRING() << "key.archive." << s; } else { char s[10]; - sprintf(s, "%05d", id); + snprintf(s, sizeof(s), "%05d", id); return PSTRING() << "archive." << s; } } @@ -342,19 +342,19 @@ void ArchiveManager::add_zero_state(BlockIdExt block_id, td::BufferSlice data, t void ArchiveManager::add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice data, td::Promise promise) { auto create_writer = [&](std::string path, td::Promise P) { - td::actor::create_actor("writefile", db_root_ + "/archive/tmp/", - std::move(path), std::move(data), std::move(P)) + td::actor::create_actor("writefile", db_root_ + "/archive/tmp/", std::move(path), std::move(data), + std::move(P)) .release(); }; add_persistent_state_impl(block_id, masterchain_block_id, std::move(promise), std::move(create_writer)); } void ArchiveManager::add_persistent_state_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, - std::function write_state, + std::function write_state, td::Promise promise) { auto create_writer = [&](std::string path, td::Promise P) { - td::actor::create_actor("writefile", db_root_ + "/archive/tmp/", - std::move(path), std::move(write_state), std::move(P)) + td::actor::create_actor("writefile", db_root_ + "/archive/tmp/", std::move(path), + std::move(write_state), std::move(P)) .release(); }; add_persistent_state_impl(block_id, masterchain_block_id, std::move(promise), std::move(create_writer)); @@ -624,8 +624,8 @@ void ArchiveManager::load_package(PackageId id) { } } - desc.file = - td::actor::create_actor("slice", id.id, id.key, id.temp, false, db_root_, archive_lru_.get(), statistics_); + desc.file = td::actor::create_actor("slice", id.id, id.key, id.temp, false, db_root_, + archive_lru_.get(), statistics_); m.emplace(id, std::move(desc)); update_permanent_slices(); @@ -659,8 +659,8 @@ const ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull FileDescription new_desc{id, false}; td::mkdir(db_root_ + id.path()).ensure(); std::string prefix = PSTRING() << db_root_ << id.path() << id.name(); - new_desc.file = - td::actor::create_actor("slice", id.id, id.key, id.temp, false, db_root_, archive_lru_.get(), statistics_); + new_desc.file = td::actor::create_actor("slice", id.id, id.key, id.temp, false, db_root_, + archive_lru_.get(), statistics_); const FileDescription &desc = f.emplace(id, std::move(new_desc)); if (!id.temp) { update_desc(f, desc, shard, seqno, ts, lt); @@ -940,7 +940,8 @@ void ArchiveManager::start_up() { void ArchiveManager::alarm() { alarm_timestamp() = td::Timestamp::in(60.0); auto stats = statistics_.to_string_and_reset(); - auto to_file_r = td::FileFd::open(db_root_ + "/db_stats.txt", td::FileFd::Truncate | td::FileFd::Create | td::FileFd::Write, 0644); + auto to_file_r = + td::FileFd::open(db_root_ + "/db_stats.txt", td::FileFd::Truncate | td::FileFd::Create | td::FileFd::Write, 0644); if (to_file_r.is_error()) { LOG(ERROR) << "Failed to open db_stats.txt: " << to_file_r.move_as_error(); return; @@ -1034,7 +1035,7 @@ void ArchiveManager::persistent_state_gc(std::pair last) { } if (res != 0) { delay_action([key, SelfId = actor_id( - this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); }, + this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); }, td::Timestamp::in(1.0)); return; } @@ -1051,7 +1052,7 @@ void ArchiveManager::persistent_state_gc(std::pair last) { } if (!allow_delete) { delay_action([key, SelfId = actor_id( - this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); }, + this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); }, td::Timestamp::in(1.0)); return; } @@ -1082,9 +1083,9 @@ void ArchiveManager::got_gc_masterchain_handle(ConstBlockHandle handle, std::pai td::unlink(db_root_ + "/archive/states/" + F.filename_short()).ignore(); perm_states_.erase(it); } - delay_action([key, SelfId = actor_id( - this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); }, - td::Timestamp::in(1.0)); + delay_action( + [key, SelfId = actor_id(this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); }, + td::Timestamp::in(1.0)); } PackageId ArchiveManager::get_temp_package_id() const { diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 1701ae588..91e9054e8 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -17,10 +17,12 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "celldb.hpp" + +#include "files-async.hpp" #include "rootdb.hpp" #include "td/db/RocksDb.h" -#include "td/utils/filesystem.h" +#include "rocksdb/utilities/optimistic_transaction_db.h" #include "ton/ton-tl.hpp" #include "ton/ton-io.hpp" @@ -29,7 +31,6 @@ namespace ton { namespace validator { - class CellDbAsyncExecutor : public vm::DynamicBagOfCellsDb::AsyncExecutor { public: explicit CellDbAsyncExecutor(td::actor::ActorId cell_db) : cell_db_(std::move(cell_db)) { @@ -38,11 +39,13 @@ class CellDbAsyncExecutor : public vm::DynamicBagOfCellsDb::AsyncExecutor { void execute_async(std::function f) override { class Runner : public td::actor::Actor { public: - explicit Runner(std::function f) : f_(std::move(f)) {} + explicit Runner(std::function f) : f_(std::move(f)) { + } void start_up() override { f_(); stop(); } + private: std::function f_; }; @@ -52,6 +55,7 @@ class CellDbAsyncExecutor : public vm::DynamicBagOfCellsDb::AsyncExecutor { void execute_sync(std::function f) override { td::actor::send_closure(cell_db_, &CellDbBase::execute_sync, std::move(f)); } + private: td::actor::ActorId cell_db_; }; @@ -97,13 +101,30 @@ void CellDbIn::start_up() { LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(opts_->get_celldb_cache_size().value()); } db_options.use_direct_reads = opts_->get_celldb_direct_io(); - cell_db_ = std::make_shared(td::RocksDb::open(path_, std::move(db_options)).move_as_ok()); + if (opts_->get_celldb_in_memory()) { + td::RocksDbOptions read_db_options; + read_db_options.use_direct_reads = true; + read_db_options.no_block_cache = true; + read_db_options.block_cache = {}; + LOG(WARNING) << "Loading all cells in memory (because of --celldb-in-memory)"; + td::Timer timer; + auto read_cell_db = + std::make_shared(td::RocksDb::open(path_, std::move(read_db_options)).move_as_ok()); + boc_ = vm::DynamicBagOfCellsDb::create_in_memory(read_cell_db.get(), {}); + in_memory_load_time_ = timer.elapsed(); + td::actor::send_closure(parent_, &CellDb::set_in_memory_boc, boc_); + } - boc_ = vm::DynamicBagOfCellsDb::create(); - boc_->set_celldb_compress_depth(opts_->get_celldb_compress_depth()); - boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); - td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); + auto rocks_db = std::make_shared(td::RocksDb::open(path_, std::move(db_options)).move_as_ok()); + rocks_db_ = rocks_db->raw_db(); + cell_db_ = std::move(rocks_db); + if (!opts_->get_celldb_in_memory()) { + boc_ = vm::DynamicBagOfCellsDb::create(); + boc_->set_celldb_compress_depth(opts_->get_celldb_compress_depth()); + boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); + td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); + } alarm_timestamp() = td::Timestamp::in(10.0); @@ -117,31 +138,39 @@ void CellDbIn::start_up() { if (opts_->get_celldb_preload_all()) { // Iterate whole DB in a separate thread - delay_action([snapshot = cell_db_->snapshot()]() { - LOG(WARNING) << "CellDb: pre-loading all keys"; - td::uint64 total = 0; - td::Timer timer; - auto S = snapshot->for_each([&](td::Slice, td::Slice) { - ++total; - if (total % 1000000 == 0) { - LOG(INFO) << "CellDb: iterated " << total << " keys"; - } - return td::Status::OK(); - }); - if (S.is_error()) { - LOG(ERROR) << "CellDb: pre-load failed: " << S.move_as_error(); - } else { - LOG(WARNING) << "CellDb: iterated " << total << " keys in " << timer.elapsed() << "s"; - } - }, td::Timestamp::now()); + delay_action( + [snapshot = cell_db_->snapshot()]() { + LOG(WARNING) << "CellDb: pre-loading all keys"; + td::uint64 total = 0; + td::Timer timer; + auto S = snapshot->for_each([&](td::Slice, td::Slice) { + ++total; + if (total % 1000000 == 0) { + LOG(INFO) << "CellDb: iterated " << total << " keys"; + } + return td::Status::OK(); + }); + if (S.is_error()) { + LOG(ERROR) << "CellDb: pre-load failed: " << S.move_as_error(); + } else { + LOG(WARNING) << "CellDb: iterated " << total << " keys in " << timer.elapsed() << "s"; + } + }, + td::Timestamp::now()); } } void CellDbIn::load_cell(RootHash hash, td::Promise> promise) { + if (opts_->get_celldb_in_memory()) { + auto result = boc_->load_root(hash.as_slice()); + async_apply("load_cell_result", std::move(promise), std::move(result)); + return; + } boc_->load_cell_async(hash.as_slice(), async_executor, std::move(promise)); } void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promise> promise) { + TD_PERF_COUNTER(celldb_store_cell); td::PerfWarningTimer timer{"storecell", 0.1}; auto key_hash = get_key_hash(block_id); auto R = get_block(key_hash); @@ -181,8 +210,10 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promi set_block(key_hash, std::move(D)); cell_db_->commit_write_batch().ensure(); - boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); - td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); + if (!opts_->get_celldb_in_memory()) { + boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); + td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); + } promise.set_result(boc_->load_cell(cell->get_hash().as_slice())); if (!opts_->get_disable_rocksdb_stats()) { @@ -199,12 +230,50 @@ void CellDbIn::get_last_deleted_mc_state(td::Promise promise) { promise.set_result(last_deleted_mc_state_); } +std::vector> CellDbIn::prepare_stats() { + TD_PERF_COUNTER(celldb_prepare_stats); + auto r_boc_stats = boc_->get_stats(); + if (r_boc_stats.is_ok()) { + cell_db_statistics_.boc_stats_ = r_boc_stats.move_as_ok(); + } + cell_db_statistics_.in_memory_load_time_ = in_memory_load_time_; + auto stats = cell_db_statistics_.prepare_stats(); + auto add_stat = [&](const auto& key, const auto& value) { stats.emplace_back(key, PSTRING() << value); }; + + add_stat("started", "true"); + auto r_mem_stat = td::mem_stat(); + auto r_total_mem_stat = td::get_total_mem_stat(); + td::uint64 celldb_size = 0; + bool ok_celldb_size = rocks_db_->GetIntProperty("rocksdb.total-sst-files-size", &celldb_size); + if (celldb_size > 0 && r_mem_stat.is_ok() && r_total_mem_stat.is_ok() && ok_celldb_size) { + auto mem_stat = r_mem_stat.move_as_ok(); + auto total_mem_stat = r_total_mem_stat.move_as_ok(); + add_stat("rss", td::format::as_size(mem_stat.resident_size_)); + add_stat("available_ram", td::format::as_size(total_mem_stat.available_ram)); + add_stat("total_ram", td::format::as_size(total_mem_stat.total_ram)); + add_stat("actual_ram_to_celldb_ratio", double(total_mem_stat.available_ram) / double(celldb_size)); + add_stat("if_restarted_ram_to_celldb_ratio", + double(total_mem_stat.available_ram + mem_stat.resident_size_ - 10 * (td::uint64(1) << 30)) / + double(celldb_size)); + add_stat("max_possible_ram_to_celldb_ratio", double(total_mem_stat.total_ram) / double(celldb_size)); + } + + return stats; + // do not clear statistics, it is needed for flush_db_stats +} void CellDbIn::flush_db_stats() { if (opts_->get_disable_rocksdb_stats()) { return; } - auto stats = td::RocksDb::statistics_to_string(statistics_) + snapshot_statistics_->to_string() + - cell_db_statistics_.to_string(); + + auto celldb_stats = prepare_stats(); + td::StringBuilder ss; + for (auto& [key, value] : celldb_stats) { + ss << "ton.celldb." << key << " " << value << "\n"; + } + + auto stats = + td::RocksDb::statistics_to_string(statistics_) + snapshot_statistics_->to_string() + ss.as_cslice().str(); auto to_file_r = td::FileFd::open(path_ + "/db_stats.txt", td::FileFd::Truncate | td::FileFd::Create | td::FileFd::Write, 0644); if (to_file_r.is_error()) { @@ -284,8 +353,11 @@ void CellDbIn::gc_cont(BlockHandle handle) { } void CellDbIn::gc_cont2(BlockHandle handle) { + TD_PERF_COUNTER(celldb_gc_cell); td::PerfWarningTimer timer{"gccell", 0.1}; + td::PerfWarningTimer timer_all{"gccell_all", 0.05}; + td::PerfWarningTimer timer_get_keys{"gccell_get_keys", 0.05}; auto key_hash = get_key_hash(handle->id()); auto FR = get_block(key_hash); FR.ensure(); @@ -304,22 +376,41 @@ void CellDbIn::gc_cont2(BlockHandle handle) { P.prev = P.next; N.next = N.prev; } + timer_get_keys.reset(); + td::PerfWarningTimer timer_boc{"gccell_boc", 0.05}; auto cell = boc_->load_cell(F.root_hash.as_slice()).move_as_ok(); boc_->dec(cell); boc_->prepare_commit().ensure(); vm::CellStorer stor{*cell_db_}; + timer_boc.reset(); + + td::PerfWarningTimer timer_write_batch{"gccell_write_batch", 0.05}; cell_db_->begin_write_batch().ensure(); boc_->commit(stor).ensure(); + cell_db_->erase(get_key(key_hash)).ensure(); set_block(F.prev, std::move(P)); set_block(F.next, std::move(N)); cell_db_->commit_write_batch().ensure(); alarm_timestamp() = td::Timestamp::now(); + timer_write_batch.reset(); + + td::PerfWarningTimer timer_free_cells{"gccell_free_cells", 0.05}; + auto before = td::ref_get_delete_count(); + cell = {}; + auto after = td::ref_get_delete_count(); + if (timer_free_cells.elapsed() > 0.04) { + LOG(ERROR) << "deleted " << after - before << " cells"; + } + timer_free_cells.reset(); - boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); - td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); + td::PerfWarningTimer timer_finish{"gccell_finish", 0.05}; + if (!opts_->get_celldb_in_memory()) { + boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); + td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); + } DCHECK(get_block(key_hash).is_error()); if (!opts_->get_disable_rocksdb_stats()) { @@ -329,6 +420,8 @@ void CellDbIn::gc_cont2(BlockHandle handle) { last_deleted_mc_state_ = handle->id().seqno(); } LOG(DEBUG) << "Deleted state " << handle->id().to_str(); + timer_finish.reset(); + timer_all.reset(); } void CellDbIn::skip_gc() { @@ -401,7 +494,7 @@ void CellDbIn::migrate_cells() { boc_->set_loader(std::make_unique(*loader)).ensure(); cell_db_->begin_write_batch().ensure(); td::uint32 checked = 0, migrated = 0; - for (auto it = cells_to_migrate_.begin(); it != cells_to_migrate_.end() && checked < 128; ) { + for (auto it = cells_to_migrate_.begin(); it != cells_to_migrate_.end() && checked < 128;) { ++checked; td::Bits256 hash = *it; it = cells_to_migrate_.erase(it); @@ -438,7 +531,32 @@ void CellDbIn::migrate_cells() { } } +void CellDb::prepare_stats(td::Promise>> promise) { + promise.set_value(decltype(prepared_stats_)(prepared_stats_)); +} + +void CellDb::update_stats(td::Result>> r_stats) { + if (r_stats.is_error()) { + LOG(ERROR) << "error updating stats: " << r_stats.error(); + } else { + prepared_stats_ = r_stats.move_as_ok(); + } + alarm_timestamp() = td::Timestamp::in(2.0); +} + +void CellDb::alarm() { + send_closure(cell_db_, &CellDbIn::prepare_stats, td::promise_send_closure(actor_id(this), &CellDb::update_stats)); +} + void CellDb::load_cell(RootHash hash, td::Promise> promise) { + if (in_memory_boc_) { + auto result = in_memory_boc_->load_root_thread_safe(hash.as_slice()); + if (result.is_ok()) { + return async_apply("load_cell_result", std::move(promise), std::move(result)); + } else { + LOG(ERROR) << "load_root_thread_safe failed - this is suspicious"; + } + } if (!started_) { td::actor::send_closure(cell_db_, &CellDbIn::load_cell, hash, std::move(promise)); } else { @@ -496,12 +614,24 @@ td::BufferSlice CellDbIn::DbEntry::release() { return create_serialize_tl_object(create_tl_block_id(block_id), prev, next, root_hash); } -std::string CellDbIn::CellDbStatistics::to_string() { - td::StringBuilder ss; - ss << "ton.celldb.store_cell.micros " << store_cell_time_.to_string() << "\n"; - ss << "ton.celldb.gc_cell.micros " << gc_cell_time_.to_string() << "\n"; - ss << "ton.celldb.total_time.micros : " << (td::Timestamp::now().at() - stats_start_time_.at()) * 1e6 << "\n"; - return ss.as_cslice().str(); +std::vector> CellDbIn::CellDbStatistics::prepare_stats() { + std::vector> stats; + stats.emplace_back("store_cell.micros", PSTRING() << store_cell_time_.to_string()); + stats.emplace_back("gc_cell.micros", PSTRING() << gc_cell_time_.to_string()); + stats.emplace_back("total_time.micros", PSTRING() << (td::Timestamp::now().at() - stats_start_time_.at()) * 1e6); + stats.emplace_back("in_memory", PSTRING() << bool(in_memory_load_time_)); + if (in_memory_load_time_) { + stats.emplace_back("in_memory_load_time", PSTRING() << in_memory_load_time_.value()); + } + if (boc_stats_) { + stats.emplace_back("cells_count", PSTRING() << boc_stats_->cells_total_count); + stats.emplace_back("cells_size", PSTRING() << boc_stats_->cells_total_size); + stats.emplace_back("roots_count", PSTRING() << boc_stats_->roots_total_count); + for (auto& [key, value] : boc_stats_->custom_stats) { + stats.emplace_back(key, value); + } + } + return stats; } } // namespace validator diff --git a/validator/db/celldb.hpp b/validator/db/celldb.hpp index 335d8a08e..7cd36689b 100644 --- a/validator/db/celldb.hpp +++ b/validator/db/celldb.hpp @@ -29,9 +29,12 @@ #include "db-utils.h" #include "td/db/RocksDb.h" +#include + namespace rocksdb { class Statistics; -} +class DB; +} // namespace rocksdb namespace ton { @@ -58,6 +61,7 @@ class CellDbIn : public CellDbBase { public: using KeyHash = td::Bits256; + std::vector> prepare_stats(); void load_cell(RootHash hash, td::Promise> promise); void store_cell(BlockIdExt block_id, td::Ref cell, td::Promise> promise); void get_cell_db_reader(td::Promise> promise); @@ -111,13 +115,15 @@ class CellDbIn : public CellDbBase { std::string path_; td::Ref opts_; - std::unique_ptr boc_; + std::shared_ptr boc_; std::shared_ptr cell_db_; + std::shared_ptr rocks_db_; std::function on_load_callback_; std::set cells_to_migrate_; td::Timestamp migrate_after_ = td::Timestamp::never(); bool migration_active_ = false; + std::optional in_memory_load_time_; struct MigrationStats { td::Timer start_; @@ -133,8 +139,10 @@ class CellDbIn : public CellDbBase { PercentileStats store_cell_time_; PercentileStats gc_cell_time_; td::Timestamp stats_start_time_ = td::Timestamp::now(); + std::optional in_memory_load_time_; + std::optional boc_stats_; - std::string to_string(); + std::vector> prepare_stats(); void clear() { *this = CellDbStatistics{}; } @@ -162,12 +170,25 @@ class CellDbIn : public CellDbBase { class CellDb : public CellDbBase { public: + void prepare_stats(td::Promise>> promise); void load_cell(RootHash hash, td::Promise> promise); void store_cell(BlockIdExt block_id, td::Ref cell, td::Promise> promise); void update_snapshot(std::unique_ptr snapshot) { + CHECK(!opts_->get_celldb_in_memory()); + if (!started_) { + alarm(); + } started_ = true; boc_->set_loader(std::make_unique(std::move(snapshot), on_load_callback_)).ensure(); } + void set_in_memory_boc(std::shared_ptr in_memory_boc) { + CHECK(opts_->get_celldb_in_memory()); + if (!started_) { + alarm(); + } + started_ = true; + in_memory_boc_ = std::move(in_memory_boc); + } void get_cell_db_reader(td::Promise> promise); void get_last_deleted_mc_state(td::Promise promise); @@ -185,9 +206,14 @@ class CellDb : public CellDbBase { td::actor::ActorOwn cell_db_; std::unique_ptr boc_; + std::shared_ptr in_memory_boc_; bool started_ = false; + std::vector> prepared_stats_{{"started", "false"}}; std::function on_load_callback_; + + void update_stats(td::Result>> stats); + void alarm() override; }; } // namespace validator diff --git a/validator/db/fileref.cpp b/validator/db/fileref.cpp index 378e382c9..feefd1f9c 100644 --- a/validator/db/fileref.cpp +++ b/validator/db/fileref.cpp @@ -79,14 +79,14 @@ std::string Block::filename() const { std::string Block::filename_short() const { char s[33]; - sprintf(s, "%llx", static_cast(block_id.id.shard)); + snprintf(s, sizeof(s), "%llx", static_cast(block_id.id.shard)); return PSTRING() << "block_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_" << hash().to_hex(); } std::string BlockShort::filename_short() const { char s[33]; - sprintf(s, "%llx", static_cast(block_id.shard)); + snprintf(s, sizeof(s), "%llx", static_cast(block_id.shard)); return PSTRING() << "block_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex(); } @@ -116,14 +116,14 @@ std::string PersistentState::filename() const { std::string PersistentState::filename_short() const { char s[33]; - sprintf(s, "%llx", static_cast(block_id.id.shard)); + snprintf(s, sizeof(s), "%llx", static_cast(block_id.id.shard)); return PSTRING() << "state_" << masterchain_block_id.seqno() << "_" << block_id.id.workchain << "_" << s << "_" << hash().to_hex(); } std::string PersistentStateShort::filename_short() const { char s[33]; - sprintf(s, "%llx", static_cast(shard_id.shard)); + snprintf(s, sizeof(s), "%llx", static_cast(shard_id.shard)); return PSTRING() << "state_" << masterchain_seqno << "_" << shard_id.workchain << "_" << s << "_" << hash().to_hex(); } @@ -137,14 +137,14 @@ std::string Proof::filename() const { std::string Proof::filename_short() const { char s[33]; - sprintf(s, "%llx", static_cast(block_id.id.shard)); + snprintf(s, sizeof(s), "%llx", static_cast(block_id.id.shard)); return PSTRING() << "proof_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_" << hash().to_hex(); } std::string ProofShort::filename_short() const { char s[33]; - sprintf(s, "%llx", static_cast(block_id.shard)); + snprintf(s, sizeof(s), "%llx", static_cast(block_id.shard)); return PSTRING() << "proof_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex(); } @@ -158,14 +158,14 @@ std::string ProofLink::filename() const { std::string ProofLink::filename_short() const { char s[33]; - sprintf(s, "%llx", static_cast(block_id.id.shard)); + snprintf(s, sizeof(s), "%llx", static_cast(block_id.id.shard)); return PSTRING() << "prooflink_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_" << hash().to_hex(); } std::string ProofLinkShort::filename_short() const { char s[33]; - sprintf(s, "%llx", static_cast(block_id.shard)); + snprintf(s, sizeof(s), "%llx", static_cast(block_id.shard)); return PSTRING() << "prooflink_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex(); } @@ -179,14 +179,14 @@ std::string Signatures::filename() const { std::string Signatures::filename_short() const { char s[33]; - sprintf(s, "%llx", static_cast(block_id.id.shard)); + snprintf(s, sizeof(s), "%llx", static_cast(block_id.id.shard)); return PSTRING() << "signatures_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_" << hash().to_hex(); } std::string SignaturesShort::filename_short() const { char s[33]; - sprintf(s, "%llx", static_cast(block_id.shard)); + snprintf(s, sizeof(s), "%llx", static_cast(block_id.shard)); return PSTRING() << "signatures_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex(); } @@ -202,14 +202,14 @@ std::string Candidate::filename() const { std::string Candidate::filename_short() const { char s[33]; - sprintf(s, "%llx", static_cast(block_id.id.shard)); + snprintf(s, sizeof(s), "%llx", static_cast(block_id.id.shard)); return PSTRING() << "candidate_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_" << hash().to_hex(); } std::string CandidateShort::filename_short() const { char s[33]; - sprintf(s, "%llx", static_cast(block_id.shard)); + snprintf(s, sizeof(s), "%llx", static_cast(block_id.shard)); return PSTRING() << "candidate_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex(); } @@ -223,14 +223,14 @@ std::string CandidateRef::filename() const { std::string CandidateRef::filename_short() const { char s[33]; - sprintf(s, "%llx", static_cast(block_id.id.shard)); + snprintf(s, sizeof(s), "%llx", static_cast(block_id.id.shard)); return PSTRING() << "candidateref_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_" << hash().to_hex(); } std::string CandidateRefShort::filename_short() const { char s[33]; - sprintf(s, "%llx", static_cast(block_id.shard)); + snprintf(s, sizeof(s), "%llx", static_cast(block_id.shard)); return PSTRING() << "candidateref_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex(); } @@ -245,14 +245,14 @@ std::string BlockInfo::filename() const { std::string BlockInfo::filename_short() const { char s[33]; - sprintf(s, "%llx", static_cast(block_id.id.shard)); + snprintf(s, sizeof(s), "%llx", static_cast(block_id.id.shard)); return PSTRING() << "info_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_" << hash().to_hex(); } std::string BlockInfoShort::filename_short() const { char s[33]; - sprintf(s, "%llx", static_cast(block_id.shard)); + snprintf(s, sizeof(s), "%llx", static_cast(block_id.shard)); return PSTRING() << "info_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex(); } diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 31a8162e0..f15220de1 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -439,6 +439,7 @@ void RootDb::allow_block_gc(BlockIdExt block_id, td::Promise promise) { void RootDb::prepare_stats(td::Promise>> promise) { auto merger = StatsMerger::create(std::move(promise)); + td::actor::send_closure(cell_db_, &CellDb::prepare_stats, merger.make_promise("celldb.")); } void RootDb::truncate(BlockSeqno seqno, ConstBlockHandle handle, td::Promise promise) { diff --git a/validator/downloaders/wait-block-state.cpp b/validator/downloaders/wait-block-state.cpp index 0ae82beaa..f8d2cdcbe 100644 --- a/validator/downloaders/wait-block-state.cpp +++ b/validator/downloaders/wait-block-state.cpp @@ -230,6 +230,7 @@ void WaitBlockState::got_block_data(td::Ref data) { } void WaitBlockState::apply() { + TD_PERF_COUNTER(apply_block_to_state); td::PerfWarningTimer t{"applyblocktostate", 0.1}; auto S = prev_state_.write().apply_block(handle_->id(), block_); if (S.is_error()) { diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index e9d89cd05..53595fa86 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -4775,11 +4775,11 @@ bool Collator::check_block_overload() { } char buffer[17]; if (history_weight(overload_history_) >= 0) { - sprintf(buffer, "%016llx", (unsigned long long)overload_history_); + snprintf(buffer, sizeof(buffer), "%016llx", (unsigned long long)overload_history_); LOG(INFO) << "want_split set because of overload history " << buffer; want_split_ = true; } else if (history_weight(underload_history_) >= 0) { - sprintf(buffer, "%016llx", (unsigned long long)underload_history_); + snprintf(buffer, sizeof(buffer), "%016llx", (unsigned long long)underload_history_); LOG(INFO) << "want_merge set because of underload history " << buffer; want_merge_ = true; } diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index a9cfa31a5..39f9cc9ff 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -1860,7 +1860,7 @@ void LiteQuery::perform_getConfigParams(BlockIdExt blkid, int mode, std::vector< request_mc_block_data_state(blkid); } else { // get configuration from previous key block - load_prevKeyBlock(blkid, [this, blkid, mode, param_list = std::move(param_list)]( + load_prevKeyBlock(blkid, [this, mode, param_list = std::move(param_list)]( td::Result>> res) mutable { if (res.is_error()) { this->abort_query(res.move_as_error()); @@ -2057,7 +2057,7 @@ void LiteQuery::perform_lookupBlockWithProof(BlockId blkid, BlockIdExt mc_blkid, ton::AccountIdPrefixFull pfx{blkid.workchain, blkid.shard}; auto P = td::PromiseCreator::lambda( - [Self = actor_id(this), mc_blkid, manager = manager_, mode, pfx](td::Result res) { + [Self = actor_id(this), mc_blkid, manager = manager_, pfx](td::Result res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); return; @@ -2073,7 +2073,7 @@ void LiteQuery::perform_lookupBlockWithProof(BlockId blkid, BlockIdExt mc_blkid, } LOG(DEBUG) << "requesting data for block " << handle->id().to_str(); td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, handle, - [Self, mc_ref_blkid = handle->masterchain_ref_block(), mc_blkid, pfx, mode](td::Result> res) { + [Self, mc_ref_blkid = handle->masterchain_ref_block(), pfx](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index 0a710b5f0..0ab216f71 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -290,6 +290,7 @@ td::Result, td::Ref>> ShardStateQ::spl } td::Result ShardStateQ::serialize() const { + TD_PERF_COUNTER(serialize_state); td::PerfWarningTimer perf_timer_{"serializestate", 0.1}; if (!data.is_null()) { return data.clone(); @@ -314,6 +315,7 @@ td::Result ShardStateQ::serialize() const { } td::Status ShardStateQ::serialize_to_file(td::FileFd& fd) const { + TD_PERF_COUNTER(serialize_state_to_file); td::PerfWarningTimer perf_timer_{"serializestate", 0.1}; if (!data.is_null()) { auto cur_data = data.clone(); diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 2f80e28ce..e15def2af 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -370,6 +370,10 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } + void prepare_actor_stats(td::Promise promise) override { + UNREACHABLE(); + } + void prepare_perf_timer_stats(td::Promise> promise) override { UNREACHABLE(); } diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 648a4a1b5..a43d4a703 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -432,6 +432,10 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } + void prepare_actor_stats(td::Promise promise) override { + UNREACHABLE(); + } + void prepare_perf_timer_stats(td::Promise> promise) override { UNREACHABLE(); } diff --git a/validator/manager.cpp b/validator/manager.cpp index cb53f1057..5d1e6753a 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1622,6 +1622,7 @@ void ValidatorManagerImpl::send_block_broadcast(BlockBroadcast broadcast, bool c void ValidatorManagerImpl::start_up() { db_ = create_db_actor(actor_id(this), db_root_, opts_); + actor_stats_ = td::actor::create_actor("actor_stats"); lite_server_cache_ = create_liteserver_cache_actor(actor_id(this), db_root_); token_manager_ = td::actor::create_actor("tokenmanager"); td::mkdir(db_root_ + "/tmp/").ensure(); @@ -2137,8 +2138,8 @@ void ValidatorManagerImpl::update_shards() { new_next_validator_groups_.emplace(val_group_id, std::move(it->second)); } else { new_next_validator_groups_.emplace( - val_group_id, - ValidatorGroupEntry{create_validator_group(val_group_id, shard, val_set, key_seqno, opts, started_), shard}); + val_group_id, ValidatorGroupEntry{ + create_validator_group(val_group_id, shard, val_set, key_seqno, opts, started_), shard}); } } } @@ -2771,6 +2772,10 @@ void ValidatorManagerImpl::send_peek_key_block_request() { send_get_next_key_blocks_request(last_known_key_block_handle_->id(), 1, std::move(P)); } +void ValidatorManagerImpl::prepare_actor_stats(td::Promise promise) { + send_closure(actor_stats_, &td::actor::ActorStats::prepare_stats, std::move(promise)); +} + void ValidatorManagerImpl::prepare_stats(td::Promise>> promise) { auto merger = StatsMerger::create(std::move(promise)); @@ -2803,6 +2808,9 @@ void ValidatorManagerImpl::prepare_stats(td::Promise R) mutable { @@ -3026,18 +3034,18 @@ void ValidatorManagerImpl::get_block_state_for_litequery(BlockIdExt block_id, promise.set_result(R.move_as_ok()); return; } - td::actor::send_closure(manager, &ValidatorManagerImpl::get_block_handle_for_litequery, - block_id, [manager, promise = std::move(promise)](td::Result R) mutable { - TRY_RESULT_PROMISE(promise, handle, std::move(R)); - td::actor::send_closure_later(manager, &ValidatorManager::get_shard_state_from_db, std::move(handle), - std::move(promise)); - }); + td::actor::send_closure(manager, &ValidatorManagerImpl::get_block_handle_for_litequery, block_id, + [manager, promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, handle, std::move(R)); + td::actor::send_closure_later(manager, &ValidatorManager::get_shard_state_from_db, + std::move(handle), std::move(promise)); + }); }); } } void ValidatorManagerImpl::get_block_by_lt_for_litequery(AccountIdPrefixFull account, LogicalTime lt, - td::Promise promise) { + td::Promise promise) { get_block_by_lt_from_db( account, lt, [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { if (R.is_ok() && R.ok()->is_applied()) { @@ -3050,7 +3058,7 @@ void ValidatorManagerImpl::get_block_by_lt_for_litequery(AccountIdPrefixFull acc } void ValidatorManagerImpl::get_block_by_unix_time_for_litequery(AccountIdPrefixFull account, UnixTime ts, - td::Promise promise) { + td::Promise promise) { get_block_by_unix_time_from_db( account, ts, [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { if (R.is_ok() && R.ok()->is_applied()) { @@ -3063,7 +3071,7 @@ void ValidatorManagerImpl::get_block_by_unix_time_for_litequery(AccountIdPrefixF } void ValidatorManagerImpl::get_block_by_seqno_for_litequery(AccountIdPrefixFull account, BlockSeqno seqno, - td::Promise promise) { + td::Promise promise) { get_block_by_seqno_from_db( account, seqno, [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { diff --git a/validator/manager.hpp b/validator/manager.hpp index 28cba9704..289498379 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -21,6 +21,7 @@ #include "common/refcnt.hpp" #include "interfaces/validator-manager.h" #include "interfaces/db.h" +#include "td/actor/ActorStats.h" #include "td/actor/PromiseFuture.h" #include "td/utils/SharedSlice.h" #include "td/utils/buffer.h" @@ -582,6 +583,8 @@ class ValidatorManagerImpl : public ValidatorManager { void prepare_stats(td::Promise>> promise) override; + void prepare_actor_stats(td::Promise promise) override; + void prepare_perf_timer_stats(td::Promise> promise) override; void add_perf_timer_stat(std::string name, double duration) override; @@ -678,6 +681,7 @@ class ValidatorManagerImpl : public ValidatorManager { private: std::unique_ptr callback_; td::actor::ActorOwn db_; + td::actor::ActorOwn actor_stats_; bool started_ = false; bool allow_validate_ = false; diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index 516d8177f..ab38a6e99 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -22,6 +22,7 @@ #include "ton/ton-io.hpp" #include "common/delay.h" #include "td/utils/filesystem.h" +#include "td/utils/HashSet.h" namespace ton { @@ -227,17 +228,17 @@ void AsyncStateSerializer::got_masterchain_handle(BlockHandle handle) { class CachedCellDbReader : public vm::CellDbReader { public: CachedCellDbReader(std::shared_ptr parent, - std::shared_ptr>> cache) + std::shared_ptr cache) : parent_(std::move(parent)), cache_(std::move(cache)) { } td::Result> load_cell(td::Slice hash) override { ++total_reqs_; DCHECK(hash.size() == 32); if (cache_) { - auto it = cache_->find(td::Bits256{(const unsigned char*)hash.data()}); + auto it = cache_->find(hash); if (it != cache_->end()) { ++cached_reqs_; - TRY_RESULT(loaded_cell, it->second->load_cell()); + TRY_RESULT(loaded_cell, (*it)->load_cell()); return loaded_cell.data_cell; } } @@ -248,7 +249,7 @@ class CachedCellDbReader : public vm::CellDbReader { } private: std::shared_ptr parent_; - std::shared_ptr>> cache_; + std::shared_ptr cache_; td::uint64 total_reqs_ = 0; td::uint64 cached_reqs_ = 0; @@ -272,10 +273,9 @@ void AsyncStateSerializer::PreviousStateCache::prepare_cache(ShardIdFull shard) td::Timer timer; LOG(WARNING) << "Preloading previous persistent state for shard " << shard.to_str() << " (" << cur_shards.size() << " files)"; - std::map> cells; + vm::CellHashSet cells; std::function)> dfs = [&](td::Ref cell) { - td::Bits256 hash = cell->get_hash().bits(); - if (!cells.emplace(hash, cell).second) { + if (!cells.insert(cell).second) { return; } bool is_special; @@ -303,7 +303,7 @@ void AsyncStateSerializer::PreviousStateCache::prepare_cache(ShardIdFull shard) dfs(r_root.move_as_ok()); } LOG(WARNING) << "Preloaded previous state: " << cells.size() << " cells in " << timer.elapsed() << "s"; - cache = std::make_shared>>(std::move(cells)); + cache = std::make_shared(std::move(cells)); } void AsyncStateSerializer::got_masterchain_state(td::Ref state, @@ -322,15 +322,18 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref state shards_.push_back(v->top_block_id()); } - auto write_data = [shard = state->get_shard(), hash = state->root_cell()->get_hash(), cell_db_reader, + auto write_data = [shard = state->get_shard(), root = state->root_cell(), cell_db_reader, previous_state_cache = previous_state_cache_, fast_serializer_enabled = opts_->get_fast_state_serializer_enabled(), cancellation_token = cancellation_token_source_.get_cancellation_token()](td::FileFd& fd) mutable { + if (!cell_db_reader) { + return vm::std_boc_serialize_to_file(root, fd, 31, std::move(cancellation_token)); + } if (fast_serializer_enabled) { previous_state_cache->prepare_cache(shard); } auto new_cell_db_reader = std::make_shared(cell_db_reader, previous_state_cache->cache); - auto res = vm::std_boc_serialize_to_file_large(new_cell_db_reader, hash, fd, 31, std::move(cancellation_token)); + auto res = vm::std_boc_serialize_to_file_large(new_cell_db_reader, root->get_hash(), fd, 31, std::move(cancellation_token)); new_cell_db_reader->print_stats(); return res; }; @@ -384,15 +387,18 @@ void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Refid().id.to_str(); - auto write_data = [shard = state->get_shard(), hash = state->root_cell()->get_hash(), cell_db_reader, + auto write_data = [shard = state->get_shard(), root = state->root_cell(), cell_db_reader, previous_state_cache = previous_state_cache_, fast_serializer_enabled = opts_->get_fast_state_serializer_enabled(), cancellation_token = cancellation_token_source_.get_cancellation_token()](td::FileFd& fd) mutable { + if (!cell_db_reader) { + return vm::std_boc_serialize_to_file(root, fd, 31, std::move(cancellation_token)); + } if (fast_serializer_enabled) { previous_state_cache->prepare_cache(shard); } auto new_cell_db_reader = std::make_shared(cell_db_reader, previous_state_cache->cache); - auto res = vm::std_boc_serialize_to_file_large(new_cell_db_reader, hash, fd, 31, std::move(cancellation_token)); + auto res = vm::std_boc_serialize_to_file_large(new_cell_db_reader, root->get_hash(), fd, 31, std::move(cancellation_token)); new_cell_db_reader->print_stats(); return res; }; diff --git a/validator/state-serializer.hpp b/validator/state-serializer.hpp index 68606d1ea..b38a216b7 100644 --- a/validator/state-serializer.hpp +++ b/validator/state-serializer.hpp @@ -51,7 +51,7 @@ class AsyncStateSerializer : public td::actor::Actor { std::vector shards_; struct PreviousStateCache { std::vector> state_files; - std::shared_ptr>> cache; + std::shared_ptr cache; std::vector cur_shards; void prepare_cache(ShardIdFull shard); diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index 900a682fd..203aa5ebc 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -138,6 +138,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool get_celldb_preload_all() const override { return celldb_preload_all_; } + bool get_celldb_in_memory() const override { + return celldb_in_memory_; + } td::optional get_catchain_max_block_delay() const override { return catchain_max_block_delay_; } @@ -230,6 +233,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_celldb_preload_all(bool value) override { celldb_preload_all_ = value; } + void set_celldb_in_memory(bool value) override { + celldb_in_memory_ = value; + } void set_catchain_max_block_delay(double value) override { catchain_max_block_delay_ = value; } @@ -295,6 +301,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { td::optional celldb_cache_size_; bool celldb_direct_io_ = false; bool celldb_preload_all_ = false; + bool celldb_in_memory_ = false; td::optional catchain_max_block_delay_, catchain_max_block_delay_slow_; bool state_serializer_enabled_ = true; td::Ref collator_options_{true}; diff --git a/validator/validator.h b/validator/validator.h index 1a78d2295..45e617f90 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -102,6 +102,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual BlockSeqno sync_upto() const = 0; virtual std::string get_session_logs_file() const = 0; virtual td::uint32 get_celldb_compress_depth() const = 0; + virtual bool get_celldb_in_memory() const = 0; virtual size_t get_max_open_archive_files() const = 0; virtual double get_archive_preload_period() const = 0; virtual bool get_disable_rocksdb_stats() const = 0; @@ -141,6 +142,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_celldb_cache_size(td::uint64 value) = 0; virtual void set_celldb_direct_io(bool value) = 0; virtual void set_celldb_preload_all(bool value) = 0; + virtual void set_celldb_in_memory(bool value) = 0; virtual void set_catchain_max_block_delay(double value) = 0; virtual void set_catchain_max_block_delay_slow(double value) = 0; virtual void set_state_serializer_enabled(bool value) = 0; @@ -275,6 +277,7 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void run_ext_query(td::BufferSlice data, td::Promise promise) = 0; virtual void prepare_stats(td::Promise>> promise) = 0; + virtual void prepare_actor_stats(td::Promise promise) = 0; virtual void prepare_perf_timer_stats(td::Promise> promise) = 0; virtual void add_perf_timer_stat(std::string name, double duration) = 0;