From e7c4f3adad9da662e4822411af0612160c58724b Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Thu, 5 Oct 2023 16:32:49 -0400 Subject: [PATCH 1/8] Update check for out-of-memory (oom) score and allow chainbase to flush pages. --- libraries/chain/controller.cpp | 10 ++++++++-- libraries/chainbase | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index df5d2c8bde..3b10242cef 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -3486,8 +3486,14 @@ void controller::validate_db_available_size() const { // give a change to chainbase to write some pages to disk if memory becomes scarce. if (is_write_window()) { - if (auto flushed_pages = mutable_db().check_memory_and_flush_if_needed()) { - ilog("CHAINBASE: flushed ${p} pages to disk to decrease memory pressure", ("p", flushed_pages)); + auto check_time = fc::time_point::now(); + if (auto res = mutable_db().check_memory_and_flush_if_needed()) { + auto [oom, flushed_pages] = *res; + if (flushed_pages > 0) { + auto duration = (fc::time_point::now() - check_time).count()/1000; // in milliseconds + ilog("chainbase: oom = ${oom}, flushed ${p} state pages to disk (in ${dur} ms) to decrease memory pressure", + ("oom", oom)("p", flushed_pages)("dur", duration)); + } } } } diff --git a/libraries/chainbase b/libraries/chainbase index 3bfb0d00b6..11f90901ca 160000 --- a/libraries/chainbase +++ b/libraries/chainbase @@ -1 +1 @@ -Subproject commit 3bfb0d00b61087bff804c622ffa5929fec2fa4ed +Subproject commit 11f90901ca5afee68f0ad0a1decb21a60e3139a3 From 45c05b80d89667e3e2dd23ae94d6c40127b3fa3f Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Fri, 6 Oct 2023 12:23:27 -0400 Subject: [PATCH 2/8] Update chainbase to branch tip. --- libraries/chainbase | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chainbase b/libraries/chainbase index 11f90901ca..ba91bc2d5f 160000 --- a/libraries/chainbase +++ b/libraries/chainbase @@ -1 +1 @@ -Subproject commit 11f90901ca5afee68f0ad0a1decb21a60e3139a3 +Subproject commit ba91bc2d5f21641f971d58bafac8212524caaf08 From 5e4cda775c24b72c987405a7a379851884ad9b77 Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Fri, 6 Oct 2023 17:24:03 -0400 Subject: [PATCH 3/8] Update chainbase branch to tip. --- libraries/chainbase | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chainbase b/libraries/chainbase index ba91bc2d5f..c55e62ed23 160000 --- a/libraries/chainbase +++ b/libraries/chainbase @@ -1 +1 @@ -Subproject commit ba91bc2d5f21641f971d58bafac8212524caaf08 +Subproject commit c55e62ed236528e4a928ea3a55686f24afea6679 From ebe185080ee94aeb155d6280156e115ccad14317 Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Sat, 7 Oct 2023 07:48:26 -0400 Subject: [PATCH 4/8] Update chainbase to branch tip. --- libraries/chainbase | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chainbase b/libraries/chainbase index c55e62ed23..923444c222 160000 --- a/libraries/chainbase +++ b/libraries/chainbase @@ -1 +1 @@ -Subproject commit c55e62ed236528e4a928ea3a55686f24afea6679 +Subproject commit 923444c22235fa879d7ef6e13a298bb6578597e5 From b2d51530bb850260c8a43e4e20745daaacece52b Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Tue, 10 Oct 2023 08:56:11 -0400 Subject: [PATCH 5/8] Improve reporting on chainbase memory issues. --- libraries/chain/controller.cpp | 12 +++++++----- libraries/chainbase | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 3b10242cef..09c498417d 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -3484,15 +3484,17 @@ void controller::validate_db_available_size() const { const auto guard = my->conf.state_guard_size; EOS_ASSERT(free >= guard, database_guard_exception, "database free: ${f}, guard size: ${g}", ("f", free)("g",guard)); - // give a change to chainbase to write some pages to disk if memory becomes scarce. + // give chainbase a chance to write some pages to disk if memory becomes scarce. if (is_write_window()) { auto check_time = fc::time_point::now(); if (auto res = mutable_db().check_memory_and_flush_if_needed()) { - auto [oom, flushed_pages] = *res; - if (flushed_pages > 0) { + if (res->num_pages_written > 0) { auto duration = (fc::time_point::now() - check_time).count()/1000; // in milliseconds - ilog("chainbase: oom = ${oom}, flushed ${p} state pages to disk (in ${dur} ms) to decrease memory pressure", - ("oom", oom)("p", flushed_pages)("dur", duration)); + wlog("chainbase: oom = ${oom_prev}, flushed ${p} state pages to disk (in ${dur} ms) to decrease memory pressure", + ("oom_prev", res->oom_score_before)("p", res->num_pages_written)("dur", duration)); + wlog("chainbase: oom score after flush: ${oom_post}", ("oom_post", res->oom_score_after)); + wlog("chainbase: The system is running low in memory for running nodeos. " + "If this is not a temporary condition, consider increasing the available RAM"); } } } diff --git a/libraries/chainbase b/libraries/chainbase index 923444c222..b782713659 160000 --- a/libraries/chainbase +++ b/libraries/chainbase @@ -1 +1 @@ -Subproject commit 923444c22235fa879d7ef6e13a298bb6578597e5 +Subproject commit b7827136592b4a0403e0ecd5a7cdbebe71b32bef From 46dd32e935bb009a8809bb2f26c9145e2a253ccc Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Tue, 10 Oct 2023 11:27:14 -0400 Subject: [PATCH 6/8] Improve `oom` log messages. --- libraries/chain/controller.cpp | 2 ++ libraries/chainbase | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 09c498417d..0b42650e4c 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -3495,6 +3495,8 @@ void controller::validate_db_available_size() const { wlog("chainbase: oom score after flush: ${oom_post}", ("oom_post", res->oom_score_after)); wlog("chainbase: The system is running low in memory for running nodeos. " "If this is not a temporary condition, consider increasing the available RAM"); + } else { + ilog("chainbase: oom_score = ${oom_prev} within acceptable range, no action taken", ("oom_prev", res->oom_score_before)); } } } diff --git a/libraries/chainbase b/libraries/chainbase index b782713659..34a5033de1 160000 --- a/libraries/chainbase +++ b/libraries/chainbase @@ -1 +1 @@ -Subproject commit b7827136592b4a0403e0ecd5a7cdbebe71b32bef +Subproject commit 34a5033de1f23397725daa5994855cf283d3a2e5 From 049400deb6760a73023fd3bcfcefcd68814237ee Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Tue, 10 Oct 2023 12:42:20 -0400 Subject: [PATCH 7/8] Update comment for accuracy. --- libraries/chain/controller.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 0b42650e4c..00498af8ab 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -3484,7 +3484,7 @@ void controller::validate_db_available_size() const { const auto guard = my->conf.state_guard_size; EOS_ASSERT(free >= guard, database_guard_exception, "database free: ${f}, guard size: ${g}", ("f", free)("g",guard)); - // give chainbase a chance to write some pages to disk if memory becomes scarce. + // give chainbase a chance to flush its private pages to disk if memory becomes scarce. if (is_write_window()) { auto check_time = fc::time_point::now(); if (auto res = mutable_db().check_memory_and_flush_if_needed()) { From 6a27fba088fcf2b4c507917ee63f4926767ddcff Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Fri, 13 Oct 2023 10:59:46 -0400 Subject: [PATCH 8/8] Update chainbase to branch tip --- libraries/chainbase | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chainbase b/libraries/chainbase index 34a5033de1..580a743720 160000 --- a/libraries/chainbase +++ b/libraries/chainbase @@ -1 +1 @@ -Subproject commit 34a5033de1f23397725daa5994855cf283d3a2e5 +Subproject commit 580a743720a82cccbeb04fc84da9b270978d0609