Skip to content

Commit

Permalink
Merge pull request #32783 from vespa-engine/toregge/rewrite-searchcor…
Browse files Browse the repository at this point in the history 
…e-memory-flush-config-updater-unit-test-to-gtest

Rewrite searchcore memory flush config updater unit test to gtest.
  • Loading branch information
@geirst
geirst authored Nov 7, 2024
2 parents 9be0d54 + 495aaf3 commit c92512c
Show file tree
Hide file tree
Showing 3 changed files with 83 additions and 57 deletions.
2 changes: 1 addition & 1 deletion searchcore/src/tests/proton/server/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
vespa_add_executable(searchcore_proton_server_vespa_test_app TEST
SOURCES
vespa_testrunner.cpp
memory_flush_config_updater_test.cpp
memoryconfigstore_test.cpp
move_operation_limiter_test.cpp
threading_service_config_test.cpp
Expand All @@ -19,6 +18,7 @@ vespa_add_executable(searchcore_proton_server_gtest_test_app TEST
feeddebugger_test.cpp
feedstates_test.cpp
health_adapter_test.cpp
memory_flush_config_updater_test.cpp
DEPENDS
searchcore_server
searchcore_feedoperation
Expand Down
3 changes: 3 additions & 0 deletions searchcore/src/tests/proton/server/gtest_runner.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,7 @@

#include <vespa/vespalib/gtest/gtest.h>

#include <vespa/log/log.h>
LOG_SETUP("proton_server_test");

GTEST_MAIN_RUN_ALL_TESTS()
135 changes: 79 additions & 56 deletions searchcore/src/tests/proton/server/memory_flush_config_updater_test.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include <vespa/vespalib/testkit/test_kit.h>
#include <vespa/searchcore/proton/server/memory_flush_config_updater.h>
#include <vespa/vespalib/gtest/gtest.h>
#include <vespa/vespalib/util/size_literals.h>

using namespace proton;
using vespa::config::search::core::ProtonConfig;
using vespalib::HwInfo;

namespace memory_flush_config_updater_test {
inline namespace memory_flush_config_updater_test {

ProtonConfig::Flush::Memory
getConfig(int64_t maxMemory, int64_t eachMaxMemory, int64_t maxTlsSize,
Expand Down Expand Up @@ -59,14 +59,20 @@ struct Fixture
: strategy(std::make_shared<MemoryFlush>(MemoryFlushConfigUpdater::convertConfig(getDefaultConfig(), defaultMemory))),
updater(strategy, getDefaultConfig(), defaultMemory)
{}
~Fixture();
void assertStrategyConfig(uint64_t expMaxGlobalMemory, int64_t expMaxEachMemory, uint64_t expMaxGlobalTlsSize) {
EXPECT_EQUAL(expMaxGlobalMemory, strategy->getConfig().maxGlobalMemory);
EXPECT_EQUAL(expMaxEachMemory, strategy->getConfig().maxMemoryGain);
EXPECT_EQUAL(expMaxGlobalTlsSize, strategy->getConfig().maxGlobalTlsSize);
EXPECT_EQ(expMaxGlobalMemory, strategy->getConfig().maxGlobalMemory);
EXPECT_EQ(expMaxEachMemory, strategy->getConfig().maxMemoryGain);
EXPECT_EQ(expMaxGlobalTlsSize, strategy->getConfig().maxGlobalTlsSize);
}
void assertStrategyDiskConfig(double expGlobalDiskBloatFactor, double expDiskBloatFactor) {
EXPECT_APPROX(expGlobalDiskBloatFactor, strategy->getConfig().globalDiskBloatFactor, 0.00001);
EXPECT_APPROX(expDiskBloatFactor, strategy->getConfig().diskBloatFactor, 0.00001);
void assertStrategyConfig(const std::string& label, uint64_t expMaxGlobalMemory, int64_t expMaxEachMemory, uint64_t expMaxGlobalTlsSize) {
SCOPED_TRACE(label);
assertStrategyConfig(expMaxGlobalMemory, expMaxEachMemory, expMaxGlobalTlsSize);
}
void assertStrategyDiskConfig(const std::string& label, double expGlobalDiskBloatFactor, double expDiskBloatFactor) {
SCOPED_TRACE(label);
EXPECT_NEAR(expGlobalDiskBloatFactor, strategy->getConfig().globalDiskBloatFactor, 0.00001);
EXPECT_NEAR(expDiskBloatFactor, strategy->getConfig().diskBloatFactor, 0.00001);
}
void notifyDiskMemUsage(const ResourceUsageState &diskState, const ResourceUsageState &memoryState) {
updater.notifyDiskMemUsage(DiskMemUsageState(diskState, memoryState));
Expand All @@ -76,11 +82,7 @@ struct Fixture
}
};

TEST_F("require that strategy is updated when setting new config", Fixture)
{
f.updater.setConfig(getConfig(6, 3, 30));
TEST_DO(f.assertStrategyConfig(6, 3, 30));
}
Fixture::~Fixture() = default;

void
expectEqual(const MemoryFlush::Config & a, const MemoryFlush::Config & b) {
Expand All @@ -102,7 +104,17 @@ expectNotEqual(const MemoryFlush::Config & a, const MemoryFlush::Config & b) {
EXPECT_TRUE( b != a);
}

TEST("require that MemoryFlush::Config equal is correct") {
}

TEST(MemoryFlushConfigUpdaterTest, require_that_strategy_is_updated_when_setting_new_config)
{
Fixture f;
f.updater.setConfig(getConfig(6, 3, 30));
f.assertStrategyConfig(6, 3, 30);
}

TEST(MemoryFlushConfigUpdaterTest, require_that_MemoryFlush_Config_equal_is_correct)
{
MemoryFlush::Config a, b;
expectEqual(a, b);
a.maxGlobalMemory = 7;
Expand Down Expand Up @@ -131,112 +143,123 @@ TEST("require that MemoryFlush::Config equal is correct") {
expectEqual(a, b);
}

TEST("require that we use configured memory limits") {
TEST(MemoryFlushConfigUpdaterTest, require_that_we_use_configured_memory_limits)
{
auto cfg = MemoryFlushConfigUpdater::convertConfig(getConfig(6, 3, 30), defaultMemory);
EXPECT_EQUAL(cfg.maxGlobalMemory, 6u);
EXPECT_EQUAL(cfg.maxMemoryGain, 3);
EXPECT_EQ(cfg.maxGlobalMemory, 6u);
EXPECT_EQ(cfg.maxMemoryGain, 3);
}

TEST("require that we cap configured limits based on available memory") {
TEST(MemoryFlushConfigUpdaterTest, require_that_we_cap_configured_limits_based_on_available_memory)
{
const uint64_t LIMIT = defaultMemory.sizeBytes()/4;
auto cfg = MemoryFlushConfigUpdater::convertConfig(getConfig(4_Gi, 4_Gi, 30), defaultMemory);
EXPECT_EQUAL(cfg.maxGlobalMemory, LIMIT);
EXPECT_EQUAL(uint64_t(cfg.maxMemoryGain), LIMIT);
EXPECT_EQ(cfg.maxGlobalMemory, LIMIT);
EXPECT_EQ(uint64_t(cfg.maxMemoryGain), LIMIT);
}

TEST_F("require that strategy is updated with normal values if no limits are reached", Fixture)
TEST(MemoryFlushConfigUpdaterTest, require_that_strategy_is_updated_with_normal_values_if_no_limits_are_reached)
{
Fixture f;
f.updater.notifyDiskMemUsage(DiskMemUsageState());
TEST_DO(f.assertStrategyConfig(4, 1, 20));
f.assertStrategyConfig(4, 1, 20);
}

TEST_F("require that strategy is updated with conservative max tls size value if disk limit is reached", Fixture)
TEST(MemoryFlushConfigUpdaterTest, require_that_strategy_is_updated_with_conservative_max_tls_size_value_if_disk_limit_is_reached)
{
Fixture f;
f.notifyDiskMemUsage(aboveLimit(), belowLimit());
TEST_DO(f.assertStrategyConfig(4, 1, 12));
f.assertStrategyConfig(4, 1, 12);
}

TEST_F("require that strategy is updated with conservative max memory value if memory limit is reached", Fixture)
TEST(MemoryFlushConfigUpdaterTest, require_that_strategy_is_updated_with_conservative_max_memory_value_if_memory_limit_is_reached)
{
Fixture f;
f.notifyDiskMemUsage(belowLimit(), aboveLimit());
TEST_DO(f.assertStrategyConfig(2, 0, 20));
f.assertStrategyConfig(2, 0, 20);
}

TEST_F("require that strategy is updated with all conservative values if both limits are reached", Fixture)
TEST(MemoryFlushConfigUpdaterTest, require_that_strategy_is_updated_with_all_conservative_values_if_both_limits_are_reached)
{
Fixture f;
f.notifyDiskMemUsage(aboveLimit(), aboveLimit());
TEST_DO(f.assertStrategyConfig(2, 0, 12));
f.assertStrategyConfig(2, 0, 12);
}

TEST_F("require that last disk/memory usage state is remembered when setting new config", Fixture)
TEST(MemoryFlushConfigUpdaterTest, require_that_last_disk_and_memory_usage_state_is_remembered_when_setting_new_config)
{
Fixture f;
f.notifyDiskMemUsage(aboveLimit(), belowLimit());
f.updater.setConfig(getConfig(6, 3, 30));
TEST_DO(f.assertStrategyConfig(6, 3, 18));
f.assertStrategyConfig(6, 3, 18);
}

TEST_F("require that last config if remembered when setting new disk/memory usage state", Fixture)
TEST(MemoryFlushConfigUpdaterTest, require_that_last_config_if_remembered_when_setting_new_disk_and_memory_usage_state)
{
Fixture f;
f.updater.setConfig(getConfig(6, 3, 30));
f.notifyDiskMemUsage(aboveLimit(), belowLimit());
TEST_DO(f.assertStrategyConfig(6, 3, 18));
f.assertStrategyConfig(6, 3, 18);
}

TEST_F("Use conservative settings when above high watermark for disk usage", Fixture)
TEST(MemoryFlushConfigUpdaterTest, Use_conservative_settings_when_above_high_watermark_for_disk_usage)
{
Fixture f;
// The high watermark limit is 0.63 (0.7 * 0.9 (factor)).
f.notifyDiskMemUsage(ResourceUsageState(0.7, 0.62), belowLimit());
TEST_DO(f.assertStrategyConfig(4, 1, 20));
f.assertStrategyConfig("1st notify", 4, 1, 20);
f.notifyDiskMemUsage(ResourceUsageState(0.7, 0.64), belowLimit());
TEST_DO(f.assertStrategyConfig(4, 1, 12));
f.assertStrategyConfig("2nd notify", 4, 1, 12);
}

TEST_F("Use conservative settings when above high watermark for memory usage", Fixture)
TEST(MemoryFlushConfigUpdaterTest, Use_conservative_settings_when_above_high_watermark_for_memory_usage)
{
Fixture f;
// The high watermark limit is 0.54 (0.6 * 0.9 (factor)).
f.notifyDiskMemUsage(belowLimit(), ResourceUsageState(0.6, 0.53));
TEST_DO(f.assertStrategyConfig(4, 1, 20));
f.assertStrategyConfig("1st notify", 4, 1, 20);
f.notifyDiskMemUsage(belowLimit(), ResourceUsageState(0.6, 0.55));
TEST_DO(f.assertStrategyConfig(2, 0, 20));
f.assertStrategyConfig("2nd notify", 2, 0, 20);
}

TEST_F("require that we must go below low watermark for disk usage before using normal tls size value again", Fixture)
TEST(MemoryFlushConfigUpdaterTest, require_that_we_must_go_below_low_watermark_for_disk_usage_before_using_normal_tls_size_value_again)
{
Fixture f;
f.notifyDiskMemUsage(ResourceUsageState(0.7, 0.8), belowLimit());
TEST_DO(f.assertStrategyConfig(4, 1, 12));
f.assertStrategyConfig("1st notify", 4, 1, 12);
f.notifyDiskMemUsage(ResourceUsageState(0.7, 0.7), belowLimit());
TEST_DO(f.assertStrategyConfig(4, 1, 12));
f.assertStrategyConfig("2nd notify", 4, 1, 12);
f.notifyDiskMemUsage(ResourceUsageState(0.7, 0.56), belowLimit());
TEST_DO(f.assertStrategyConfig(4, 1, 12));
f.assertStrategyConfig("3rd notify", 4, 1, 12);
f.notifyDiskMemUsage(ResourceUsageState(0.7, 0.55), belowLimit());
TEST_DO(f.assertStrategyConfig(4, 1, 20));
f.assertStrategyConfig("4th notify", 4, 1, 20);
f.notifyDiskMemUsage(ResourceUsageState(0.7, 0.6), belowLimit());
TEST_DO(f.assertStrategyConfig(4, 1, 20));
f.assertStrategyConfig("5th notify", 4, 1, 20);
}

TEST_F("require that we must go below low watermark for memory usage before using normal max memory value again", Fixture)
TEST(MemoryFlushConfigUpdaterTest, require_that_we_must_go_below_low_watermark_for_memory_usage_before_using_normal_max_memory_value_again)
{
Fixture f;
f.notifyDiskMemUsage(belowLimit(), ResourceUsageState(0.7, 0.8));
TEST_DO(f.assertStrategyConfig(2, 0, 20));
f.assertStrategyConfig("1st notify", 2, 0, 20);
f.notifyDiskMemUsage(belowLimit(), ResourceUsageState(0.7, 0.7));
TEST_DO(f.assertStrategyConfig(2, 0, 20));
f.assertStrategyConfig("2nd notify", 2, 0, 20);
f.notifyDiskMemUsage(belowLimit(), ResourceUsageState(0.7, 0.56));
TEST_DO(f.assertStrategyConfig(2, 0, 20));
f.assertStrategyConfig("3rd notify", 2, 0, 20);
f.notifyDiskMemUsage(belowLimit(), ResourceUsageState(0.7, 0.55));
TEST_DO(f.assertStrategyConfig(4, 1, 20));
f.assertStrategyConfig("4th notify", 4, 1, 20);
f.notifyDiskMemUsage(belowLimit(), ResourceUsageState(0.7, 0.6));
TEST_DO(f.assertStrategyConfig(4, 1, 20));
f.assertStrategyConfig("5th notify", 4, 1, 20);
}

TEST_F("require that more disk bloat is allowed while node state is retired or maintenance", Fixture)
TEST(MemoryFlushConfigUpdaterTest, require_that_more_disk_bloat_is_allowed_while_node_state_is_retired_or_maintenance)
{
Fixture f;
constexpr double DEFAULT_DISK_BLOAT = 0.25;
f.notifyDiskMemUsage(ResourceUsageState(0.7, 0.3), belowLimit());
TEST_DO(f.assertStrategyDiskConfig(DEFAULT_DISK_BLOAT, DEFAULT_DISK_BLOAT));
f.assertStrategyDiskConfig("1st notify", DEFAULT_DISK_BLOAT, DEFAULT_DISK_BLOAT);
f.set_node_retired_or_maintenance(true);
TEST_DO(f.assertStrategyDiskConfig((0.8 - ((0.3/0.7)*(1 - DEFAULT_DISK_BLOAT))) / 0.8, 1.0));
f.assertStrategyDiskConfig("2nd notify", (0.8 - ((0.3/0.7)*(1 - DEFAULT_DISK_BLOAT))) / 0.8, 1.0);
f.notifyDiskMemUsage(belowLimit(), belowLimit());
TEST_DO(f.assertStrategyDiskConfig(DEFAULT_DISK_BLOAT, DEFAULT_DISK_BLOAT));
}

f.assertStrategyDiskConfig("erd notify", DEFAULT_DISK_BLOAT, DEFAULT_DISK_BLOAT);
}

0 comments on commit c92512c

Please sign in to comment.