diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp index 7e610277a..f80c5c4a6 100644 --- a/source/adios2/engine/bp5/BP5Reader.cpp +++ b/source/adios2/engine/bp5/BP5Reader.cpp @@ -289,6 +289,7 @@ void BP5Reader::PerformRemoteGets() if (getenv("useKVCache")) { m_KVCacheCommon.openConnection(); + std::cout << "kv cache version control: v1.1" << std::endl; } for (auto &Req : GetRequests) { @@ -302,7 +303,6 @@ void BP5Reader::PerformRemoteGets() { if (m_KVCacheCommon.exists(cacheKey)) { - #define declare_type_get(T) \ if (varType == helper::GetDataType()) \ { \ @@ -315,13 +315,42 @@ void BP5Reader::PerformRemoteGets() #undef declare_type_get continue; } else { - int max_depth = 1; + int max_depth = 8; std::set samePrefixKeys; m_KVCacheCommon.keyPrefixExistence(keyPrefix, samePrefixKeys); + // print the size of samePrefixKeys + std::cout << "same prefix keys size: " << samePrefixKeys.size() << std::endl; + // print out samePrefixKeys + for (auto &key : samePrefixKeys) + { + std::cout << "same prefix keys: " << key << std::endl; + } std::vector regularBoxes; std::vector cachedBoxes; std::vector cachedKeys; - m_KVCacheCommon.getMaxInteractBox(samePrefixKeys, targetBox, max_depth, 0, regularBoxes, cachedBoxes, cachedKeys); + if (samePrefixKeys.size() > 0) + { + std::cout << "same prefix keys size > 0" << std::endl; + targetBox.getMaxInteractBox(samePrefixKeys, max_depth, 0, regularBoxes, cachedBoxes, cachedKeys); + } else { + std::cout << "no same prefix keys" << std::endl; + regularBoxes.push_back(targetBox); + } + // print out regularBoxes and cachedBoxes size + std::cout << "regularBoxes size: " << regularBoxes.size() << std::endl; + std::cout << "cachedBoxes size: " << cachedBoxes.size() << std::endl; + + // print out regularBoxes and cachedBoxes by toString + for (int i = 0; i < regularBoxes.size(); i++) + { + std::cout << "regular i box: " << regularBoxes[i].toString() << std::endl; + } + + for (int i = 0; i < cachedBoxes.size(); i++) + { + std::cout << "cached i box: " << cachedBoxes[i].toString() << std::endl; + } + #define declare_type_full_contain(T) \ if (varType == helper::GetDataType()) \ @@ -350,6 +379,7 @@ void BP5Reader::PerformRemoteGets() ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type_full_contain) #undef declare_type_full_contain + continue; /* bool fullContained = false; for (auto &key : samePrefixKeys) diff --git a/source/adios2/toolkit/cache/KVCacheCommon.inl b/source/adios2/toolkit/cache/KVCacheCommon.inl index 16c067a8d..9c284d222 100644 --- a/source/adios2/toolkit/cache/KVCacheCommon.inl +++ b/source/adios2/toolkit/cache/KVCacheCommon.inl @@ -107,7 +107,7 @@ bool KVCacheCommon::exists(std::string key) std::cout << "Key does not exist" << std::endl; return false; } - std::cout << "EXISTS: " << m_redisReply->str << std::endl; + // std::cout << "EXISTS: " << m_redisReply->str << std::endl; freeReplyObject(m_redisReply); return true; } @@ -153,44 +153,45 @@ void KVCacheCommon::keyPrefixExistence(const std::string &key_prefix, std::set &samePrefixKeys, const QueryBox &queryBox, const size_t &max_depth, size_t current_depth, std::vector ®ularBoxes, std::vector &cachedBox, std::vector &cachedKeys) -{ - if (current_depth > max_depth) - { - return; - } - current_depth++; - QueryBox maxInteractBox; - std::string maxInteractKey; - for (auto &key : samePrefixKeys) - { - QueryBox cachedBox(key); - QueryBox intersection; - if (queryBox.isInteracted(cachedBox, intersection)) - { - if (maxInteractBox.size() < intersection.size()) - { - maxInteractBox = intersection; - maxInteractKey = key; - } - } - } +// void KVCacheCommon::getMaxInteractBox(const std::set &samePrefixKeys, const QueryBox &queryBox, const size_t &max_depth, size_t current_depth, std::vector ®ularBoxes, std::vector &cachedBox, std::vector &cachedKeys) +// { +// if (current_depth > max_depth) +// { +// return; +// } +// current_depth++; +// QueryBox maxInteractBox; +// std::string maxInteractKey; +// for (auto &key : samePrefixKeys) +// { +// std::cout << "key111: " << key << std::endl; +// QueryBox const box(key); +// QueryBox intersection; +// if (queryBox.isInteracted(box, intersection)) +// { +// if (maxInteractBox.size() < intersection.size()) +// { +// maxInteractBox = intersection; +// maxInteractKey = key; +// } +// } +// } - cachedBox.push_back(maxInteractBox); - cachedKeys.push_back(maxInteractKey); +// cachedBox.push_back(maxInteractBox); +// cachedKeys.push_back(maxInteractKey); - if (current_depth == max_depth) - { - maxInteractBox.interactionCut(queryBox, regularBoxes); - } else { - std::vector nextBoxes; - maxInteractBox.interactionCut(queryBox, nextBoxes); - for (auto &box : nextBoxes) - { - getMaxInteractBox(samePrefixKeys, box, max_depth, current_depth, regularBoxes, cachedBox, cachedKeys); - } - } -} +// if (current_depth == max_depth) +// { +// maxInteractBox.interactionCut(queryBox, regularBoxes); +// } else { +// std::vector nextBoxes; +// maxInteractBox.interactionCut(queryBox, nextBoxes); +// for (auto &box : nextBoxes) +// { +// getMaxInteractBox(samePrefixKeys, box, max_depth, current_depth, regularBoxes, cachedBox, cachedKeys); +// } +// } +// } std::string KVCacheCommon::base64Encode(const std::vector &data) { diff --git a/source/adios2/toolkit/cache/QueryBox.h b/source/adios2/toolkit/cache/QueryBox.h index 6e42d1292..b0f50b494 100644 --- a/source/adios2/toolkit/cache/QueryBox.h +++ b/source/adios2/toolkit/cache/QueryBox.h @@ -7,7 +7,7 @@ #include #include "json.hpp" - +#include #include "adios2/common/ADIOSTypes.h" namespace adios2 @@ -26,9 +26,13 @@ class QueryBox // sample key: "U3218446744073709551615__count_:_64_64_64___start_:_0_0_0__", count [64, 64, 64], start [0, 0, 0] // using Dims = std::vector; auto lf_ExtractDimensions = [](const std::string &key, const std::string &delimiter) -> Dims { - size_t pos = key.find(delimiter); - size_t end = key.find("__", pos + delimiter.length()); + std::cout << "key: " << key << std::endl; + std::cout << "delimiter: " << delimiter << std::endl; + size_t const pos = key.find(delimiter); + size_t const end = key.find("__", pos + delimiter.length()); + std::cout << "pos: " << pos << " end: " << end << std::endl; std::string dimStr = key.substr(pos + delimiter.length(), end - pos - delimiter.length()); + std::cout << "dimStr: " << dimStr << std::endl; Dims dimensions; std::istringstream dimStream(dimStr); std::string token; @@ -93,6 +97,8 @@ class QueryBox return false; } } + intersection.start.resize(start.size()); + intersection.count.resize(count.size()); for (size_t i = 0; i < start.size(); ++i) { intersection.start[i] = std::max(start[i], box.start[i]); @@ -189,6 +195,79 @@ class QueryBox interactionCut(remainingBox1, regularBoxes); } } + + void getMaxInteractBox(const std::set &samePrefixKeys, const size_t &max_depth, size_t current_depth, std::vector ®ularBoxes, std::vector &cachedBox, std::vector &cachedKeys) + { + if (current_depth > max_depth) + { + return; + } + current_depth++; + QueryBox maxInteractBox; + std::string maxInteractKey; + for (auto &key : samePrefixKeys) + { + std::cout << "key11111: " << key << " Current Depth: " << current_depth << std::endl; + QueryBox const box(key); + QueryBox intersection; + if (this->isInteracted(box, intersection)) + { + if (maxInteractBox.size() < intersection.size()) + { + maxInteractBox = intersection; + maxInteractKey = key; + } + } + } + + std::cout << "pushing maxInteractBox: " << maxInteractBox.toString() << "key: " << maxInteractKey << "size: " << maxInteractBox.size() << std::endl; + std::cout << "maxInteractBox.count size: " << maxInteractBox.count.size() << std::endl; + + if (maxInteractBox.count.size() == 0) + { + return; + } + + cachedBox.push_back(maxInteractBox); + cachedKeys.push_back(maxInteractKey); + + if (current_depth == max_depth) + { + maxInteractBox.interactionCut(*this, regularBoxes); + } else { + std::vector nextBoxes; + maxInteractBox.interactionCut(*this, nextBoxes); + for (auto &box : nextBoxes) + { + box.getMaxInteractBox(samePrefixKeys, max_depth, current_depth, regularBoxes, cachedBox, cachedKeys); + } + } + } + + // rewrite toString + std::string toString() const + { + std::string str = "Box start: ["; + for (size_t i = 0; i < start.size(); ++i) + { + str += std::to_string(start[i]); + if (i != start.size() - 1) + { + str += ", "; + } + } + str += "], count: ["; + for (size_t i = 0; i < count.size(); ++i) + { + str += std::to_string(count[i]); + if (i != count.size() - 1) + { + str += ", "; + } + } + str += "]"; + return str; + } }; };