Skip to content

Commit

Permalink
fix and work for fully contained and partial first -> integrate
Browse files Browse the repository at this point in the history
  • Loading branch information
Chang Guo committed May 21, 2024
1 parent 94dbfb7 commit 80cee71
Show file tree
Hide file tree
Showing 3 changed files with 153 additions and 43 deletions.
36 changes: 33 additions & 3 deletions source/adios2/engine/bp5/BP5Reader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,7 @@ void BP5Reader::PerformRemoteGets()
if (getenv("useKVCache"))
{
m_KVCacheCommon.openConnection();
std::cout << "kv cache version control: v1.1" << std::endl;
}
for (auto &Req : GetRequests)
{
Expand All @@ -302,7 +303,6 @@ void BP5Reader::PerformRemoteGets()
{
if (m_KVCacheCommon.exists(cacheKey))
{

#define declare_type_get(T) \
if (varType == helper::GetDataType<T>()) \
{ \
Expand All @@ -315,13 +315,42 @@ void BP5Reader::PerformRemoteGets()
#undef declare_type_get
continue;
} else {
int max_depth = 1;
int max_depth = 8;
std::set<std::string> samePrefixKeys;
m_KVCacheCommon.keyPrefixExistence(keyPrefix, samePrefixKeys);
// print the size of samePrefixKeys
std::cout << "same prefix keys size: " << samePrefixKeys.size() << std::endl;
// print out samePrefixKeys
for (auto &key : samePrefixKeys)
{
std::cout << "same prefix keys: " << key << std::endl;
}
std::vector<QueryBox> regularBoxes;
std::vector<QueryBox> cachedBoxes;
std::vector<std::string> cachedKeys;
m_KVCacheCommon.getMaxInteractBox(samePrefixKeys, targetBox, max_depth, 0, regularBoxes, cachedBoxes, cachedKeys);
if (samePrefixKeys.size() > 0)
{
std::cout << "same prefix keys size > 0" << std::endl;
targetBox.getMaxInteractBox(samePrefixKeys, max_depth, 0, regularBoxes, cachedBoxes, cachedKeys);
} else {
std::cout << "no same prefix keys" << std::endl;
regularBoxes.push_back(targetBox);
}
// print out regularBoxes and cachedBoxes size
std::cout << "regularBoxes size: " << regularBoxes.size() << std::endl;
std::cout << "cachedBoxes size: " << cachedBoxes.size() << std::endl;

// print out regularBoxes and cachedBoxes by toString
for (int i = 0; i < regularBoxes.size(); i++)
{
std::cout << "regular i box: " << regularBoxes[i].toString() << std::endl;
}

for (int i = 0; i < cachedBoxes.size(); i++)
{
std::cout << "cached i box: " << cachedBoxes[i].toString() << std::endl;
}


#define declare_type_full_contain(T) \
if (varType == helper::GetDataType<T>()) \
Expand Down Expand Up @@ -350,6 +379,7 @@ void BP5Reader::PerformRemoteGets()
ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type_full_contain)
#undef declare_type_full_contain

continue;
/*
bool fullContained = false;
for (auto &key : samePrefixKeys)
Expand Down
75 changes: 38 additions & 37 deletions source/adios2/toolkit/cache/KVCacheCommon.inl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ bool KVCacheCommon::exists(std::string key)
std::cout << "Key does not exist" << std::endl;
return false;
}
std::cout << "EXISTS: " << m_redisReply->str << std::endl;
// std::cout << "EXISTS: " << m_redisReply->str << std::endl;
freeReplyObject(m_redisReply);
return true;
}
Expand Down Expand Up @@ -153,44 +153,45 @@ void KVCacheCommon::keyPrefixExistence(const std::string &key_prefix, std::set<s
}
}

void KVCacheCommon::getMaxInteractBox(const std::set<std::string> &samePrefixKeys, const QueryBox &queryBox, const size_t &max_depth, size_t current_depth, std::vector<QueryBox> &regularBoxes, std::vector<QueryBox> &cachedBox, std::vector<std::string> &cachedKeys)
{
if (current_depth > max_depth)
{
return;
}
current_depth++;
QueryBox maxInteractBox;
std::string maxInteractKey;
for (auto &key : samePrefixKeys)
{
QueryBox cachedBox(key);
QueryBox intersection;
if (queryBox.isInteracted(cachedBox, intersection))
{
if (maxInteractBox.size() < intersection.size())
{
maxInteractBox = intersection;
maxInteractKey = key;
}
}
}
// void KVCacheCommon::getMaxInteractBox(const std::set<std::string> &samePrefixKeys, const QueryBox &queryBox, const size_t &max_depth, size_t current_depth, std::vector<QueryBox> &regularBoxes, std::vector<QueryBox> &cachedBox, std::vector<std::string> &cachedKeys)
// {
// if (current_depth > max_depth)
// {
// return;
// }
// current_depth++;
// QueryBox maxInteractBox;
// std::string maxInteractKey;
// for (auto &key : samePrefixKeys)
// {
// std::cout << "key111: " << key << std::endl;
// QueryBox const box(key);
// QueryBox intersection;
// if (queryBox.isInteracted(box, intersection))
// {
// if (maxInteractBox.size() < intersection.size())
// {
// maxInteractBox = intersection;
// maxInteractKey = key;
// }
// }
// }

cachedBox.push_back(maxInteractBox);
cachedKeys.push_back(maxInteractKey);
// cachedBox.push_back(maxInteractBox);
// cachedKeys.push_back(maxInteractKey);

if (current_depth == max_depth)
{
maxInteractBox.interactionCut(queryBox, regularBoxes);
} else {
std::vector<QueryBox> nextBoxes;
maxInteractBox.interactionCut(queryBox, nextBoxes);
for (auto &box : nextBoxes)
{
getMaxInteractBox(samePrefixKeys, box, max_depth, current_depth, regularBoxes, cachedBox, cachedKeys);
}
}
}
// if (current_depth == max_depth)
// {
// maxInteractBox.interactionCut(queryBox, regularBoxes);
// } else {
// std::vector<QueryBox> nextBoxes;
// maxInteractBox.interactionCut(queryBox, nextBoxes);
// for (auto &box : nextBoxes)
// {
// getMaxInteractBox(samePrefixKeys, box, max_depth, current_depth, regularBoxes, cachedBox, cachedKeys);
// }
// }
// }

std::string KVCacheCommon::base64Encode(const std::vector<char> &data)
{
Expand Down
85 changes: 82 additions & 3 deletions source/adios2/toolkit/cache/QueryBox.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

#include <set>
#include "json.hpp"

#include <iostream>
#include "adios2/common/ADIOSTypes.h"

namespace adios2
Expand All @@ -26,9 +26,13 @@ class QueryBox
// sample key: "U3218446744073709551615__count_:_64_64_64___start_:_0_0_0__", count [64, 64, 64], start [0, 0, 0]
// using Dims = std::vector<size_t>;
auto lf_ExtractDimensions = [](const std::string &key, const std::string &delimiter) -> Dims {
size_t pos = key.find(delimiter);
size_t end = key.find("__", pos + delimiter.length());
std::cout << "key: " << key << std::endl;
std::cout << "delimiter: " << delimiter << std::endl;
size_t const pos = key.find(delimiter);
size_t const end = key.find("__", pos + delimiter.length());
std::cout << "pos: " << pos << " end: " << end << std::endl;
std::string dimStr = key.substr(pos + delimiter.length(), end - pos - delimiter.length());
std::cout << "dimStr: " << dimStr << std::endl;
Dims dimensions;
std::istringstream dimStream(dimStr);
std::string token;
Expand Down Expand Up @@ -93,6 +97,8 @@ class QueryBox
return false;
}
}
intersection.start.resize(start.size());
intersection.count.resize(count.size());
for (size_t i = 0; i < start.size(); ++i)
{
intersection.start[i] = std::max(start[i], box.start[i]);
Expand Down Expand Up @@ -189,6 +195,79 @@ class QueryBox
interactionCut(remainingBox1, regularBoxes);
}
}

void getMaxInteractBox(const std::set<std::string> &samePrefixKeys, const size_t &max_depth, size_t current_depth, std::vector<QueryBox> &regularBoxes, std::vector<QueryBox> &cachedBox, std::vector<std::string> &cachedKeys)
{
if (current_depth > max_depth)
{
return;
}
current_depth++;
QueryBox maxInteractBox;
std::string maxInteractKey;
for (auto &key : samePrefixKeys)
{
std::cout << "key11111: " << key << " Current Depth: " << current_depth << std::endl;
QueryBox const box(key);
QueryBox intersection;
if (this->isInteracted(box, intersection))
{
if (maxInteractBox.size() < intersection.size())
{
maxInteractBox = intersection;
maxInteractKey = key;
}
}
}

std::cout << "pushing maxInteractBox: " << maxInteractBox.toString() << "key: " << maxInteractKey << "size: " << maxInteractBox.size() << std::endl;
std::cout << "maxInteractBox.count size: " << maxInteractBox.count.size() << std::endl;

if (maxInteractBox.count.size() == 0)
{
return;
}

cachedBox.push_back(maxInteractBox);
cachedKeys.push_back(maxInteractKey);

if (current_depth == max_depth)
{
maxInteractBox.interactionCut(*this, regularBoxes);
} else {
std::vector<QueryBox> nextBoxes;
maxInteractBox.interactionCut(*this, nextBoxes);
for (auto &box : nextBoxes)
{
box.getMaxInteractBox(samePrefixKeys, max_depth, current_depth, regularBoxes, cachedBox, cachedKeys);
}
}
}

// rewrite toString
std::string toString() const
{
std::string str = "Box start: [";
for (size_t i = 0; i < start.size(); ++i)
{
str += std::to_string(start[i]);
if (i != start.size() - 1)
{
str += ", ";
}
}
str += "], count: [";
for (size_t i = 0; i < count.size(); ++i)
{
str += std::to_string(count[i]);
if (i != count.size() - 1)
{
str += ", ";
}
}
str += "]";
return str;
}

};
};
Expand Down

0 comments on commit 80cee71

Please sign in to comment.