From 526dc0670f1487001bc4c9a9299aa5e6843740dd Mon Sep 17 00:00:00 2001 From: fractasy Date: Thu, 7 Sep 2023 14:13:15 +0000 Subject: [PATCH 1/3] Delete class StateRoot and use strings again. Virtual state roots follow the 0:0:0:n pattern. --- src/hashdb64/smt_64.cpp | 5 +- src/hashdb64/state_manager_64.cpp | 101 +++++++++++++----------------- src/hashdb64/state_manager_64.hpp | 27 ++++---- src/hashdb64/state_root.hpp | 72 --------------------- 4 files changed, 56 insertions(+), 149 deletions(-) delete mode 100644 src/hashdb64/state_root.hpp diff --git a/src/hashdb64/smt_64.cpp b/src/hashdb64/smt_64.cpp index fd7efcef1..9bfcd6eb4 100644 --- a/src/hashdb64/smt_64.cpp +++ b/src/hashdb64/smt_64.cpp @@ -28,8 +28,7 @@ zkresult Smt64::set (const string &batchUUID, uint64_t tx, Database64 &db, const // Set the old state root string oldRootString; oldRootString = fea2string(fr, oldRoot); - bool bIsVirtual = stateManager64.isVirtualStateRoot(oldRootString); - stateManager64.setOldStateRoot(batchUUID, tx, oldRootString, bIsVirtual, persistence); + stateManager64.setOldStateRoot(batchUUID, tx, oldRootString, persistence); // Write the key-value pair string hashString = NormalizeToNFormat(fea2string(fr, key), 64); @@ -45,7 +44,7 @@ zkresult Smt64::set (const string &batchUUID, uint64_t tx, Database64 &db, const stateManager64.getVirtualStateRoot(newRoot, newRootString); // Set the new sttae root - stateManager64.setNewStateRoot(batchUUID, tx, newRootString, true, persistence); + stateManager64.setNewStateRoot(batchUUID, tx, newRootString, persistence); result.newRoot[0] = newRoot[0]; result.newRoot[1] = newRoot[1]; diff --git a/src/hashdb64/state_manager_64.cpp b/src/hashdb64/state_manager_64.cpp index 9daf41edd..0d35fe88a 100644 --- a/src/hashdb64/state_manager_64.cpp +++ b/src/hashdb64/state_manager_64.cpp @@ -11,7 +11,7 @@ Goldilocks frSM64; PoseidonGoldilocks poseidonSM64; StateManager64 stateManager64(frSM64, poseidonSM64); -zkresult StateManager64::setStateRoot(const string &batchUUID, uint64_t tx, const string &_stateRoot, const bool bIsOldStateRoot, const bool bVirtual, const Persistence persistence) +zkresult StateManager64::setStateRoot(const string &batchUUID, uint64_t tx, const string &_stateRoot, const bool bIsOldStateRoot, const Persistence persistence) { #ifdef LOG_TIME_STATISTICS_STATE_MANAGER struct timeval t; @@ -48,7 +48,7 @@ zkresult StateManager64::setStateRoot(const string &batchUUID, uint64_t tx, cons return ZKR_STATE_MANAGER; } BatchState64 batchState; - batchState.oldStateRoot.set(stateRoot, bVirtual); + batchState.oldStateRoot = stateRoot; state[batchUUID] = batchState; it = state.find(batchUUID); zkassert(it != state.end()); @@ -56,7 +56,7 @@ zkresult StateManager64::setStateRoot(const string &batchUUID, uint64_t tx, cons BatchState64 &batchState = it->second; // Set the current state root - batchState.currentStateRoot.set(stateRoot, bVirtual); + batchState.currentStateRoot = stateRoot; // Create tx states, if needed if (tx >= batchState.txState.size()) @@ -106,7 +106,7 @@ zkresult StateManager64::setStateRoot(const string &batchUUID, uint64_t tx, cons } // Record the old state root - txState.persistence[persistence].oldStateRoot.set(stateRoot, bVirtual); + txState.persistence[persistence].oldStateRoot = stateRoot; } // If it is not the first sub-state, it must have been called with the previous new state root @@ -131,13 +131,13 @@ zkresult StateManager64::setStateRoot(const string &batchUUID, uint64_t tx, cons // Create TX sub-state TxSubState64 txSubState; - txSubState.oldStateRoot.set(stateRoot, bVirtual); + txSubState.oldStateRoot = stateRoot; txSubState.previousSubState = txState.persistence[persistence].currentSubState; // Copy the key-value data from the previous state, if it exists for (uint64_t i = 0; i < currentSubStateSize; i++) { - if (txState.persistence[persistence].subState[i].newStateRoot.equals(stateRoot, bVirtual)) + if (txState.persistence[persistence].subState[i].newStateRoot == stateRoot) { txSubState.dbWrite = txState.persistence[persistence].subState[i].dbWrite; break; @@ -170,8 +170,8 @@ zkresult StateManager64::setStateRoot(const string &batchUUID, uint64_t tx, cons } // Record the new state root in the tx sub-state, and in the tx state - txState.persistence[persistence].subState[txState.persistence[persistence].currentSubState].newStateRoot.set(stateRoot, bVirtual); - txState.persistence[persistence].newStateRoot.set(stateRoot, bVirtual); + txState.persistence[persistence].subState[txState.persistence[persistence].currentSubState].newStateRoot = stateRoot; + txState.persistence[persistence].newStateRoot = stateRoot; } #ifdef LOG_TIME_STATISTICS_STATE_MANAGER @@ -329,7 +329,6 @@ zkresult StateManager64::semiFlush(const string &batchUUID, const string &_state // Normalize state root format string stateRoot = NormalizeToNFormat(_stateRoot, 64); stateRoot = stringToLower(stateRoot); - bool bVirtual = isVirtualStateRoot(stateRoot); // Check persistence range if (persistence >= PERSISTENCE_SIZE) @@ -373,17 +372,17 @@ zkresult StateManager64::semiFlush(const string &batchUUID, const string &_state TxState64 &txState = batchState.txState[batchState.currentTx]; TxPersistenceState64 &txPersistenceState = txState.persistence[persistence]; - if (txPersistenceState.newStateRoot.equals(stateRoot, bVirtual)) + if (txPersistenceState.newStateRoot == stateRoot) { // This is the expected case } - else if (txPersistenceState.oldStateRoot.equals(stateRoot, bVirtual)) + else if (txPersistenceState.oldStateRoot == stateRoot) { if (config.stateManagerPurge) { // The TX ended up with the same state root as the beginning, so we can delete all data txPersistenceState.subState.clear(); - txPersistenceState.newStateRoot.set(stateRoot, bVirtual); + txPersistenceState.newStateRoot = stateRoot; txPersistenceState.currentSubState = 0; } } @@ -397,7 +396,7 @@ zkresult StateManager64::semiFlush(const string &batchUUID, const string &_state uint64_t subStateSize = txPersistenceState.subState.size(); for (i = 0; i < subStateSize; i++) { - if (!bFound && txPersistenceState.subState[i].oldStateRoot.equals(stateRoot, bVirtual)) + if (!bFound && (txPersistenceState.subState[i].oldStateRoot == stateRoot)) { bFound = true; break; @@ -405,7 +404,7 @@ zkresult StateManager64::semiFlush(const string &batchUUID, const string &_state } if (bFound) { - txPersistenceState.newStateRoot.set(stateRoot, bVirtual); + txPersistenceState.newStateRoot = stateRoot; txPersistenceState.currentSubState = (i == 0) ? 0 : i - 1; for (; i < subStateSize; i++) { @@ -445,7 +444,6 @@ zkresult StateManager64::flush(const string &batchUUID, const string &_newStateR // Format the new state root string newStateRoot = NormalizeToNFormat(_newStateRoot, 64); - bool bVirtual = isVirtualStateRoot(newStateRoot); zkresult zkr; @@ -483,7 +481,7 @@ zkresult StateManager64::flush(const string &batchUUID, const string &_newStateR int64_t tx = -1; for (tx = batchState.txState.size() - 1; tx >= 0; tx--) { - if (batchState.txState[tx].persistence[PERSISTENCE_DATABASE].newStateRoot.equals(newStateRoot, bVirtual)) + if (batchState.txState[tx].persistence[PERSISTENCE_DATABASE].newStateRoot == newStateRoot) { break; } @@ -534,9 +532,9 @@ zkresult StateManager64::flush(const string &batchUUID, const string &_newStateR if (txState.persistence[persistence].subState[txState.persistence[persistence].currentSubState].newStateRoot != txState.persistence[persistence].newStateRoot) { zklog.error("StateManager64::flush() found inconsistent new state roots: batchUUID=" + batchUUID + - " tx=" + to_string(tx) + " txState.newStateRoot=" + txState.persistence[persistence].newStateRoot.toString() + + " tx=" + to_string(tx) + " txState.newStateRoot=" + txState.persistence[persistence].newStateRoot + " currentSubState=" + to_string(txState.persistence[persistence].currentSubState) + - " substate.newStateRoot=" + txState.persistence[persistence].subState[txState.persistence[persistence].currentSubState].newStateRoot.toString()); + " substate.newStateRoot=" + txState.persistence[persistence].subState[txState.persistence[persistence].currentSubState].newStateRoot); #ifdef LOG_TIME_STATISTICS_STATE_MANAGER batchState.timeMetricStorage.add("flush UUID inconsistent new state roots", TimeDiff(t)); @@ -564,9 +562,9 @@ zkresult StateManager64::flush(const string &batchUUID, const string &_newStateR if (txState.persistence[persistence].subState[currentSubState].oldStateRoot != txState.persistence[persistence].oldStateRoot) { zklog.error("StateManager64::flush() found inconsistent old state roots: batchUUID=" + batchUUID + - " tx=" + to_string(tx) + " txState.oldStateRoot=" + txState.persistence[persistence].oldStateRoot.toString() + + " tx=" + to_string(tx) + " txState.oldStateRoot=" + txState.persistence[persistence].oldStateRoot + " currentSubState=" + to_string(txState.persistence[persistence].currentSubState) + - " substate.oldStateRoot=" + txState.persistence[persistence].subState[currentSubState].oldStateRoot.toString()); + " substate.oldStateRoot=" + txState.persistence[persistence].subState[currentSubState].oldStateRoot); #ifdef LOG_TIME_STATISTICS_STATE_MANAGER batchState.timeMetricStorage.add("flush UUID inconsistent old state roots", TimeDiff(t)); @@ -605,9 +603,9 @@ zkresult StateManager64::flush(const string &batchUUID, const string &_newStateR { zklog.error("StateManager64::flush() could not find previous tx sub-state: batchUUID=" + batchUUID + " tx=" + to_string(tx) + - " txState.oldStateRoot=" + txState.persistence[persistence].oldStateRoot.toString() + + " txState.oldStateRoot=" + txState.persistence[persistence].oldStateRoot + " currentSubState=" + to_string(txState.persistence[persistence].currentSubState) + - " substate.oldStateRoot=" + txState.persistence[persistence].subState[currentSubState].oldStateRoot.toString()); + " substate.oldStateRoot=" + txState.persistence[persistence].subState[currentSubState].oldStateRoot); #ifdef LOG_TIME_STATISTICS_STATE_MANAGER batchState.timeMetricStorage.add("flush UUID cannot find previous tx sub-state", TimeDiff(t)); batchState.timeMetricStorage.print("State Manager calls"); @@ -722,7 +720,7 @@ zkresult StateManager64::flush(const string &batchUUID, const string &_newStateR // Get old state root for this tx Goldilocks::Element oldRoot[4]; - string2fea(fr, txState.persistence[persistence].oldStateRoot.realStateRoot, oldRoot); + string2fea(fr, txState.persistence[persistence].oldStateRoot, oldRoot); // Get the key-values for this tx vector keyValues; @@ -743,7 +741,7 @@ zkresult StateManager64::flush(const string &batchUUID, const string &_newStateR { zklog.error("StateManager64::flush() failed calling WriteTree zkr=" + zkresult2string(zkr) + " tx=" + to_string(tx) + - " txState.oldStateRoot=" + txState.persistence[persistence].oldStateRoot.toString()); + " txState.oldStateRoot=" + txState.persistence[persistence].oldStateRoot); #ifdef LOG_TIME_STATISTICS_STATE_MANAGER batchState.timeMetricStorage.add("WriteTree failed", TimeDiff(t)); batchState.timeMetricStorage.print("State Manager calls"); @@ -754,18 +752,17 @@ zkresult StateManager64::flush(const string &batchUUID, const string &_newStateR // Save the real state root of this tx string newRootString = fea2string(fr, newRoot); - txState.persistence[persistence].newStateRoot.realStateRoot = newRootString; + txState.persistence[persistence].newStateRoot = newRootString; // Save the old state root of the next tx, if any if (tx < batchState.txState.size() - 1) { - zkassertpermanent(batchState.txState[tx+1].persistence[persistence].oldStateRoot.realStateRoot.size() == 0); - batchState.txState[tx+1].persistence[persistence].oldStateRoot.realStateRoot = newRootString; + batchState.txState[tx+1].persistence[persistence].oldStateRoot = newRootString; } // If this is the last tx, then save the new state root of the batch else { - batchState.newStateRoot.realStateRoot = newRootString; + batchState.newStateRoot = newRootString; } // Create a new version, i.e. read latest version and increment it @@ -775,7 +772,7 @@ zkresult StateManager64::flush(const string &batchUUID, const string &_newStateR { zklog.error("StateManager64::flush() failed calling db.readLatestVersion zkr=" + zkresult2string(zkr) + " tx=" + to_string(tx) + - " txState.oldStateRoot=" + txState.persistence[persistence].oldStateRoot.toString()); + " txState.oldStateRoot=" + txState.persistence[persistence].oldStateRoot); #ifdef LOG_TIME_STATISTICS_STATE_MANAGER batchState.timeMetricStorage.add("db.createLatestVersion failed", TimeDiff(t)); batchState.timeMetricStorage.print("State Manager calls"); @@ -791,7 +788,7 @@ zkresult StateManager64::flush(const string &batchUUID, const string &_newStateR { zklog.error("StateManager64::flush() failed calling db.writeKV zkr=" + zkresult2string(zkr) + " tx=" + to_string(tx) + - " txState.oldStateRoot=" + txState.persistence[persistence].oldStateRoot.toString()); + " txState.oldStateRoot=" + txState.persistence[persistence].oldStateRoot); #ifdef LOG_TIME_STATISTICS_STATE_MANAGER batchState.timeMetricStorage.add("db.writeKV failed", TimeDiff(t)); batchState.timeMetricStorage.print("State Manager calls"); @@ -806,7 +803,7 @@ zkresult StateManager64::flush(const string &batchUUID, const string &_newStateR { zklog.error("StateManager64::flush() failed calling db.writeVersion zkr=" + zkresult2string(zkr) + " tx=" + to_string(tx) + - " txState.oldStateRoot=" + txState.persistence[persistence].oldStateRoot.toString()); + " txState.oldStateRoot=" + txState.persistence[persistence].oldStateRoot); #ifdef LOG_TIME_STATISTICS_STATE_MANAGER batchState.timeMetricStorage.add("db.writeVersion failed", TimeDiff(t)); batchState.timeMetricStorage.print("State Manager calls"); @@ -821,7 +818,7 @@ zkresult StateManager64::flush(const string &batchUUID, const string &_newStateR { zklog.error("StateManager64::flush() failed calling db.writeLatestVersion zkr=" + zkresult2string(zkr) + " tx=" + to_string(tx) + - " txState.oldStateRoot=" + txState.persistence[persistence].oldStateRoot.toString()); + " txState.oldStateRoot=" + txState.persistence[persistence].oldStateRoot); #ifdef LOG_TIME_STATISTICS_STATE_MANAGER batchState.timeMetricStorage.add("db.writeLatestVersion failed", TimeDiff(t)); batchState.timeMetricStorage.print("State Manager calls"); @@ -868,8 +865,8 @@ void StateManager64::print(bool bDbContent) zklog.info(" batchState=" + to_string(batchStateCounter)); batchStateCounter++; zklog.info(" BatchUUID=" + stateIt->first); - zklog.info(" oldStateRoot=" + batchState.oldStateRoot.toString()); - zklog.info(" currentStateRoot=" + batchState.currentStateRoot.toString()); + zklog.info(" oldStateRoot=" + batchState.oldStateRoot); + zklog.info(" currentStateRoot=" + batchState.currentStateRoot); zklog.info(" currentTx=" + to_string(batchState.currentTx)); for (uint64_t tx = 0; tx < batchState.txState.size(); tx++) @@ -881,16 +878,16 @@ void StateManager64::print(bool bDbContent) for (uint64_t persistence = 0; persistence < PERSISTENCE_SIZE; persistence++) { zklog.info(" persistence=" + to_string(persistence) + "=" + persistence2string((Persistence)persistence)); - zklog.info(" oldStateRoot=" + txState.persistence[persistence].oldStateRoot.toString()); - zklog.info(" newStateRoot=" + txState.persistence[persistence].newStateRoot.toString()); + zklog.info(" oldStateRoot=" + txState.persistence[persistence].oldStateRoot); + zklog.info(" newStateRoot=" + txState.persistence[persistence].newStateRoot); zklog.info(" currentSubState=" + to_string(txState.persistence[persistence].currentSubState)); zklog.info(" txSubState.size=" + to_string(txState.persistence[persistence].subState.size())); for (uint64_t i = 0; i < txState.persistence[persistence].subState.size(); i++) { const TxSubState64 &txSubState = txState.persistence[persistence].subState[i]; zklog.info(" txSubState=" + to_string(i)); - zklog.info(" oldStateRoot=" + txSubState.oldStateRoot.toString()); - zklog.info(" newStateRoot=" + txSubState.newStateRoot.toString()); + zklog.info(" oldStateRoot=" + txSubState.oldStateRoot); + zklog.info(" newStateRoot=" + txSubState.newStateRoot); zklog.info(" valid=" + to_string(txSubState.bValid)); zklog.info(" previousSubState=" + to_string(txSubState.previousSubState)); zklog.info(" dbWrite.size=" + to_string(txSubState.dbWrite.size())); @@ -921,30 +918,16 @@ void StateManager64::print(bool bDbContent) void StateManager64::getVirtualStateRoot(Goldilocks::Element (&newStateRoot)[4], string &newStateRootString) { lastVirtualStateRoot++; - Goldilocks::Element fea[12]; - fea[0] = fr.fromU64(lastVirtualStateRoot); - for (uint64_t i = 1; i < 12; i++) - { - fea[i] = fr.zero(); - } - poseidon.hash(newStateRoot, fea); + newStateRoot[0] = fr.fromU64(lastVirtualStateRoot); + newStateRoot[1] = fr.zero(); + newStateRoot[2] = fr.zero(); + newStateRoot[3] = fr.zero(); newStateRootString = NormalizeToNFormat(fea2string(fr, newStateRoot), 64); - lastVirtualStateRootString = newStateRootString; - virtualStateRoots[newStateRootString] = lastVirtualStateRoot; } bool StateManager64::isVirtualStateRoot(const string &stateRoot) { - if (lastVirtualStateRootString == stateRoot) - { - return true; - } - else if (virtualStateRoots.find(stateRoot) != virtualStateRoots.end()) - { - return true; - } - else - { - return false; - } + Goldilocks::Element root[4]; + string2fea(fr, stateRoot, root); + return fr.isZero(root[1]) && fr.isZero(root[2]) && fr.isZero(root[3]); } \ No newline at end of file diff --git a/src/hashdb64/state_manager_64.hpp b/src/hashdb64/state_manager_64.hpp index 4df4387f5..97d0925fb 100644 --- a/src/hashdb64/state_manager_64.hpp +++ b/src/hashdb64/state_manager_64.hpp @@ -11,15 +11,14 @@ #include "database_64.hpp" #include "utils/time_metric.hpp" #include "poseidon_goldilocks.hpp" -#include "state_root.hpp" using namespace std; class TxSubState64 { public: - StateRoot oldStateRoot; - StateRoot newStateRoot; + string oldStateRoot; + string newStateRoot; uint64_t previousSubState; bool bValid; unordered_map dbWrite; @@ -32,8 +31,8 @@ class TxSubState64 class TxPersistenceState64 { public: - StateRoot oldStateRoot; - StateRoot newStateRoot; + string oldStateRoot; + string newStateRoot; uint64_t currentSubState; vector subState; TxPersistenceState64() : currentSubState(0) @@ -51,9 +50,9 @@ class TxState64 class BatchState64 { public: - StateRoot oldStateRoot; - StateRoot currentStateRoot; - StateRoot newStateRoot; + string oldStateRoot; + string currentStateRoot; + string newStateRoot; uint64_t currentTx; vector txState; unordered_map dbWrite; @@ -76,8 +75,6 @@ class StateManager64 Config config; pthread_mutex_t mutex; // Mutex to protect the multi write queues uint64_t lastVirtualStateRoot; - unordered_map virtualStateRoots; - string lastVirtualStateRootString; public: StateManager64(Goldilocks &fr, PoseidonGoldilocks &poseidon) : fr(fr), poseidon(poseidon), lastVirtualStateRoot(0) { @@ -85,19 +82,19 @@ class StateManager64 pthread_mutex_init(&mutex, NULL); }; private: - zkresult setStateRoot (const string &batchUUID, uint64_t tx, const string &stateRoot, const bool bIsOldStateRoot, const bool bIsVirtual, const Persistence persistence); + zkresult setStateRoot (const string &batchUUID, uint64_t tx, const string &stateRoot, const bool bIsOldStateRoot, const Persistence persistence); public: void init (const Config &_config) { config = _config; } - zkresult setOldStateRoot (const string &batchUUID, uint64_t tx, const string &stateRoot, const bool bIsVirtual, const Persistence persistence) + zkresult setOldStateRoot (const string &batchUUID, uint64_t tx, const string &stateRoot, const Persistence persistence) { - return setStateRoot(batchUUID, tx, stateRoot, true, bIsVirtual, persistence); + return setStateRoot(batchUUID, tx, stateRoot, true, persistence); } - zkresult setNewStateRoot (const string &batchUUID, uint64_t tx, const string &stateRoot, const bool bIsVirtual, const Persistence persistence) + zkresult setNewStateRoot (const string &batchUUID, uint64_t tx, const string &stateRoot, const Persistence persistence) { - return setStateRoot(batchUUID, tx, stateRoot, false, bIsVirtual, persistence); + return setStateRoot(batchUUID, tx, stateRoot, false, persistence); } zkresult write (const string &batchUUID, uint64_t tx, const string &_key, const mpz_class &value, const Persistence persistence); zkresult read (const string &batchUUID, const string &_key, mpz_class &value, DatabaseMap *dbReadLog); diff --git a/src/hashdb64/state_root.hpp b/src/hashdb64/state_root.hpp deleted file mode 100644 index c7b833c9d..000000000 --- a/src/hashdb64/state_root.hpp +++ /dev/null @@ -1,72 +0,0 @@ -#ifndef STATE_ROOT_HPP -#define STATE_ROOT_HPP - -#include - -using namespace std; - -class StateRoot -{ -public: - string realStateRoot; - string virtualStateRoot; - - void set (const string stateRoot, const bool bVirtual) - { - if (bVirtual) - { - virtualStateRoot = stateRoot; - } - else - { - realStateRoot = stateRoot; - } - } - - bool empty (void) - { - return (realStateRoot.size() == 0) && (virtualStateRoot.size() == 0); - } - - bool equals (const string stateRoot, const bool bVirtual) - { - if (bVirtual) - { - return virtualStateRoot == stateRoot; - } - else - { - return realStateRoot == stateRoot; - } - } - - bool operator== (const StateRoot &other) - { - return (realStateRoot == other.realStateRoot) && (virtualStateRoot == other.virtualStateRoot); - } - - bool operator!= (const StateRoot &other) - { - return (realStateRoot != other.realStateRoot) || (virtualStateRoot != other.virtualStateRoot); - } - - const string toString (void) const - { - string result; - if (virtualStateRoot.size() > 0) - { - result = "(virtual)" + virtualStateRoot; - } - if (realStateRoot.size() > 0) - { - if (result.size() > 0) - { - result += "/"; - } - result += "(real)" + realStateRoot; - } - return result; - } -}; - -#endif \ No newline at end of file From 6796300ea51d30e73bd637c5fbac3c70eb8b887c Mon Sep 17 00:00:00 2001 From: fractasy Date: Thu, 7 Sep 2023 15:29:28 +0000 Subject: [PATCH 2/3] Refactor KeyValue to use Goldilocks::Element key[4] --- src/hashdb64/database_64.cpp | 22 ++++++++--------- src/hashdb64/key.hpp | 12 ---------- src/hashdb64/key_value.hpp | 3 +-- src/hashdb64/smt_64.cpp | 1 - src/hashdb64/smt_64.hpp | 1 - src/hashdb64/state_manager_64.cpp | 2 +- src/hashdb64/tree_64.cpp | 40 +++++++++++++++---------------- src/hashdb64/tree_64.hpp | 1 - src/utils/key_utils.cpp | 1 - test/hashdb/smt_64_test.cpp | 22 ++++++++--------- 10 files changed, 44 insertions(+), 61 deletions(-) delete mode 100644 src/hashdb64/key.hpp diff --git a/src/hashdb64/database_64.cpp b/src/hashdb64/database_64.cpp index abccd20f9..fe8c40466 100644 --- a/src/hashdb64/database_64.cpp +++ b/src/hashdb64/database_64.cpp @@ -467,10 +467,10 @@ zkresult Database64::readKV(const Goldilocks::Element (&root)[4], vector #include "state_manager_64.hpp" -#include "key.hpp" #include "key_utils.hpp" #include "tree_chunk.hpp" diff --git a/src/hashdb64/smt_64.hpp b/src/hashdb64/smt_64.hpp index f9e2c9cd0..761abb619 100644 --- a/src/hashdb64/smt_64.hpp +++ b/src/hashdb64/smt_64.hpp @@ -15,7 +15,6 @@ #include "smt_set_result.hpp" #include "smt_get_result.hpp" #include "tree_chunk.hpp" -#include "key.hpp" #include "key_value.hpp" using namespace std; diff --git a/src/hashdb64/state_manager_64.cpp b/src/hashdb64/state_manager_64.cpp index 0d35fe88a..4443cc79c 100644 --- a/src/hashdb64/state_manager_64.cpp +++ b/src/hashdb64/state_manager_64.cpp @@ -729,7 +729,7 @@ zkresult StateManager64::flush(const string &batchUUID, const string &_newStateR for (it = dbWrite.begin(); it != dbWrite.end(); it++) { KeyValue keyValue; - string2fea(fr, it->first, keyValue.key.fe); + string2fea(fr, it->first, keyValue.key); keyValue.value = it->second; keyValues.emplace_back(keyValue); } diff --git a/src/hashdb64/tree_64.cpp b/src/hashdb64/tree_64.cpp index 2bb19407b..b94ca51f5 100644 --- a/src/hashdb64/tree_64.cpp +++ b/src/hashdb64/tree_64.cpp @@ -76,7 +76,7 @@ zkresult Tree64::WriteTree (Database64 &db, const Goldilocks::Element (&oldRoot) for (uint64_t j=0; jlist.size(); j++) { bool keyBits[256]; - splitKey(fr, keyValues[chunks[i]->list[j]].key.fe, keyBits); + splitKey(fr, keyValues[chunks[i]->list[j]].key, keyBits); uint64_t k = getKeyChildren64Position(keyBits, level); switch (chunks[i]->getChild(k).type) { @@ -84,17 +84,17 @@ zkresult Tree64::WriteTree (Database64 &db, const Goldilocks::Element (&oldRoot) { if (keyValues[chunks[i]->list[j]].value != 0) { - chunks[i]->setLeafChild(k, keyValues[chunks[i]->list[j]].key.fe, keyValues[chunks[i]->list[j]].value); + chunks[i]->setLeafChild(k, keyValues[chunks[i]->list[j]].key, keyValues[chunks[i]->list[j]].value); } break; } case LEAF: { // If the key is the same, then check the value - if (fr.equal(chunks[i]->getChild(k).leaf.key[0], keyValues[chunks[i]->list[j]].key.fe[0]) && - fr.equal(chunks[i]->getChild(k).leaf.key[1], keyValues[chunks[i]->list[j]].key.fe[1]) && - fr.equal(chunks[i]->getChild(k).leaf.key[2], keyValues[chunks[i]->list[j]].key.fe[2]) && - fr.equal(chunks[i]->getChild(k).leaf.key[3], keyValues[chunks[i]->list[j]].key.fe[3])) + if (fr.equal(chunks[i]->getChild(k).leaf.key[0], keyValues[chunks[i]->list[j]].key[0]) && + fr.equal(chunks[i]->getChild(k).leaf.key[1], keyValues[chunks[i]->list[j]].key[1]) && + fr.equal(chunks[i]->getChild(k).leaf.key[2], keyValues[chunks[i]->list[j]].key[2]) && + fr.equal(chunks[i]->getChild(k).leaf.key[3], keyValues[chunks[i]->list[j]].key[3])) { // If value is different, copy it if (chunks[i]->getChild(k).leaf.value != keyValues[chunks[i]->list[j]].value) @@ -105,7 +105,7 @@ zkresult Tree64::WriteTree (Database64 &db, const Goldilocks::Element (&oldRoot) } else { - chunks[i]->setLeafChild(k, keyValues[chunks[i]->list[j]].key.fe, keyValues[chunks[i]->list[j]].value); + chunks[i]->setLeafChild(k, keyValues[chunks[i]->list[j]].key, keyValues[chunks[i]->list[j]].value); } } } @@ -124,10 +124,10 @@ zkresult Tree64::WriteTree (Database64 &db, const Goldilocks::Element (&oldRoot) // We create a KeyValue from the original leaf node KeyValue kv; - kv.key.fe[0] = chunks[i]->getChild(k).leaf.key[0]; - kv.key.fe[1] = chunks[i]->getChild(k).leaf.key[1]; - kv.key.fe[2] = chunks[i]->getChild(k).leaf.key[2]; - kv.key.fe[3] = chunks[i]->getChild(k).leaf.key[3]; + kv.key[0] = chunks[i]->getChild(k).leaf.key[0]; + kv.key[1] = chunks[i]->getChild(k).leaf.key[1]; + kv.key[2] = chunks[i]->getChild(k).leaf.key[2]; + kv.key[3] = chunks[i]->getChild(k).leaf.key[3]; kv.value = chunks[i]->getChild(k).leaf.value; // We add to the list the original leaf node @@ -391,7 +391,7 @@ zkresult Tree64::ReadTree (Database64 &db, const Goldilocks::Element (&root)[4], for (uint64_t j=0; jlist.size(); j++) { bool keyBits[256]; - splitKey(fr, keyValues[chunks[i]->list[j]].key.fe, keyBits); + splitKey(fr, keyValues[chunks[i]->list[j]].key, keyBits); uint64_t k = getKeyChildren64Position(keyBits, level); switch (chunks[i]->getChild(k).type) { @@ -399,10 +399,10 @@ zkresult Tree64::ReadTree (Database64 &db, const Goldilocks::Element (&root)[4], { for (uint64_t kv=0; kvlist[j]].key.fe[0]) && - fr.equal(keyValues[kv].key.fe[1], keyValues[chunks[i]->list[j]].key.fe[1]) && - fr.equal(keyValues[kv].key.fe[2], keyValues[chunks[i]->list[j]].key.fe[2]) && - fr.equal(keyValues[kv].key.fe[3], keyValues[chunks[i]->list[j]].key.fe[3])) + if (fr.equal(keyValues[kv].key[0], keyValues[chunks[i]->list[j]].key[0]) && + fr.equal(keyValues[kv].key[1], keyValues[chunks[i]->list[j]].key[1]) && + fr.equal(keyValues[kv].key[2], keyValues[chunks[i]->list[j]].key[2]) && + fr.equal(keyValues[kv].key[3], keyValues[chunks[i]->list[j]].key[3])) { keyValues[kv].value = 0; } @@ -412,10 +412,10 @@ zkresult Tree64::ReadTree (Database64 &db, const Goldilocks::Element (&root)[4], case LEAF: { // If the key is the same, then check the value - if (fr.equal(chunks[i]->getChild(k).leaf.key[0], keyValues[chunks[i]->list[j]].key.fe[0]) && - fr.equal(chunks[i]->getChild(k).leaf.key[1], keyValues[chunks[i]->list[j]].key.fe[1]) && - fr.equal(chunks[i]->getChild(k).leaf.key[2], keyValues[chunks[i]->list[j]].key.fe[2]) && - fr.equal(chunks[i]->getChild(k).leaf.key[3], keyValues[chunks[i]->list[j]].key.fe[3])) + if (fr.equal(chunks[i]->getChild(k).leaf.key[0], keyValues[chunks[i]->list[j]].key[0]) && + fr.equal(chunks[i]->getChild(k).leaf.key[1], keyValues[chunks[i]->list[j]].key[1]) && + fr.equal(chunks[i]->getChild(k).leaf.key[2], keyValues[chunks[i]->list[j]].key[2]) && + fr.equal(chunks[i]->getChild(k).leaf.key[3], keyValues[chunks[i]->list[j]].key[3])) { keyValues[chunks[i]->list[j]].value = chunks[i]->getChild(k).leaf.value; } diff --git a/src/hashdb64/tree_64.hpp b/src/hashdb64/tree_64.hpp index b8c2a76e7..d5b74cbce 100644 --- a/src/hashdb64/tree_64.hpp +++ b/src/hashdb64/tree_64.hpp @@ -15,7 +15,6 @@ #include "smt_set_result.hpp" #include "smt_get_result.hpp" #include "tree_chunk.hpp" -#include "key.hpp" #include "key_value.hpp" using namespace std; diff --git a/src/utils/key_utils.cpp b/src/utils/key_utils.cpp index 8f1d9ca2d..3413562db 100644 --- a/src/utils/key_utils.cpp +++ b/src/utils/key_utils.cpp @@ -1,5 +1,4 @@ #include -#include "key.hpp" #include "scalar.hpp" #include "zklog.hpp" #include "exit_process.hpp" diff --git a/test/hashdb/smt_64_test.cpp b/test/hashdb/smt_64_test.cpp index 0f702da7c..3577e6172 100644 --- a/test/hashdb/smt_64_test.cpp +++ b/test/hashdb/smt_64_test.cpp @@ -49,14 +49,14 @@ uint64_t Smt64Test (const Config &config) if (bRandomKeys) { keyValue[0] = fr.fromU64(i); - poseidon.hash(kv.key.fe, keyValue); + poseidon.hash(kv.key, keyValue); } else { - kv.key.fe[0] = fr.fromU64(i); - kv.key.fe[1] = fr.zero(); - kv.key.fe[2] = fr.zero(); - kv.key.fe[3] = fr.zero(); + kv.key[0] = fr.fromU64(i); + kv.key[1] = fr.zero(); + kv.key[2] = fr.zero(); + kv.key[3] = fr.zero(); } kv.value = i; keyValues[i/SMT64_TEST_KEYS_PER_WRITE].emplace_back(kv); @@ -156,12 +156,12 @@ uint64_t Smt64Test (const Config &config) for (uint64_t j=0; jset("", 0, root, keyValues[i][j].key, keyValues[i][j].value, PERSISTENCE_DATABASE, root, NULL, NULL); if (zkr != ZKR_SUCCESS) { zklog.error("Smt64Test() failed calling pHashDB->set() result=" + zkresult2string(zkr)); From f4499a0a24a6ca05afa27cb94ca2941c9f84afb6 Mon Sep 17 00:00:00 2001 From: fractasy Date: Thu, 7 Sep 2023 16:05:38 +0000 Subject: [PATCH 3/3] Add flush and reset uuid in hashDBTestClientThread --- test/service/hashdb/hashdb_test_client.cpp | 90 +++++++++++++++++++--- 1 file changed, 78 insertions(+), 12 deletions(-) diff --git a/test/service/hashdb/hashdb_test_client.cpp b/test/service/hashdb/hashdb_test_client.cpp index 8ea6d4a39..df9d10deb 100644 --- a/test/service/hashdb/hashdb_test_client.cpp +++ b/test/service/hashdb/hashdb_test_client.cpp @@ -1,5 +1,6 @@ #include "hashdb_test.hpp" #include +#include #include "hashdb_interface.hpp" #include "scalar.hpp" #include "zkassert.hpp" @@ -69,11 +70,14 @@ void* hashDBTestClientThread (const Config& config) zkassertpermanent(config.hashDB64 || (fr.isZero(root[0]) && fr.isZero(root[1]) && fr.isZero(root[2]) && fr.isZero(root[3]))); zkr = client->flush(uuid, fea2string(fr, root), persistence, flushId, storedFlushId); - zkassertpermanent(config.hashDB64 || zkr==ZKR_SUCCESS); + zkassertpermanent(zkr==ZKR_SUCCESS); cout << "HashDB client test 1 done" << endl; } + uuid = getUUID(); + tx = 0; + // It should update an element 1 { SmtSetResult setResult; @@ -128,11 +132,14 @@ void* hashDBTestClientThread (const Config& config) zkassertpermanent(value==2); zkr = client->flush(uuid, fea2string(fr, root), persistence, flushId, storedFlushId); - zkassertpermanent(config.hashDB64 || zkr==ZKR_SUCCESS); + zkassertpermanent(zkr==ZKR_SUCCESS); cout << "HashDB client test 2 done" << endl; } + uuid = getUUID(); + tx = 0; + // It should add a shared element 2 { SmtSetResult setResult; @@ -172,11 +179,17 @@ void* hashDBTestClientThread (const Config& config) zkr = client->set(uuid, tx, root, key2, value, persistence, newRoot, &setResult, NULL); cout << "SET zkr=" << zkresult2string(zkr) << " root=" << fea2string(fr, root) << " key2=" << fea2string(fr, key2) << " value=" << value.get_str() << " newRoot=" << fea2string(fr, newRoot) << endl; for (uint64_t i=0; i<4; i++) root[i] = setResult.newRoot[i]; - zkassertpermanent(fr.isZero(root[0]) && fr.isZero(root[1]) && fr.isZero(root[2]) && fr.isZero(root[3])); + zkassertpermanent(config.hashDB64 || (fr.isZero(root[0]) && fr.isZero(root[1]) && fr.isZero(root[2]) && fr.isZero(root[3]))); + + zkr = client->flush(uuid, fea2string(fr, root), persistence, flushId, storedFlushId); + zkassertpermanent(zkr==ZKR_SUCCESS); cout << "HashDB client test 3 done" << endl; } + uuid = getUUID(); + tx = 0; + // It should add a shared element 3 { SmtSetResult setResult; @@ -216,11 +229,17 @@ void* hashDBTestClientThread (const Config& config) zkr = client->set(uuid, tx, root, key2, value, persistence, newRoot, &setResult, NULL); cout << "SET zkr=" << zkresult2string(zkr) << " root=" << fea2string(fr, root) << " key2=" << fea2string(fr, key2) << " value=" << value.get_str() << " newRoot=" << fea2string(fr, newRoot) << endl; for (uint64_t i=0; i<4; i++) root[i] = setResult.newRoot[i]; - zkassertpermanent(fr.isZero(root[0]) && fr.isZero(root[1]) && fr.isZero(root[2]) && fr.isZero(root[3])); + zkassertpermanent(config.hashDB64 || (fr.isZero(root[0]) && fr.isZero(root[1]) && fr.isZero(root[2]) && fr.isZero(root[3]))); + + zkr = client->flush(uuid, fea2string(fr, root), persistence, flushId, storedFlushId); + zkassertpermanent(zkr==ZKR_SUCCESS); cout << "HashDB client test 4 done" << endl; } + uuid = getUUID(); + tx = 0; + // It should add a shared element { @@ -277,11 +296,17 @@ void* hashDBTestClientThread (const Config& config) cout << "SET zkr=" << zkresult2string(zkr) << " root=" << fea2string(fr, root) << " key3=" << fea2string(fr, key3) << " value=" << value.get_str() << " newRoot=" << fea2string(fr, newRoot) << endl; for (uint64_t i=0; i<4; i++) root[i] = setResult.newRoot[i]; - zkassertpermanent(fr.isZero(root[0]) && fr.isZero(root[1]) && fr.isZero(root[2]) && fr.isZero(root[3])); + zkassertpermanent(config.hashDB64 || (fr.isZero(root[0]) && fr.isZero(root[1]) && fr.isZero(root[2]) && fr.isZero(root[3]))); + + zkr = client->flush(uuid, fea2string(fr, root), persistence, flushId, storedFlushId); + zkassertpermanent(zkr==ZKR_SUCCESS); cout << "HashDB client test 5 done" << endl; } + uuid = getUUID(); + tx = 0; + // Add-Remove 128 elements { SmtSetResult setResult; @@ -316,11 +341,17 @@ void* hashDBTestClientThread (const Config& config) for (uint64_t i=0; i<4; i++) root[i] = setResult.newRoot[i]; } - zkassertpermanent(fr.isZero(root[0]) && fr.isZero(root[1]) && fr.isZero(root[2]) && fr.isZero(root[3])); + zkassertpermanent(config.hashDB64 || (fr.isZero(root[0]) && fr.isZero(root[1]) && fr.isZero(root[2]) && fr.isZero(root[3]))); + + zkr = client->flush(uuid, fea2string(fr, root), persistence, flushId, storedFlushId); + zkassertpermanent(zkr==ZKR_SUCCESS); cout << "HashDB client test 6 done" << endl; } + uuid = getUUID(); + tx = 0; + // Should read random { SmtSetResult setResult; @@ -351,10 +382,16 @@ void* hashDBTestClientThread (const Config& config) cout << "GET zkr=" << zkresult2string(zkr) << " root=" << fea2string(fr, root) << " key=" << fea2string(fr, key) << " value=" << value.get_str() << endl; zkassertpermanent(getResult.value==(i+1000)); } + + zkr = client->flush(uuid, fea2string(fr, root), persistence, flushId, storedFlushId); + zkassertpermanent(zkr==ZKR_SUCCESS); cout << "HashDB client test 7 done" << endl; } + uuid = getUUID(); + tx = 0; + // It should add elements with similar keys { SmtSetResult setResult; @@ -395,11 +432,17 @@ void* hashDBTestClientThread (const Config& config) for (uint64_t i=0; i<4; i++) root[i] = setResult.newRoot[i]; zkassertpermanent(!fr.isZero(root[0]) || !fr.isZero(root[1]) || !fr.isZero(root[2]) || !fr.isZero(root[3])); - zkassertpermanent(fr.equal(expectedRoot[0], root[0]) && fr.equal(expectedRoot[1], root[1]) && fr.equal(expectedRoot[2], root[2]) && fr.equal(expectedRoot[3], root[3])); + zkassertpermanent(config.hashDB64 || (fr.equal(expectedRoot[0], root[0]) && fr.equal(expectedRoot[1], root[1]) && fr.equal(expectedRoot[2], root[2]) && fr.equal(expectedRoot[3], root[3]))); + + zkr = client->flush(uuid, fea2string(fr, root), persistence, flushId, storedFlushId); + zkassertpermanent(zkr==ZKR_SUCCESS); cout << "HashDB client test 8 done" << endl; } + uuid = getUUID(); + tx = 0; + // It should update leaf with more than one level depth { @@ -456,11 +499,17 @@ void* hashDBTestClientThread (const Config& config) for (uint64_t i=0; i<4; i++) root[i] = setResult.newRoot[i]; zkassertpermanent(!fr.isZero(root[0]) || !fr.isZero(root[1]) || !fr.isZero(root[2]) || !fr.isZero(root[3])); - zkassertpermanent(fr.equal(expectedRoot[0], root[0]) && fr.equal(expectedRoot[1], root[1]) && fr.equal(expectedRoot[2], root[2]) && fr.equal(expectedRoot[3], root[3])); + zkassertpermanent(config.hashDB64 || (fr.equal(expectedRoot[0], root[0]) && fr.equal(expectedRoot[1], root[1]) && fr.equal(expectedRoot[2], root[2]) && fr.equal(expectedRoot[3], root[3]))); + + zkr = client->flush(uuid, fea2string(fr, root), persistence, flushId, storedFlushId); + zkassertpermanent(zkr==ZKR_SUCCESS); cout << "HashDB client test 9 done" << endl; } + uuid = getUUID(); + tx = 0; + // It should Zero to Zero with isOldZero=0 { SmtSetResult setResult; @@ -495,12 +544,18 @@ void* hashDBTestClientThread (const Config& config) cout << "SET zkr=" << zkresult2string(zkr) << " root=" << fea2string(fr, root) << " key=" << fea2string(fr, key) << " value=" << value.get_str() << " newRoot=" << fea2string(fr, newRoot) << endl; for (uint64_t i=0; i<4; i++) root[i] = setResult.newRoot[i]; - zkassertpermanent(setResult.mode=="zeroToZero"); - zkassertpermanent(!setResult.isOld0); + zkassertpermanent(config.hashDB64 || (setResult.mode=="zeroToZero")); + zkassertpermanent(config.hashDB64 || !setResult.isOld0); + + zkr = client->flush(uuid, fea2string(fr, root), persistence, flushId, storedFlushId); + zkassertpermanent(zkr==ZKR_SUCCESS); cout << "HashDB client test 10 done" << endl; } + uuid = getUUID(); + tx = 0; + // It should Zero to Zero with isOldZero=0 { @@ -529,12 +584,18 @@ void* hashDBTestClientThread (const Config& config) cout << "SET zkr=" << zkresult2string(zkr) << " root=" << fea2string(fr, root) << " key=" << fea2string(fr, key) << " value=" << value.get_str() << " newRoot=" << fea2string(fr, newRoot) << endl; for (uint64_t i=0; i<4; i++) root[i] = setResult.newRoot[i]; - zkassertpermanent(setResult.mode=="zeroToZero"); - zkassertpermanent(!setResult.isOld0); + zkassertpermanent(config.hashDB64 || (setResult.mode=="zeroToZero")); + zkassertpermanent(config.hashDB64 || !setResult.isOld0); + + zkr = client->flush(uuid, fea2string(fr, root), persistence, flushId, storedFlushId); + zkassertpermanent(zkr==ZKR_SUCCESS); cout << "HashDB client test 11 done" << endl; } + uuid = getUUID(); + tx = 0; + // It should add program data (setProgram) and retrieve it (getProgram) { std::random_device rd; @@ -558,10 +619,15 @@ void* hashDBTestClientThread (const Config& config) for (uint8_t i=0; i<128; i++) { zkassertpermanent(in[i]==out[i]); } + + //zkr = client->flush(uuid, fea2string(fr, root), persistence, flushId, storedFlushId); + //zkassertpermanent(zkr==ZKR_SUCCESS); cout << "HashDB client test 12 done" << endl; } + sleep(1); + delete client; cout << "HashDB test client done" << endl;