Skip to content

Commit

Permalink
Merge pull request #525 from 0xPolygonHermez/fractasy_hashdb_64
Browse files Browse the repository at this point in the history 
Fractasy hashdb 64
  • Loading branch information
@fractasy
fractasy authored Aug 31, 2023
2 parents f4fac54 + aa886e5 commit cfe67be
Show file tree
Hide file tree
Showing 7 changed files with 306 additions and 45 deletions.
1 change: 1 addition & 0 deletions src/hashdb64/child.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ class Child
{
switch (type)
{
case UNSPECIFIED: return "U";
case ZERO: return "z";
case LEAF: return "L";
case INTERMEDIATE: return "I";
Expand Down
5 changes: 3 additions & 2 deletions src/hashdb64/database_64.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -346,6 +346,7 @@ zkresult Database64::write (vector<DB64Query> &dbQueries, const bool persistent)
zkresult zkr;
for (uint64_t i=0; i<dbQueries.size(); i++)
{
//zklog.info("Database64::write() writing hash=" + dbQueries[i].key + " size=" + to_string(dbQueries[i].value.size()) + " data=" + ba2string(dbQueries[i].value));
zkr = write(dbQueries[i].key, dbQueries[i].keyFea, dbQueries[i].value, persistent);
if (zkr != ZKR_SUCCESS)
{
Expand Down Expand Up @@ -585,7 +586,7 @@ zkresult Database64::readRemote(bool bProgram, const string &key, string &value)
exitProcess();
}
pqxx::field const fieldData = row[1];
value = removeBSXIfExists64(fieldData.c_str());
value = string2ba(removeBSXIfExists64(fieldData.c_str()));
}
catch (const std::exception &e)
{
Expand Down Expand Up @@ -1385,7 +1386,7 @@ zkresult Database64::sendData (void)
data.multiQuery.queries[currentQuery].query += ", ";
}
firstValue = false;
data.multiQuery.queries[currentQuery].query += "( E\'\\\\x" + it->first + "\', E\'\\\\x" + it->second + "\' ) ";
data.multiQuery.queries[currentQuery].query += "( E\'\\\\x" + it->first + "\', E\'\\\\x" + ba2string(it->second) + "\' ) ";
#ifdef LOG_DB_SEND_DATA
zklog.info("Database64::sendData() inserting node key=" + it->first + " value=" + it->second);
#endif
Expand Down
233 changes: 218 additions & 15 deletions src/hashdb64/smt_64.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
#include "key_utils.hpp"
#include "tree_chunk.hpp"

zkresult Smt64::writeTree(Database64 &db, const Goldilocks::Element (&oldRoot)[4], const vector<KeyValue> keyValues, Goldilocks::Element (&newRoot)[4])
zkresult Smt64::writeTree (Database64 &db, const Goldilocks::Element (&oldRoot)[4], const vector<KeyValue> &keyValues, Goldilocks::Element (&newRoot)[4], uint64_t &flushId, uint64_t &lastSentFlushId)
{
zkresult zkr;

Expand Down Expand Up @@ -46,10 +46,7 @@ zkresult Smt64::writeTree(Database64 &db, const Goldilocks::Element (&oldRoot)[4
}

// Copy the key values list into the root tree chunk
for (uint64_t i = 0; i < keyValues.size(); i++)
{
c->list.emplace_back(keyValues[i]);
}
c->list = keyValues;

while (chunksProcessed < chunks.size())
{
Expand Down Expand Up @@ -117,7 +114,7 @@ zkresult Smt64::writeTree(Database64 &db, const Goldilocks::Element (&oldRoot)[4
TreeChunk *c = new TreeChunk(db, poseidon);
if (c == NULL)
{
zklog.error("Smt64::MultiWrite() failed calling new TreeChunk()");
zklog.error("Smt64::writeTree() failed calling new TreeChunk()");
exitProcess();
}

Expand Down Expand Up @@ -158,7 +155,7 @@ zkresult Smt64::writeTree(Database64 &db, const Goldilocks::Element (&oldRoot)[4
TreeChunk *c = new TreeChunk(db, poseidon);
if (c == NULL)
{
zklog.error("Smt64::MultiWrite() failed calling new TreeChunk()");
zklog.error("Smt64::writeTree() failed calling new TreeChunk()");
exitProcess();
}
c->setLevel(level + 6);
Expand All @@ -179,7 +176,7 @@ zkresult Smt64::writeTree(Database64 &db, const Goldilocks::Element (&oldRoot)[4
}
default:
{
zklog.error("Smt64::MultiWrite() found invalid chunks[i]->getChild(k).type=" + to_string(chunks[i]->getChild(k).type));
zklog.error("Smt64::writeTree() found invalid chunks[i]->getChild(k).type=" + to_string(chunks[i]->getChild(k).type));
exitProcess();
}
}
Expand All @@ -197,7 +194,7 @@ zkresult Smt64::writeTree(Database64 &db, const Goldilocks::Element (&oldRoot)[4
zkr = calculateHash(result, chunks, dbQueries, 0, 0);
if (zkr != ZKR_SUCCESS)
{
zklog.error("Smt64::MultiWrite() failed calling calculateHash() result=" + zkresult2string(zkr));
zklog.error("Smt64::writeTree() failed calling calculateHash() result=" + zkresult2string(zkr));
for (uint c = 0; c < chunks.size(); c++) delete chunks[c];
return zkr;
}
Expand All @@ -216,7 +213,7 @@ zkresult Smt64::writeTree(Database64 &db, const Goldilocks::Element (&oldRoot)[4
zkr = chunks[0]->children2data();
if (zkr != ZKR_SUCCESS)
{
zklog.error("Smt64::MultiWrite() failed calling chunks[0]->children2data() result=" + zkresult2string(zkr));
zklog.error("Smt64::writeTree() failed calling chunks[0]->children2data() result=" + zkresult2string(zkr));
for (uint c = 0; c < chunks.size(); c++) delete chunks[c];
return zkr;
}
Expand All @@ -240,7 +237,7 @@ zkresult Smt64::writeTree(Database64 &db, const Goldilocks::Element (&oldRoot)[4
}
else
{
zklog.error("Smt64::MultiWrite() found invalid result.type=" + to_string(result.type));
zklog.error("Smt64::writeTree() found invalid result.type=" + to_string(result.type));
for (uint c = 0; c < chunks.size(); c++) delete chunks[c];
return zkr;
}
Expand All @@ -249,15 +246,24 @@ zkresult Smt64::writeTree(Database64 &db, const Goldilocks::Element (&oldRoot)[4
zkr = db.write(dbQueries, true);
if (zkr != ZKR_SUCCESS)
{
zklog.error("Smt64::MultiWrite() failed calling db.write() result=" + zkresult2string(zkr));
zklog.error("Smt64::writeTree() failed calling db.write() result=" + zkresult2string(zkr));
for (uint c = 0; c < chunks.size(); c++) delete chunks[c];
return zkr;
}

// Flush written data to database
zkr = db.flush(flushId, lastSentFlushId);
if (zkr != ZKR_SUCCESS)
{
zklog.error("Smt64::writeTree() failed calling db.flush() result=" + zkresult2string(zkr));
for (uint c = 0; c < chunks.size(); c++) delete chunks[c];
return zkr;
}

// Print chunks
for (uint c = 0; c < chunks.size(); c++)
{
zklog.info("Smt64::MultiWrite() chunk " + to_string(c));
zklog.info("Smt64::writeTree() chunk " + to_string(c));
chunks[c]->print();
}

Expand All @@ -283,21 +289,218 @@ zkresult Smt64::calculateHash (Child &result, vector<TreeChunk *> &chunks, vecto
}
}

// Now, calculate the hash of this chunk
// Calculate the hash of this chunk
zkr = chunks[chunkId]->calculateHash();
if (zkr != ZKR_SUCCESS)
{
zklog.error("Smt64::recalculateHash() failed calling chunks[chunkId]->calculateHash() result=" + zkresult2string(zkr));
return zkr;
}

// Copy the result child
result = chunks[chunkId]->getChild1();

// Add to the database queries
if (result.type != ZERO)
{
// Encode the 64 children into database format
zkr = chunks[chunkId]->children2data();
if (zkr != ZKR_SUCCESS)
{
zklog.error("Smt64::recalculateHash() failed calling chunks[chunkId]->children2data() result=" + zkresult2string(zkr));
return zkr;
}

Goldilocks::Element hash[4];
chunks[chunkId]->getHash(hash);
DB64Query dbQuery(fea2string(fr, hash), hash, chunks[chunkId]->data);
dbQueries.emplace_back(dbQuery);
}

return ZKR_SUCCESS;
}

zkresult Smt64::readTree(Database64 &db, const Goldilocks::Element (&root)[4], const vector<Key> keys, vector<KeyValue> (&keyValues))
zkresult Smt64::readTree (Database64 &db, const Goldilocks::Element (&root)[4], vector<KeyValue> &keyValues)
{
zkresult zkr;

vector<TreeChunk *> chunks;
vector<DB64Query> dbQueries;

// Tree level; we start at level 0, then we increase it 6 by 6
uint64_t level = 0;

// Create the first tree chunk (the root one), and store it in chunks[0]
TreeChunk *c = new TreeChunk(db, poseidon);
if (c == NULL)
{
zklog.error("Smt64::readTree() failed calling new TreeChunk()");
exitProcess();
}
chunks.push_back(c);

uint64_t chunksProcessed = 0;

// Get the old root as a string
string rootString = fea2string(fr, root);

// If root is zero, return all values as zero
if (rootString == "0")
{
delete c;
for (uint64_t i=0; i<keyValues.size(); i++)
{
keyValues[i].value = 0;
}
return ZKR_SUCCESS;
}
else
{
DB64Query dbQuery(rootString, root, chunks[0]->data);
dbQueries.push_back(dbQuery);
}

// Copy the key values list into the root tree chunk
c->list = keyValues;

while (chunksProcessed < chunks.size())
{
zkr = db.read(dbQueries);
if (zkr != ZKR_SUCCESS)
{
zklog.error("Smt64::readTree() failed calling db.multiRead() result=" + zkresult2string(zkr));
for (uint c = 0; c < chunks.size(); c++) delete chunks[c];
return zkr;
}
dbQueries.clear();

int chunksToProcess = chunks.size();

for (int i=chunksProcessed; i<chunksToProcess; i++)
{
if (chunks[i]->data.size() > 0)
{
zkr = chunks[i]->data2children();
if (zkr != ZKR_SUCCESS)
{
zklog.error("Smt64::readTree() failed calling chunks[i]->data2children() result=" + zkresult2string(zkr));
return zkr;
}
}
for (uint64_t j=0; j<chunks[i]->list.size(); j++)
{
bool keyBits[256];
splitKey(fr, chunks[i]->list[j].key.fe, keyBits);
uint64_t k = getKeyChildren64Position(keyBits, level);
switch (chunks[i]->getChild(k).type)
{
case ZERO:
{
for (uint64_t kv=0; kv<keyValues.size(); kv++)
{
if (fr.equal(keyValues[kv].key.fe[0], chunks[i]->list[j].key.fe[0]) &&
fr.equal(keyValues[kv].key.fe[1], chunks[i]->list[j].key.fe[1]) &&
fr.equal(keyValues[kv].key.fe[2], chunks[i]->list[j].key.fe[2]) &&
fr.equal(keyValues[kv].key.fe[3], chunks[i]->list[j].key.fe[3]))
{
keyValues[kv].value = 0;
}
}
break;
}
case LEAF:
{
// If the key is the same, then check the value
if (fr.equal(chunks[i]->getChild(k).leaf.key[0], chunks[i]->list[j].key.fe[0]) &&
fr.equal(chunks[i]->getChild(k).leaf.key[1], chunks[i]->list[j].key.fe[1]) &&
fr.equal(chunks[i]->getChild(k).leaf.key[2], chunks[i]->list[j].key.fe[2]) &&
fr.equal(chunks[i]->getChild(k).leaf.key[3], chunks[i]->list[j].key.fe[3]))
{

for (uint64_t kv=0; kv<keyValues.size(); kv++)
{
if (fr.equal(keyValues[kv].key.fe[0], chunks[i]->list[j].key.fe[0]) &&
fr.equal(keyValues[kv].key.fe[1], chunks[i]->list[j].key.fe[1]) &&
fr.equal(keyValues[kv].key.fe[2], chunks[i]->list[j].key.fe[2]) &&
fr.equal(keyValues[kv].key.fe[3], chunks[i]->list[j].key.fe[3]))
{
keyValues[kv].value = chunks[i]->getChild(k).leaf.value;
}
}
}
else
{
for (uint64_t kv=0; kv<keyValues.size(); kv++)
{
if (fr.equal(keyValues[kv].key.fe[0], chunks[i]->list[j].key.fe[0]) &&
fr.equal(keyValues[kv].key.fe[1], chunks[i]->list[j].key.fe[1]) &&
fr.equal(keyValues[kv].key.fe[2], chunks[i]->list[j].key.fe[2]) &&
fr.equal(keyValues[kv].key.fe[3], chunks[i]->list[j].key.fe[3]))
{
keyValues[kv].value = 0;
}
}
}
break;
}
case TREE_CHUNK:
{
// Simply add it to the list of the descendant tree chunk
chunks[chunks[i]->getChild(k).treeChunkId]->list.push_back(chunks[i]->list[j]);

break;
}
// If this is an intermediate node, then create the corresponding tree chunk
case INTERMEDIATE:
{
// We create a new trunk
TreeChunk *c = new TreeChunk(db, poseidon);
if (c == NULL)
{
zklog.error("Smt64::readTree() failed calling new TreeChunk()");
exitProcess();
}
c->setLevel(level + 6);

// Create a new query to populate this tree chunk from database
DB64Query dbQuery(fea2string(fr, chunks[i]->getChild(k).intermediate.hash),
chunks[i]->getChild(k).intermediate.hash,
c->data);
dbQueries.push_back(dbQuery);

// Add the requested key-value to the new tree chunk list
c->list.push_back(chunks[i]->list[j]);
int cId = chunks.size();
chunks.push_back(c);
chunks[i]->setTreeChunkChild(k, cId);

break;
}
default:
{
zklog.error("Smt64::readTree() found invalid chunks[i]->getChild(k).type=" + to_string(chunks[i]->getChild(k).type));
exitProcess();
}
}
}
}

chunksProcessed = chunksToProcess;
level += 6;
}

dbQueries.clear();

// Print chunks
for (uint c = 0; c < chunks.size(); c++)
{
zklog.info("Smt64::readTree() chunk " + to_string(c));
chunks[c]->print();
}

// Free memory
for (uint c = 0; c < chunks.size(); c++) delete chunks[c];

return ZKR_SUCCESS;
}

Expand Down
5 changes: 2 additions & 3 deletions src/hashdb64/smt_64.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,11 +98,10 @@ class Smt64
capacityOne[3] = fr.zero();
}

zkresult writeTree(Database64 &db, const Goldilocks::Element (&oldRoot)[4], const vector<KeyValue> keyValues, Goldilocks::Element (&newRoot)[4]);
zkresult writeTree(Database64 &db, const Goldilocks::Element (&oldRoot)[4], const vector<KeyValue> &keyValues, Goldilocks::Element (&newRoot)[4], uint64_t &flushId, uint64_t &lastSentFlushId);
zkresult calculateHash (Child &result, std::vector<TreeChunk *> &chunks, vector<DB64Query> &dbQueries, int idChunk, int level);

// TODO: return a flush ID, store to DB in background
zkresult readTree(Database64 &db, const Goldilocks::Element (&root)[4], const vector<Key> keys, vector<KeyValue> (&keyValues));
zkresult readTree(Database64 &db, const Goldilocks::Element (&root)[4], vector<KeyValue> &keyValues);

zkresult set(const string &batchUUID, uint64_t tx, Database64 &db, const Goldilocks::Element (&oldRoot)[4], const Goldilocks::Element (&key)[4], const mpz_class &value, const Persistence persistence, SmtSetResult &result, DatabaseMap *dbReadLog = NULL);
zkresult get(const string &batchUUID, Database64 &db, const Goldilocks::Element (&root)[4], const Goldilocks::Element (&key)[4], SmtGetResult &result, DatabaseMap *dbReadLog = NULL);
Expand Down
Loading

0 comments on commit cfe67be

Please sign in to comment.