Merge pull request #503 from 0xPolygonHermez/fractasy_hashdb_64

HashDB64 infrastructure. Create config.hashDB64 to use it.
0xPolygonHermez · Aug 11, 2023 · 96ed178 · 96ed178
2 parents 9821105 + 3c919f7
commit 96ed178
Show file tree

Hide file tree

Showing 29 changed files with 6,003 additions and 135 deletions.
diff --git a/src/config/config.cpp b/src/config/config.cpp
@@ -272,6 +272,10 @@ void Config::load(json &config)
     if (config.contains("hashDBURL") && config["hashDBURL"].is_string())
         hashDBURL = config["hashDBURL"];
 
+    hashDB64 = false;
+    if (config.contains("hashDB64") && config["hashDB64"].is_boolean())
+        hashDB64 = config["hashDB64"];
+
     dbCacheSynchURL = "";
     if (config.contains("dbCacheSynchURL") && config["dbCacheSynchURL"].is_string())
         dbCacheSynchURL = config["dbCacheSynchURL"];
@@ -673,6 +677,7 @@ void Config::print(void)
     zklog.info("    executorClientCheckNewStateRoot=" + to_string(executorClientCheckNewStateRoot));
     zklog.info("    hashDBServerPort=" + to_string(hashDBServerPort));
     zklog.info("    hashDBURL=" + hashDBURL);
+    zklog.info("    hashDB64=" + to_string(hashDB64));
     zklog.info("    dbCacheSynchURL=" + dbCacheSynchURL);
     zklog.info("    aggregatorServerPort=" + to_string(aggregatorServerPort));
     zklog.info("    aggregatorClientPort=" + to_string(aggregatorClientPort));

diff --git a/src/config/config.hpp b/src/config/config.hpp
@@ -86,6 +86,7 @@ class Config
 
     uint16_t hashDBServerPort;
     string hashDBURL;
+    bool hashDB64;
     string dbCacheSynchURL;
 
     uint16_t aggregatorServerPort;
@@ -165,7 +166,6 @@ class Config
     bool ECRecoverPrecalc;
     uint64_t ECRecoverPrecalcNThreads;
 
-
     void load(json &config);
     bool generateProof(void) const { return runFileGenBatchProof || runFileGenAggregatedProof || runFileGenFinalProof || runAggregatorClient; }
     void print(void);

diff --git a/src/hashdb/database.cpp b/src/hashdb/database.cpp
@@ -46,15 +46,6 @@ Database::Database (Goldilocks &fr, const Config &config) :
     // Initialize semaphores
     sem_init(&senderSem, 0, 0);
     sem_init(&getFlushDataSem, 0, 0);
-
-    // Sender thread creation
-    pthread_create(&senderPthread, NULL, dbSenderThread, this);
-
-    if (config.dbCacheSynchURL.size() > 0)
-    {
-        pthread_create(&cacheSynchPthread, NULL, dbCacheSynchThread, this);
-
-    }
 };
 
 Database::~Database()
@@ -96,9 +87,23 @@ void Database::init(void)
     // Configure the server, if configuration is provided
     if (config.databaseURL != "local")
     {
+        // Sender thread creation
+        pthread_create(&senderPthread, NULL, dbSenderThread, this);
+
+        // Cache synchronization thread creation
+        if (config.dbCacheSynchURL.size() > 0)
+        {
+            pthread_create(&cacheSynchPthread, NULL, dbCacheSynchThread, this);
+
+        }
+
         initRemote();
         useRemoteDB = true;
-    } else useRemoteDB = false;
+    }
+    else
+    {
+        useRemoteDB = false;
+    }
 
     // Mark the database as initialized
     bInitialized = true;
@@ -1527,10 +1532,17 @@ zkresult Database::sendData (void)
             }
 
 #ifdef LOG_DB_WRITE_QUERY
-            zklog.info("Database::sendData() write query=" + data.query);
+            {
+                string query;
+                for (uint64_t i=0; i<data.multiQuery.queries.size(); i++)
+                {
+                    query += data.multiQuery.queries[i].query;
+                }
+                zklog.info("Database::sendData() write query=" + query);
+            }
 #endif
 #ifdef LOG_DB_SEND_DATA
-            zklog.info("Database::sendData() successfully processed query of size= " + to_string(data.query.size()));
+            zklog.info("Database::sendData() successfully processed query of size= " + to_string(data.multiQuery.size()));
 #endif
             // Update status
             data.multiQuery.reset();
@@ -1571,14 +1583,14 @@ zkresult Database::getFlushData(uint64_t flushId, uint64_t &storedFlushId, unord
     iResult = sem_timedwait(&getFlushDataSem, &deadline);
     if (iResult != 0)
     {
-        zklog.info("<-- getFlushData() timed out");
+        zklog.info("Database::getFlushData() timed out");
         return ZKR_SUCCESS;
     }
 
     multiWrite.Lock();
     MultiWriteData &data = multiWrite.data[multiWrite.synchronizingDataIndex];
 
-    zklog.info("getFlushData woke up: pendingToFlushDataIndex=" + to_string(multiWrite.pendingToFlushDataIndex) +
+    zklog.info("Database::getFlushData woke up: pendingToFlushDataIndex=" + to_string(multiWrite.pendingToFlushDataIndex) +
         " storingDataIndex=" + to_string(multiWrite.storingDataIndex) +
         " synchronizingDataIndex=" + to_string(multiWrite.synchronizingDataIndex) +
         " nodes=" + to_string(data.nodes.size()) +

diff --git a/src/hashdb/smt.cpp b/src/hashdb/smt.cpp
@@ -1152,53 +1152,4 @@ int64_t Smt::getUniqueSibling(vector<Goldilocks::Element> &a)
     }
     if (nFound == 1) return fnd;
     return -1;
-}
-
-string SmtSetResult::toString (Goldilocks &fr)
-{
-    string result;
-    result += "mode=" + mode + "\n";
-    result += "oldRoot=" + fea2string(fr, oldRoot) + "\n";
-    result += "newRoot=" + fea2string(fr, newRoot) + "\n";
-    result += "key=" + fea2string(fr, key) + "\n";
-    result += "insKey=" + fea2string(fr, insKey) + "\n";
-    result += "oldValue=" + oldValue.get_str(16) + "\n";
-    result += "newValue=" + newValue.get_str(16) + "\n";
-    result += "insValue=" + insValue.get_str(16) + "\n";
-    result += "isOld0=" + to_string(isOld0) + "\n";
-    result += "proofHashCounter=" + to_string(proofHashCounter) + "\n";
-    map< uint64_t, vector<Goldilocks::Element> >::const_iterator it;
-    for (it=siblings.begin(); it!=siblings.end(); it++)
-    {
-        result += "siblings[" + to_string(it->first) + "]=";
-        for (uint64_t i=0; i<it->second.size(); i++)
-        {
-            result += fr.toString(it->second[i], 16) + ":";
-        }
-        result += "\n";
-    }
-    return result;
-}
-
-string SmtGetResult::toString (Goldilocks &fr)
-{
-    string result;
-    result += "root=" + fea2string(fr, root) + "\n";
-    result += "key=" + fea2string(fr, key) + "\n";
-    result += "insKey=" + fea2string(fr, insKey) + "\n";
-    result += "value=" + value.get_str(16) + "\n";
-    result += "insValue=" + insValue.get_str(16) + "\n";
-    result += "isOld0=" + to_string(isOld0) + "\n";
-    result += "proofHashCounter=" + to_string(proofHashCounter) + "\n";
-    map< uint64_t, vector<Goldilocks::Element> >::const_iterator it;
-    for (it=siblings.begin(); it!=siblings.end(); it++)
-    {
-        result += "siblings[" + to_string(it->first) + "]=";
-        for (uint64_t i=0; i<it->second.size(); i++)
-        {
-            result += fr.toString(it->second[i], 16) + ":";
-        }
-        result += "\n";
-    }
-    return result;
 }
diff --git a/src/hashdb/smt.hpp b/src/hashdb/smt.hpp
@@ -12,41 +12,8 @@
 #include "database_map.hpp"
 #include "zkresult.hpp"
 #include "persistence.hpp"
-
-using namespace std;
-
-// SMT set method result data
-class SmtSetResult
-{
-public:
-    Goldilocks::Element oldRoot[4];
-    Goldilocks::Element newRoot[4];
-    Goldilocks::Element key[4];
-    map< uint64_t, vector<Goldilocks::Element> > siblings;
-    Goldilocks::Element insKey[4];
-    mpz_class insValue;
-    bool isOld0;
-    mpz_class oldValue;
-    mpz_class newValue;
-    string mode;
-    uint64_t proofHashCounter;
-    string toString (Goldilocks &fr);
-};
-
-// SMT get method result data
-class SmtGetResult
-{
-public:
-    Goldilocks::Element root[4]; // merkle-tree root
-    Goldilocks::Element key[4]; // key to look for
-    map< uint64_t, vector<Goldilocks::Element> > siblings; // array of siblings // array of fields??
-    Goldilocks::Element insKey[4]; // key found
-    mpz_class insValue; // value found
-    bool isOld0; // is new insert or delete
-    mpz_class value; // value retrieved
-    uint64_t proofHashCounter;
-    string toString (Goldilocks &fr);
-};
+#include "smt_set_result.hpp"
+#include "smt_get_result.hpp"
 
 class SmtContext
 {

diff --git a/src/hashdb/smt_get_result.cpp b/src/hashdb/smt_get_result.cpp
@@ -0,0 +1,27 @@
+#include "smt_get_result.hpp"
+#include "scalar.hpp"
+
+using namespace std;
+
+string SmtGetResult::toString (Goldilocks &fr)
+{
+    string result;
+    result += "root=" + fea2string(fr, root) + "\n";
+    result += "key=" + fea2string(fr, key) + "\n";
+    result += "insKey=" + fea2string(fr, insKey) + "\n";
+    result += "value=" + value.get_str(16) + "\n";
+    result += "insValue=" + insValue.get_str(16) + "\n";
+    result += "isOld0=" + to_string(isOld0) + "\n";
+    result += "proofHashCounter=" + to_string(proofHashCounter) + "\n";
+    map< uint64_t, vector<Goldilocks::Element> >::const_iterator it;
+    for (it=siblings.begin(); it!=siblings.end(); it++)
+    {
+        result += "siblings[" + to_string(it->first) + "]=";
+        for (uint64_t i=0; i<it->second.size(); i++)
+        {
+            result += fr.toString(it->second[i], 16) + ":";
+        }
+        result += "\n";
+    }
+    return result;
+}
diff --git a/src/hashdb/smt_get_result.hpp b/src/hashdb/smt_get_result.hpp
@@ -0,0 +1,27 @@
+#ifndef SMT_GET_RESULT_HPP
+#define SMT_GET_RESULT_HPP
+
+#include <vector>
+#include <map>
+
+#include "poseidon_goldilocks.hpp"
+#include "goldilocks_base_field.hpp"
+
+using namespace std;
+
+// SMT get method result data
+class SmtGetResult
+{
+public:
+    Goldilocks::Element root[4]; // merkle-tree root
+    Goldilocks::Element key[4]; // key to look for
+    map< uint64_t, vector<Goldilocks::Element> > siblings; // array of siblings // array of fields??
+    Goldilocks::Element insKey[4]; // key found
+    mpz_class insValue; // value found
+    bool isOld0; // is new insert or delete
+    mpz_class value; // value retrieved
+    uint64_t proofHashCounter;
+    string toString (Goldilocks &fr);
+};
+
+#endif
diff --git a/src/hashdb/smt_set_result.cpp b/src/hashdb/smt_set_result.cpp
@@ -0,0 +1,30 @@
+#include "smt_set_result.hpp"
+#include "scalar.hpp"
+
+using namespace std;
+
+string SmtSetResult::toString (Goldilocks &fr)
+{
+    string result;
+    result += "mode=" + mode + "\n";
+    result += "oldRoot=" + fea2string(fr, oldRoot) + "\n";
+    result += "newRoot=" + fea2string(fr, newRoot) + "\n";
+    result += "key=" + fea2string(fr, key) + "\n";
+    result += "insKey=" + fea2string(fr, insKey) + "\n";
+    result += "oldValue=" + oldValue.get_str(16) + "\n";
+    result += "newValue=" + newValue.get_str(16) + "\n";
+    result += "insValue=" + insValue.get_str(16) + "\n";
+    result += "isOld0=" + to_string(isOld0) + "\n";
+    result += "proofHashCounter=" + to_string(proofHashCounter) + "\n";
+    map< uint64_t, vector<Goldilocks::Element> >::const_iterator it;
+    for (it=siblings.begin(); it!=siblings.end(); it++)
+    {
+        result += "siblings[" + to_string(it->first) + "]=";
+        for (uint64_t i=0; i<it->second.size(); i++)
+        {
+            result += fr.toString(it->second[i], 16) + ":";
+        }
+        result += "\n";
+    }
+    return result;
+}
diff --git a/src/hashdb/smt_set_result.hpp b/src/hashdb/smt_set_result.hpp
@@ -0,0 +1,30 @@
+#ifndef SMT_SET_RESULT_HPP
+#define SMT_SET_RESULT_HPP
+
+#include <vector>
+#include <map>
+
+#include "poseidon_goldilocks.hpp"
+#include "goldilocks_base_field.hpp"
+
+using namespace std;
+
+// SMT set method result data
+class SmtSetResult
+{
+public:
+    Goldilocks::Element oldRoot[4];
+    Goldilocks::Element newRoot[4];
+    Goldilocks::Element key[4];
+    map< uint64_t, vector<Goldilocks::Element> > siblings;
+    Goldilocks::Element insKey[4];
+    mpz_class insValue;
+    bool isOld0;
+    mpz_class oldValue;
+    mpz_class newValue;
+    string mode;
+    uint64_t proofHashCounter;
+    string toString (Goldilocks &fr);
+};
+
+#endif