Rick/fix kv database

src/config/config.cpp

@@ -147,6 +147,7 @@ void Config::load(json &config)
     ParseString(config, "checkTreeRoot", "CHECK_TREE_ROOT", checkTreeRoot, "auto");
     ParseBool(config, "runDatabasePerformanceTest", "RUN_DATABASE_PERFORMANCE_TEST", runDatabasePerformanceTest, false);
     ParseBool(config, "runSMT64Test", "RUN_SMT64_TEST", runSMT64Test, false);
+    ParseBool(config, "runDbKVRemoteTest", "RUN_DBKV_REMOTE_TEST", runDbKVRemoteTest, false);
     ParseBool(config, "runUnitTest", "RUN_UNIT_TEST", runUnitTest, false);
 
     // Main SM executor
@@ -192,9 +193,12 @@ void Config::load(json &config)
 
     // MT associative cache
     ParseBool(config, "useAssociativeCache", "USE_ASSOCIATIVE_CACHE", useAssociativeCache, false);
-    ParseS64(config, "log2DbMTAssociativeCacheSize", "LOG2_DB_MT_ASSOCIATIVE_CACHE_SIZE", log2DbMTAssociativeCacheSize, 24);
+    ParseS64(config, "log2DbMTAssociativeCacheSize", "LOG2_DB_MT_ASSOCIATIVE_CACHE_SIZE", log2DbMTAssociativeCacheSize, 25);
     ParseS64(config, "log2DbMTAssociativeCacheIndexesSize", "LOG2_DB_MT_ASSOCIATIVE_CACHE_INDEXES_SIZE", log2DbMTAssociativeCacheIndexesSize, 28);
-
+    ParseS64(config, "log2DbKVAssociativeCacheSize", "LOG2_DB_KV_ASSOCIATIVE_CACHE_SIZE", log2DbKVAssociativeCacheSize, 25);
+    ParseS64(config, "log2DbKVAssociativeCacheIndexesSize", "LOG2_DB_KV_ASSOCIATIVE_CACHE_INDEXES_SIZE", log2DbKVAssociativeCacheIndexesSize, 28);
+    ParseS64(config, "log2DbVersionsAssociativeCacheSize", "LOG2_DB_VERSIONS_ASSOCIATIVE_CACHE_SIZE", log2DbVersionsAssociativeCacheSize, 25);
+    ParseS64(config, "log2DbVersionsAssociativeCacheIndexesSize", "LOG2_DB_VERSIONS_ASSOCIATIVE_CACHE_INDEXES_SIZE", log2DbVersionsAssociativeCacheIndexesSize, 28);
     // Program (SC) cache
     ParseS64(config, "dbProgramCacheSize", "DB_PROGRAM_CACHE_SIZE", dbProgramCacheSize, 1*1024); // Default = 1 GB
 
@@ -361,6 +365,8 @@ void Config::print(void)
         zklog.info("    runDatabasePerformanceTest=true");
     if (runSMT64Test)
         zklog.info("    runSMT64Test=true");
+    if (runDbKVRemoteTest)
+        zklog.info("    runDbKVRemoteTest=true");
     if (runUnitTest)
         zklog.info("    runUnitTest=true");
 
@@ -484,6 +490,10 @@ void Config::print(void)
     zklog.info("    useAssociativeCache=" + to_string(useAssociativeCache));
     zklog.info("    log2DbMTAssociativeCacheSize=" + to_string(log2DbMTAssociativeCacheSize));
     zklog.info("    log2DbMTAssociativeCacheIndexesSize=" + to_string(log2DbMTAssociativeCacheIndexesSize));
+    zklog.info("    log2DbKVAssociativeCacheSize=" + to_string(log2DbKVAssociativeCacheSize));
+    zklog.info("    log2DbKVAssociativeCacheIndexesSize=" + to_string(log2DbKVAssociativeCacheIndexesSize));
+    zklog.info("    log2DbVersionsAssociativeCacheSize=" + to_string(log2DbVersionsAssociativeCacheSize));
+    zklog.info("    log2DbVersionsAssociativeCacheIndexesSize=" + to_string(log2DbVersionsAssociativeCacheIndexesSize));
     zklog.info("    dbProgramCacheSize=" + to_string(dbProgramCacheSize));
     zklog.info("    loadDBToMemTimeout=" + to_string(loadDBToMemTimeout));
     zklog.info("    fullTracerTraceReserveSize=" + to_string(fullTracerTraceReserveSize));

src/config/config.hpp

@@ -45,6 +45,7 @@ class Config
     string checkTreeRoot;
     bool runDatabasePerformanceTest;
     bool runSMT64Test;
+    bool runDbKVRemoteTest;
     bool runUnitTest;
 
     bool executeInParallel;
@@ -65,8 +66,12 @@ class Config
     uint64_t loadDBToMemTimeout;
     int64_t dbMTCacheSize; // Size in MBytes for the cache to store MT records
     bool useAssociativeCache; // Use the associative cache for MT records?
-    int64_t log2DbMTAssociativeCacheSize; // log2 of the size in entries of the DatabaseMTAssociativeCache. Note 1 cache entry = 97 bytes
+    int64_t log2DbMTAssociativeCacheSize; // log2 of the size in entries of the DatabaseMTAssociativeCache. Note 1 cache entry = 128 bytes
     int64_t log2DbMTAssociativeCacheIndexesSize; // log2 of the size in entries of the DatabaseMTAssociativeCache indices. Note index entry = 4 bytes
+    int64_t log2DbKVAssociativeCacheSize; // log2 of the size in entries of the DatabaseKVAssociativeCache. Note 1 cache entry = 80 bytes
+    int64_t log2DbKVAssociativeCacheIndexesSize; // log2 of the size in entries of the DatabaseKVAssociativeCache indices. Note index entry = 4 bytes
+    int64_t log2DbVersionsAssociativeCacheSize; // log2 of the size in entries of the DatabaseKVAssociativeCache. Note 1 cache entry = 40 bytes
+    int64_t log2DbVersionsAssociativeCacheIndexesSize; // log2 of the size in entries of the DatabaseKVAssociativeCache indices. Note index entry = 4 bytes
 
     int64_t dbProgramCacheSize; // Size in MBytes for the cache to store Program (SC) records
     bool opcodeTracer;

src/config/zkresult.cpp

@@ -87,7 +87,8 @@ struct
     { ZKR_SM_MAIN_MEMALIGN_READ_MISMATCH, "ZKR_SM_MAIN_MEMALIGN_READ_MISMATCH" },
     { ZKR_SM_MAIN_HASHK_READ_OUT_OF_RANGE, "ZKR_SM_MAIN_HASHK_READ_OUT_OF_RANGE" },
     { ZKR_SM_MAIN_HASHP_READ_OUT_OF_RANGE, "ZKR_SM_MAIN_HASHP_READ_OUT_OF_RANGE" },
-    { ZKR_DB_VERSION_NOT_FOUND, "ZKR_DB_VERSION_NOT_FOUND" }
+    { ZKR_DB_VERSION_NOT_FOUND, "ZKR_DB_VERSION_NOT_FOUND" },
+    { ZKR_DB_VERSION_NOT_FOUND_GLOBAL, "ZKR_DB_VERSION_NOT_FOUND_GLOBAL"}
 
 
 };

src/config/zkresult.hpp

@@ -89,6 +89,8 @@ typedef enum : int
     ZKR_SM_MAIN_HASHK_READ_OUT_OF_RANGE = 79, // Main state Keccak hash check found read out of range
     ZKR_SM_MAIN_HASHP_READ_OUT_OF_RANGE = 80, // Main state Poseidon hash check found read out of range
     ZKR_DB_VERSION_NOT_FOUND = 81, // Version not found in KeyValue database
+    ZKR_DB_VERSION_NOT_FOUND_GLOBAL = 82, // Version not found in KeyValue database and not present in hashDB neither
+
 
 } zkresult;
 

src/hashdb64/database_64.cpp

@@ -443,10 +443,11 @@ zkresult Database64::readKV(const Goldilocks::Element (&root)[4], const Goldiloc
     // If we could not find the value, report the error
     if (rv != ZKR_SUCCESS || rkv != ZKR_SUCCESS)
     {
-        zklog.warning("Database64::readKV() requested a key that does not exist (ZKR_DB_KEY_NOT_FOUND): " + keyStr + ", now trying to read from the hasDB");
+        zklog.warning("Database64::readKV() requested a key that does not exist (ZKR_DB_KEY_NOT_FOUND): " + keyStr);
         string valueStr;
-        rkv = ZKR_DB_KEY_NOT_FOUND;
-        /* Pending*/
+        if( rkv == ZKR_DB_VERSION_NOT_FOUND){
+            //Pending 
+        }
     }
 #ifdef LOG_DB_READ
     {
@@ -1274,10 +1275,8 @@ zkresult Database64::readRemoteKV(const uint64_t version, const Goldilocks::Elem
             zklog.error("DatabaseKV::readRemoteKV() table=" + tableName + " got an invalid number of colums for the row: " + to_string(row.size()) + "for key=" + keyStr);
             exitProcess();
         }
-        bool foundVersion = extractVersion(row[1], version, value); 
-        if(foundVersion==false){
-            return ZKR_DB_VERSION_NOT_FOUND;
-        }
+        return extractVersion(row[1], version, value); 
+
 
     }
     catch (const std::exception &e)
@@ -1294,11 +1293,12 @@ zkresult Database64::readRemoteKV(const uint64_t version, const Goldilocks::Elem
     return ZKR_SUCCESS;
 }
 
-bool Database64::extractVersion(const pqxx::field& fieldData, const uint64_t version, mpz_class &value){
+zkresult Database64::extractVersion(const pqxx::field& fieldData, const uint64_t version, mpz_class &value){
 
+    int data_size = 0;
     if(!fieldData.is_null()){
         string data = removeBSXIfExists64(fieldData.c_str());
-        int data_size = data.size();
+        data_size = data.size();
         zkassert(data_size % 80 == 0);
         for (int i = 0; i < data_size; i += 80)
         {
@@ -1307,7 +1307,7 @@ bool Database64::extractVersion(const pqxx::field& fieldData, const uint64_t ver
             uint64_t version_ = aux.get_ui();
             if(version_==version){
                 value = mpz_class(data.substr(i + 16, 64), 16);
-                return true;
+                return ZKR_SUCCESS;
             }
         }
     }
@@ -1323,7 +1323,15 @@ bool Database64::extractVersion(const pqxx::field& fieldData, const uint64_t ver
             return true;
         }
     }*/
-    return false;
+    if(data_size < 80*maxVersions){
+        return ZKR_DB_VERSION_NOT_FOUND_GLOBAL;
+    }else{
+        if(data_size != 80*maxVersions){
+            zklog.error("Database64::extractVersion() got an invalid data size: " + to_string(data_size));
+            exitProcess();
+        }
+        return ZKR_DB_VERSION_NOT_FOUND;
+    }
 }
 
 zkresult Database64::writeRemoteKV(const uint64_t version, const Goldilocks::Element (&key)[4], const mpz_class &value, bool noMultiWrite)
@@ -1349,7 +1357,9 @@ zkresult Database64::writeRemoteKV(const uint64_t version, const Goldilocks::Ele
         const string &tableName = config.dbKeyValueTableName;
         string keyStr_ = fea2string(fr, key[0], key[1], key[2], key[3]); 
         string keyStr = NormalizeToNFormat(keyStr_, 64);
-        string insertStr = to_string(version) + value.get_str(16);
+        string valueStr = NormalizeToNFormat(value.get_str(16),64);
+        string versionStr = NormalizeToNFormat(U64toString(version,16),16); 
+        string insertStr = versionStr + valueStr;
         assert(insertStr.size() == 80);
 
         if (config.logRemoteDbReads)
@@ -1498,7 +1508,7 @@ zkresult Database64::readRemoteVersion(const Goldilocks::Element (&root)[4], uin
         }
         pqxx::field const fieldData = row[1];
         if (!fieldData.is_null()) {
-            fieldData.as<uint64_t>(version);
+            version = fieldData.as<uint64_t>();
         } else {
             zklog.error("DatabaseKV::readRemoteVersion() table=" + tableName + " got a null version for root: " + rootStr);
         }
@@ -1535,7 +1545,7 @@ zkresult Database64::writeRemoteVersion(const Goldilocks::Element (&root)[4], co
     else
     {
         const string &tableName =  config.dbVersionTableName;
-        string query = "INSERT INTO " + tableName + " ( hash, version ) VALUES ( E\'\\\\x" + rootStr + "\', E\'\\\\x" + to_string(version) + "\' ) ON CONFLICT (hash) DO NOTHING;";
+        string query = "INSERT INTO " + tableName + " ( hash, version ) VALUES ( E\'\\\\x" + rootStr + "\', " + to_string(version) + " ) ON CONFLICT (hash) DO NOTHING;";
         DatabaseConnection * pDatabaseConnection = getConnection();
 
         try
@@ -1620,7 +1630,7 @@ zkresult Database64::readRemoteLatestVersion(uint64_t &version){
         }
         pqxx::field const fieldData = row[0];
         if (!fieldData.is_null()) {
-            fieldData.as<uint64_t>(version);
+            version = fieldData.as<uint64_t>();
         } else {
             zklog.error("DatabaseKV::readRemoteLatestVersion() table=" + tableName + " got a null version;");
         }

src/hashdb64/database_64.hpp

@@ -90,7 +90,7 @@ class Database64
     zkresult readRemoteLatestVersion(uint64_t &version);
     zkresult writeRemoteLatestVersion(const uint64_t version);
 
-    bool extractVersion(const pqxx::field& fieldData, const uint64_t version, mpz_class &value);
+    zkresult extractVersion(const pqxx::field& fieldData, const uint64_t version, mpz_class &value);
 
 public:
 #ifdef DATABASE_USE_CACHE

src/main.cpp

@@ -45,6 +45,7 @@
 #include "database_performance_test.hpp"
 #include "smt_64_test.hpp"
 #include "sha256.hpp"
+#include "database_kv_remote_test.hpp"
 
 using namespace std;
 using json = nlohmann::json;
@@ -597,6 +598,11 @@ int main(int argc, char **argv)
     {
         DatabasePerformanceTest();
     }
+    // Test DbKVRemote
+    if (config.runDbKVRemoteTest)
+    {
+        DatabaseKVRemoteTest(config);
+    }
 
     // Test SMT64
     if (config.runSMT64Test)
@@ -650,8 +656,8 @@ int main(int argc, char **argv)
         Database::dbMTCache.setMaxSize(config.dbMTCacheSize*1024*1024);
     }
     if(config.hashDB64){
-        Database64::dbKVACache.postConstruct(config.log2DbMTAssociativeCacheIndexesSize, config.log2DbMTAssociativeCacheSize, "KVACache");
-        Database64::dbVersionACache.postConstruct(config.log2DbMTAssociativeCacheIndexesSize, config.log2DbMTAssociativeCacheSize, "VersionACache");
+        Database64::dbKVACache.postConstruct(config.log2DbKVAssociativeCacheIndexesSize, config.log2DbKVAssociativeCacheSize, "KVACache");
+        Database64::dbVersionACache.postConstruct(config.log2DbVersionsAssociativeCacheIndexesSize, config.log2DbVersionsAssociativeCacheSize, "VersionACache");
     }
     Database::dbProgramCache.setName("ProgramCache");
     Database::dbProgramCache.setMaxSize(config.dbProgramCacheSize*1024*1024);

src/utils/scalar.hpp

@@ -398,6 +398,20 @@ void bytes2u32 (const uint8_t * pInput, uint32_t &output, bool bBigEndian);
 /* Array of bytes to unsigned 64.  pInput must be 8 bytes long */
 void bytes2u64 (const uint8_t * pInput, uint64_t &output, bool bBigEndian);
 
+/* unsigned64 to string*/
+inline void U64toString(std::string &result, const uint64_t in1, const int radix)
+{
+    mpz_class aux = in1;
+    result = aux.get_str(radix);
+}
+
+/* unsigned64 to string*/
+inline std::string U64toString( const uint64_t in1, const int radix)
+{
+    mpz_class aux = in1;
+    string result = aux.get_str(radix);
+    return result;
+}
 /* Swap bytes, e.g. little to big endian, and vice-versa */
 uint64_t swapBytes64 (uint64_t input);
 

test/hashdb/database_kv_remote_test.cpp

@@ -0,0 +1,61 @@
+#include "database_kv_remote_test.hpp"
+#include "goldilocks_base_field.hpp"
+#include "hashdb.hpp"
+#include "hashdb_singleton.hpp"
+
+#define DBKV_REMOTE_TEST_NUMBER_OF_WRITES 1000
+
+
+uint64_t DatabaseKVRemoteTest (const Config &config){
+
+
+    uint64_t numberOfFailedTests = 0;
+
+    TimerStart(DATABASE_KV_REMOTE_TEST);
+
+    if(config.dbMultiWrite == true){
+        zklog.error("DatabaseKVRemoteTest() this test must be run with config.dbMultiWrite=false");
+        exitProcess();
+    }
+    HashDB * pHashDB = (HashDB *)hashDBSingleton.get();
+    if (pHashDB == NULL)
+    {
+        zklog.error("DatabaseKVRemoteTest() failed calling HashDBSingleton::get()");
+        exitProcess();
+    }
+    Database64 * pDatabase64 = &pHashDB->db64;
+    pDatabase64->init(); //problematic!
+
+    //
+    // Lattest version
+    //
+    uint64_t version = 1;
+    pDatabase64->writeLatestVersion(version, true);
+    version = pDatabase64->readLatestVersion(version);
+    if(version != 1)
+    {
+        zklog.error("DatabaseKVRemoteTest() failed calling Database64.readLatestVersion()");
+        numberOfFailedTests += 1;
+    }
+    pDatabase64->writeLatestVersion(0, false);
+    version = pDatabase64->readLatestVersion(version);
+    if(version != 1)
+    {
+        zklog.error("DatabaseKVRemoteTest() failed calling Database64.readLatestVersion()");
+        numberOfFailedTests += 1;
+    }
+
+    //
+    // Check number of failed tests
+    //
+    if(numberOfFailedTests != 0)
+    {
+        zklog.error("DatabaseKVRemoteTest() failed with " + to_string(numberOfFailedTests) + " errors"); 
+        exitProcess();   
+    }
+
+    TimerStopAndLog(DATABASE_KV_REMOTE_TEST);
+    return numberOfFailedTests;
+}
+
+

test/hashdb/database_kv_remote_test.hpp

@@ -0,0 +1,9 @@
+#ifndef DATABASE_KV_REMOTE_TEST_HPP
+#define DATABASE_KV_REMOTE_TEST_HPP
+
+#include <cstdint>
+#include "config.hpp"
+
+uint64_t DatabaseKVRemoteTest (const Config &config);
+
+#endif