From bfa259e9950366900323f07410f11e9266727b65 Mon Sep 17 00:00:00 2001
From: rickb80 <rickborrell@gmail.com>
Date: Mon, 28 Aug 2023 19:14:53 +0200
Subject: [PATCH 01/17] Add creation of kv tables

---
 tools/statedb/create_db.sh | 3 +++
 tools/statedb/delete_db.sh | 0
 2 files changed, 3 insertions(+)
 mode change 100644 => 100755 tools/statedb/delete_db.sh

diff --git a/tools/statedb/create_db.sh b/tools/statedb/create_db.sh
index 2fb9271b4..6a46672f5 100755
--- a/tools/statedb/create_db.sh
+++ b/tools/statedb/create_db.sh
@@ -16,6 +16,9 @@ echo "Creating table state.merkletree..."
 PGPASSWORD=$3 psql -U $2 -h 127.0.0.1 -p 5432 -d $1 -c 'create schema state;'
 PGPASSWORD=$3 psql -U $2 -h 127.0.0.1 -p 5432 -d $1 -c 'create table state.nodes (hash bytea primary key, data bytea not null);'
 PGPASSWORD=$3 psql -U $2 -h 127.0.0.1 -p 5432 -d $1 -c 'create table state.program (hash bytea primary key, data bytea not null);'
+PGPASSWORD=$3 psql -U $2 -h 127.0.0.1 -p 5432 -d $1 -c 'create table state.nodesKV (key bytea, data bytea not null, version integer not null);'
+PGPASSWORD=$3 psql -U $2 -h 127.0.0.1 -p 5432 -d $1 -c 'create table state.versionKV (version integer not null, hash bytea primary key, block integer not null);'
+
 if [ $# == 4 ]
     then
         PGPASSWORD=$3 psql -U $2 -h 127.0.0.1 -p 5432 -d $1 -f $4
diff --git a/tools/statedb/delete_db.sh b/tools/statedb/delete_db.sh
old mode 100644
new mode 100755

From 5aaf99f8833290d6dfcbae538b3e278960419425 Mon Sep 17 00:00:00 2001
From: rickb80 <rickborrell@gmail.com>
Date: Tue, 29 Aug 2023 06:53:11 +0200
Subject: [PATCH 02/17] Add DatabaseVersionsAssociativeCache

---
 src/hashdb/database_associative_cache.hpp     |   1 -
 .../database_versions_associtive_cache_64.cpp | 256 ++++++++++++++++++
 .../database_versions_associtive_cache_64.hpp |  55 ++++
 3 files changed, 311 insertions(+), 1 deletion(-)
 create mode 100644 src/hashdb64/database_versions_associtive_cache_64.cpp
 create mode 100644 src/hashdb64/database_versions_associtive_cache_64.hpp

diff --git a/src/hashdb/database_associative_cache.hpp b/src/hashdb/database_associative_cache.hpp
index 50ff24112..7a7ad3339 100644
--- a/src/hashdb/database_associative_cache.hpp
+++ b/src/hashdb/database_associative_cache.hpp
@@ -8,7 +8,6 @@
 #include "zkmax.hpp"
 
 using namespace std;
-using json = nlohmann::json;
 class DatabaseMTAssociativeCache
 {
     private:
diff --git a/src/hashdb64/database_versions_associtive_cache_64.cpp b/src/hashdb64/database_versions_associtive_cache_64.cpp
new file mode 100644
index 000000000..9dcc84827
--- /dev/null
+++ b/src/hashdb64/database_versions_associtive_cache_64.cpp
@@ -0,0 +1,256 @@
+#include "database_versions_associtive_cache_64.hpp"
+#include <vector>
+#include "goldilocks_base_field.hpp"
+#include <nlohmann/json.hpp>
+#include <mutex>
+#include "zklog.hpp"
+#include "zkmax.hpp"
+#include "exit_process.hpp"
+#include "scalar.hpp"
+
+DatabaseVersionsAssociativeCache::DatabaseVersionsAssociativeCache()
+{
+    log2IndexesSize = 0;
+    indexesSize = 0;
+    log2CacheSize = 0;
+    cacheSize = 0;
+    indexes = NULL;
+    keys = NULL;
+    versionBlock = NULL;
+    currentCacheIndex = 0;
+    attempts = 0;
+    hits = 0;
+    name = "";
+};
+
+DatabaseVersionsAssociativeCache::DatabaseVersionsAssociativeCache(int log2IndexesSize_, int cacheSize_, string name_)
+{
+    postConstruct(log2IndexesSize_, cacheSize_, name_);
+};
+
+DatabaseVersionsAssociativeCache::~DatabaseVersionsAssociativeCache()
+{
+    if (indexes != NULL)
+        delete[] indexes;
+    if (keys != NULL)
+        delete[] keys;
+    if (versionBlock != NULL)
+        delete[] versionBlock;
+
+};
+
+void DatabaseVersionsAssociativeCache::postConstruct(int log2IndexesSize_, int log2CacheSize_, string name_)
+{
+    lock_guard<recursive_mutex> guard(mlock);
+    log2IndexesSize = log2IndexesSize_;
+    if (log2IndexesSize_ > 32)
+    {
+        zklog.error("DatabaseVersionsAssociativeCache::DatabaseVersionsAssociativeCache() log2IndexesSize_ > 32");
+        exitProcess();
+    }
+    indexesSize = 1 << log2IndexesSize;
+
+    log2CacheSize = log2CacheSize_;
+    if (log2CacheSize_ > 32)
+    {
+        zklog.error("DatabaseVersionsAssociativeCache::DatabaseVersionsAssociativeCache() log2CacheSize_ > 32");
+        exitProcess();
+    }
+    cacheSize = 1 << log2CacheSize_;
+
+    if(indexes != NULL) delete[] indexes;
+    indexes = new uint32_t[indexesSize];
+    //initialization of indexes array
+    uint32_t initValue = UINT32_MAX-cacheSize-(uint32_t)1;
+    #pragma omp parallel for schedule(static) num_threads(4)
+    for (size_t i = 0; i < indexesSize; i++)
+    {
+        indexes[i] = initValue;
+    }
+    if(keys != NULL) delete[] keys;
+    keys = new Goldilocks::Element[4 * cacheSize];
+
+    if(versionBlock != NULL) delete[] versionBlock;
+    versionBlock = new uint64_t[2 * cacheSize];
+
+    currentCacheIndex = 0;
+    attempts = 0;
+    hits = 0;
+    name = name_;
+    
+    //masks for fast module, note cache size and indexes size must be power of 2
+    cacheMask = cacheSize - 1;
+    indexesMask = indexesSize - 1;
+};
+
+void DatabaseVersionsAssociativeCache::addKeyVersionBlock(Goldilocks::Element (&key)[4], const uint64_t (&vb)[2], bool update)
+{
+    lock_guard<recursive_mutex> guard(mlock);
+    //
+    // Try to add in one of my 4 slots
+    //
+    for (int i = 0; i < 4; ++i)
+    {
+        uint32_t tableIndex = (uint32_t)(key[i].fe & indexesMask);
+        uint32_t cacheIndexRaw = indexes[tableIndex];
+        uint32_t cacheIndex = cacheIndexRaw & cacheMask;
+        uint32_t cacheIndexKey, cacheIndexValue;
+        bool write = false;
+
+        if (emptyCacheSlot(cacheIndexRaw))
+        {
+            write = true;
+            indexes[tableIndex] = currentCacheIndex;
+            currentCacheIndex = (currentCacheIndex == UINT32_MAX) ? 0 : (currentCacheIndex + 1);
+
+            cacheIndex = indexes[tableIndex] & cacheMask;
+            cacheIndexKey = cacheIndex * 4;
+            cacheIndexValue = cacheIndex * 2;
+        }
+        else
+        {
+            cacheIndexKey = cacheIndex * 4;
+            cacheIndexValue = cacheIndex * 2;
+
+            if (keys[cacheIndexKey + 0].fe == key[0].fe &&
+                keys[cacheIndexKey + 1].fe == key[1].fe &&
+                keys[cacheIndexKey + 2].fe == key[2].fe &&
+                keys[cacheIndexKey + 3].fe == key[3].fe)
+            {
+                write = update;
+            }
+            else
+            {
+                continue;
+            }
+        }
+        if (write) 
+        {
+            keys[cacheIndexKey + 0].fe = key[0].fe;
+            keys[cacheIndexKey + 1].fe = key[1].fe;
+            keys[cacheIndexKey + 2].fe = key[2].fe;
+            keys[cacheIndexKey + 3].fe = key[3].fe;
+            versionBlock[cacheIndexValue + 0] = vb[0];
+            versionBlock[cacheIndexValue + 1] = vb[1];
+           
+            return;
+        }else{
+            return;
+        }
+    }
+    //
+    // forced entry insertion
+    //
+    uint32_t cacheIndex = (uint32_t)(currentCacheIndex & cacheMask);
+    currentCacheIndex = (currentCacheIndex == UINT32_MAX) ? 0 : (currentCacheIndex + 1);
+    uint32_t cacheIndexKey = cacheIndex * 4;
+    uint32_t cacheIndexValue = cacheIndex * 2;
+    keys[cacheIndexKey + 0].fe = key[0].fe;
+    keys[cacheIndexKey + 1].fe = key[1].fe;
+    keys[cacheIndexKey + 2].fe = key[2].fe;
+    keys[cacheIndexKey + 3].fe = key[3].fe;
+    versionBlock[cacheIndexValue + 0] = vb[0];
+    versionBlock[cacheIndexValue + 1] = vb[1];
+    //
+    // Forced index insertion
+    //
+    int iters = 0;
+    uint32_t usedRawCacheIndexes[10];
+    usedRawCacheIndexes[0] = currentCacheIndex-1;
+    forcedInsertion(usedRawCacheIndexes, iters);
+
+}
+
+void DatabaseVersionsAssociativeCache::forcedInsertion(uint32_t (&usedRawCacheIndexes)[10], int &iters)
+{
+    uint32_t inputRawCacheIndex = usedRawCacheIndexes[iters];
+    //
+    // avoid infinite loop
+    //
+    iters++;
+    if (iters > 9)
+    {
+        zklog.error("forcedInsertion() more than 10 iterations required. Index: " + to_string(inputRawCacheIndex));
+        exitProcess();
+    }    
+    //
+    // find a slot into my indexes
+    //
+    Goldilocks::Element *inputKey = &keys[(inputRawCacheIndex & cacheMask) * 4];
+    uint32_t minRawCacheIndex = UINT32_MAX;
+    int pos = -1;
+
+    for (int i = 0; i < 4; ++i)
+    {
+        uint32_t tableIndex_ = (uint32_t)(inputKey[i].fe & indexesMask);
+        uint32_t rawCacheIndex_ = (uint32_t)(indexes[tableIndex_]);
+        if (emptyCacheSlot(rawCacheIndex_))
+        {
+            indexes[tableIndex_] = inputRawCacheIndex;
+            return;
+        }
+        else
+        {
+            //consider minimum not used rawCacheIndex_
+            bool used = false;
+            for(int k=0; k<iters; k++){
+                if(usedRawCacheIndexes[k] == rawCacheIndex_){
+                    used = true;
+                    break;
+                }
+            }
+            if (!used && rawCacheIndex_ < minRawCacheIndex)
+            {
+                minRawCacheIndex = rawCacheIndex_;
+                pos = i;
+            }
+        }
+    }
+
+    if (pos < 0)
+    {
+        zklog.error("forcedInsertion() could not continue the recursion: " + to_string(inputRawCacheIndex));
+        exitProcess();
+    } 
+    indexes[(uint32_t)(inputKey[pos].fe & indexesMask)] = inputRawCacheIndex;
+    usedRawCacheIndexes[iters] = minRawCacheIndex; //new cache element to add in the indexes table
+    forcedInsertion(usedRawCacheIndexes, iters);
+    
+}
+
+bool DatabaseVersionsAssociativeCache::findKey(Goldilocks::Element (&key)[4], uint64_t (&vb)[2])
+{
+    lock_guard<recursive_mutex> guard(mlock);
+    attempts++; 
+    //
+    //  Statistics
+    //
+    if (attempts<<40 == 0)
+    {
+        zklog.info("DatabaseVersionsAssociativeCache::findKey() name=" + name + " indexesSize=" + to_string(indexesSize) + " cacheSize=" + to_string(cacheSize) + " attempts=" + to_string(attempts) + " hits=" + to_string(hits) + " hit ratio=" + to_string(double(hits) * 100.0 / double(zkmax(attempts, 1))) + "%");
+    }
+    //
+    // Find the value
+    //
+    for (int i = 0; i < 4; i++)
+    {
+        uint32_t cacheIndexRaw = indexes[key[i].fe & indexesMask];
+        if (emptyCacheSlot(cacheIndexRaw)) continue;
+        
+        uint32_t cacheIndex = cacheIndexRaw  & cacheMask;
+        uint32_t cacheIndexKey = cacheIndex * 4;
+
+        if (keys[cacheIndexKey + 0].fe == key[0].fe &&
+            keys[cacheIndexKey + 1].fe == key[1].fe &&
+            keys[cacheIndexKey + 2].fe == key[2].fe &&
+            keys[cacheIndexKey + 3].fe == key[3].fe)
+        {
+            uint32_t cacheIndexValue = cacheIndex * 2;
+            ++hits;
+            vb[0] = versionBlock[cacheIndexValue];
+            vb[1] = versionBlock[cacheIndexValue + 1];
+            return true;
+        }
+    }
+    return false;
+}
diff --git a/src/hashdb64/database_versions_associtive_cache_64.hpp b/src/hashdb64/database_versions_associtive_cache_64.hpp
new file mode 100644
index 000000000..86cb3f22b
--- /dev/null
+++ b/src/hashdb64/database_versions_associtive_cache_64.hpp
@@ -0,0 +1,55 @@
+#ifndef DATABASE_VERSIONS_ASSOCIATIVE_CACHE_HPP
+#define DATABASE_VERSIONS_ASSOCIATIVE_CACHE_HPP
+#include <vector>
+#include "goldilocks_base_field.hpp"
+#include <nlohmann/json.hpp>
+#include <mutex>
+#include "zklog.hpp"
+#include "zkmax.hpp"
+
+using namespace std;
+class DatabaseVersionsAssociativeCache
+{
+    private:
+        recursive_mutex mlock;
+
+        int log2IndexesSize;
+        uint32_t indexesSize;
+        int log2CacheSize;
+        uint32_t cacheSize;
+
+        uint32_t *indexes;
+        Goldilocks::Element *keys;
+        uint64_t *versionBlock;
+        uint32_t currentCacheIndex; 
+
+        uint64_t attempts;
+        uint64_t hits;
+        string name;
+
+        uint64_t indexesMask;
+        uint64_t cacheMask;
+
+
+    public:
+
+        DatabaseVersionsAssociativeCache();
+        DatabaseVersionsAssociativeCache(int log2IndexesSize_, int log2CacheSize_, string name_);
+        ~DatabaseVersionsAssociativeCache();
+
+        void postConstruct(int log2IndexesSize_, int log2CacheSize_, string name_);
+        void addKeyVersionBlock(Goldilocks::Element (&key)[4], const uint64_t (&vb)[2], bool update);
+        bool findKey(Goldilocks::Element (&key)[4], uint64_t (&vb)[2]);
+        inline bool enabled() const { return (log2IndexesSize > 0); };
+        inline uint32_t getCacheSize()  const { return cacheSize; };
+        inline uint32_t getIndexesSize() const { return indexesSize; };
+
+    private:
+        inline bool emptyCacheSlot(uint32_t cacheIndexRaw) const { 
+            return (currentCacheIndex >= cacheIndexRaw &&  currentCacheIndex - cacheIndexRaw > cacheSize) ||
+            (currentCacheIndex < cacheIndexRaw && UINT32_MAX - cacheIndexRaw + currentCacheIndex > cacheSize);
+         };
+        void forcedInsertion(uint32_t (&usedRawCacheIndexes)[10], int &iters);
+};
+#endif
+

From 44524e55ce93e97e8223aa2da3f55a9bc69f45d9 Mon Sep 17 00:00:00 2001
From: rickb80 <rickborrell@gmail.com>
Date: Tue, 29 Aug 2023 13:22:27 +0200
Subject: [PATCH 03/17] Temporal commit

---
 .../database_kv_associative_cache_64.cpp      | 284 ++++++++++++++++++
 .../database_kv_associative_cache_64.hpp      |  55 ++++
 .../database_versions_associtive_cache_64.cpp |   2 +-
 .../database_versions_associtive_cache_64.hpp |   4 +-
 4 files changed, 342 insertions(+), 3 deletions(-)
 create mode 100644 src/hashdb64/database_kv_associative_cache_64.cpp
 create mode 100644 src/hashdb64/database_kv_associative_cache_64.hpp

diff --git a/src/hashdb64/database_kv_associative_cache_64.cpp b/src/hashdb64/database_kv_associative_cache_64.cpp
new file mode 100644
index 000000000..d1a101b16
--- /dev/null
+++ b/src/hashdb64/database_kv_associative_cache_64.cpp
@@ -0,0 +1,284 @@
+#include "database_kv_associative_cache.hpp"
+#include <vector>
+#include "goldilocks_base_field.hpp"
+#include <nlohmann/json.hpp>
+#include <mutex>
+#include "zklog.hpp"
+#include "zkmax.hpp"
+#include "exit_process.hpp"
+#include "scalar.hpp"
+
+
+
+
+DatabaseKVAssociativeCache::DatabaseKVAssociativeCache()
+{
+    log2IndexesSize = 0;
+    indexesSize = 0;
+    log2CacheSize = 0;
+    cacheSize = 0;
+    indexes = NULL;
+    keys = NULL;
+    values = NULL;
+    versions = NULL;
+    currentCacheIndex = 0;
+    attempts = 0;
+    hits = 0;
+    name = "";
+};
+
+DatabaseKVAssociativeCache::DatabaseKVAssociativeCache(int log2IndexesSize_, int cacheSize_, string name_)
+{
+    postConstruct(log2IndexesSize_, cacheSize_, name_);
+};
+
+DatabaseKVAssociativeCache::~DatabaseKVAssociativeCache()
+{
+    if (indexes != NULL)
+        delete[] indexes;
+    if (keys != NULL)
+        delete[] keys;
+    if (values != NULL)
+        delete[] values;
+    if (versions != NULL)
+        delete[] versions;
+
+};
+
+void DatabaseKVAssociativeCache::postConstruct(int log2IndexesSize_, int log2CacheSize_, string name_)
+{
+    lock_guard<recursive_mutex> guard(mlock);
+    log2IndexesSize = log2IndexesSize_;
+    if (log2IndexesSize_ > 32)
+    {
+        zklog.error("DatabaseKVAssociativeCache::DatabaseKVAssociativeCache() log2IndexesSize_ > 32");
+        exitProcess();
+    }
+    indexesSize = 1 << log2IndexesSize;
+
+    log2CacheSize = log2CacheSize_;
+    if (log2CacheSize_ > 32)
+    {
+        zklog.error("DatabaseKVAssociativeCache::DatabaseKVAssociativeCache() log2CacheSize_ > 32");
+        exitProcess();
+    }
+    cacheSize = 1 << log2CacheSize_;
+
+    if(indexes != NULL) delete[] indexes;
+    indexes = new uint32_t[indexesSize];
+    //initialization of indexes array
+    uint32_t initValue = UINT32_MAX-cacheSize-(uint32_t)1;
+    #pragma omp parallel for schedule(static) num_threads(4)
+    for (size_t i = 0; i < indexesSize; i++)
+    {
+        indexes[i] = initValue;
+    }
+    if(keys != NULL) delete[] keys;
+    keys = new Goldilocks::Element[4 * cacheSize];
+
+    if(values != NULL) delete[] values;
+    values = new Goldilocks::Element[8 * cacheSize];
+
+    currentCacheIndex = 0;
+    attempts = 0;
+    hits = 0;
+    name = name_;
+    
+    //masks for fast module, note cache size and indexes size must be power of 2
+    cacheMask = cacheSize - 1;
+    indexesMask = indexesSize - 1;
+};
+
+void DatabaseKVAssociativeCache::aaddKeyValueVersion(Goldilocks::Element (&key)[4], const vector<Goldilocks::Element> &value, uint64_t version, bool update)
+{
+    lock_guard<recursive_mutex> guard(mlock);
+    //
+    // Try to add in one of my 4 slots
+    //
+    for (int i = 0; i < 4; ++i)
+    {
+        uint32_t tableIndex = (uint32_t)(key[i].fe & indexesMask);
+        uint32_t cacheIndexRaw = indexes[tableIndex];
+        uint32_t cacheIndex = cacheIndexRaw & cacheMask;
+        uint32_t cacheIndexKey, cacheIndexValue;
+        bool write = false;
+
+        if (emptyCacheSlot(cacheIndexRaw))
+        {
+            write = true;
+            indexes[tableIndex] = currentCacheIndex;
+            currentCacheIndex = (currentCacheIndex == UINT32_MAX) ? 0 : (currentCacheIndex + 1);
+
+            cacheIndex = indexes[tableIndex] & cacheMask;
+            cacheIndexKey = cacheIndex * 4;
+            cacheIndexValue = cacheIndex * 8;
+        }
+        else
+        {
+            cacheIndexKey = cacheIndex * 4;
+            cacheIndexValue = cacheIndex * 8;
+
+            if (keys[cacheIndexKey + 0].fe == key[0].fe &&
+                keys[cacheIndexKey + 1].fe == key[1].fe &&
+                keys[cacheIndexKey + 2].fe == key[2].fe &&
+                keys[cacheIndexKey + 3].fe == key[3].fe)
+            {
+                write = update;
+            }
+            else
+            {
+                continue;
+            }
+        }
+        if (write) 
+        {
+            keys[cacheIndexKey + 0].fe = key[0].fe;
+            keys[cacheIndexKey + 1].fe = key[1].fe;
+            keys[cacheIndexKey + 2].fe = key[2].fe;
+            keys[cacheIndexKey + 3].fe = key[3].fe;
+            values[cacheIndexValue + 0] = value[0];
+            values[cacheIndexValue + 1] = value[1];
+            values[cacheIndexValue + 2] = value[2];
+            values[cacheIndexValue + 3] = value[3];
+            values[cacheIndexValue + 4] = value[4];
+            values[cacheIndexValue + 5] = value[5];
+            values[cacheIndexValue + 6] = value[6];
+            values[cacheIndexValue + 7] = value[7];
+            return;
+        }else{
+            return;
+        }
+    }
+    //
+    // forced entry insertion
+    //
+    uint32_t cacheIndex = (uint32_t)(currentCacheIndex & cacheMask);
+    currentCacheIndex = (currentCacheIndex == UINT32_MAX) ? 0 : (currentCacheIndex + 1);
+    uint32_t cacheIndexKey = cacheIndex * 4;
+    uint32_t cacheIndexValue = cacheIndex * 8;
+    keys[cacheIndexKey + 0].fe = key[0].fe;
+    keys[cacheIndexKey + 1].fe = key[1].fe;
+    keys[cacheIndexKey + 2].fe = key[2].fe;
+    keys[cacheIndexKey + 3].fe = key[3].fe;
+    values[cacheIndexValue + 0] = value[0];
+    values[cacheIndexValue + 1] = value[1];
+    values[cacheIndexValue + 2] = value[2];
+    values[cacheIndexValue + 3] = value[3];
+    values[cacheIndexValue + 4] = value[4];
+    values[cacheIndexValue + 5] = value[5];
+    values[cacheIndexValue + 6] = value[6];
+    values[cacheIndexValue + 7] = value[7];
+    //
+    // Forced index insertion
+    //
+    int iters = 0;
+    uint32_t usedRawCacheIndexes[10];
+    usedRawCacheIndexes[0] = currentCacheIndex-1;
+    forcedInsertion(usedRawCacheIndexes, iters);
+
+}
+
+void DatabaseKVAssociativeCache::forcedInsertion(uint32_t (&usedRawCacheIndexes)[10], int &iters)
+{
+    uint32_t inputRawCacheIndex = usedRawCacheIndexes[iters];
+    //
+    // avoid infinite loop
+    //
+    iters++;
+    if (iters > 9)
+    {
+        zklog.error("forcedInsertion() more than 10 iterations required. Index: " + to_string(inputRawCacheIndex));
+        exitProcess();
+    }    
+    //
+    // find a slot into my indexes
+    //
+    Goldilocks::Element *inputKey = &keys[(inputRawCacheIndex & cacheMask) * 4];
+    uint32_t minRawCacheIndex = UINT32_MAX;
+    int pos = -1;
+
+    for (int i = 0; i < 4; ++i)
+    {
+        uint32_t tableIndex_ = (uint32_t)(inputKey[i].fe & indexesMask);
+        uint32_t rawCacheIndex_ = (uint32_t)(indexes[tableIndex_]);
+        if (emptyCacheSlot(rawCacheIndex_))
+        {
+            indexes[tableIndex_] = inputRawCacheIndex;
+            return;
+        }
+        else
+        {
+            //consider minimum not used rawCacheIndex_
+            bool used = false;
+            for(int k=0; k<iters; k++){
+                if(usedRawCacheIndexes[k] == rawCacheIndex_){
+                    used = true;
+                    break;
+                }
+            }
+            if (!used && rawCacheIndex_ < minRawCacheIndex)
+            {
+                minRawCacheIndex = rawCacheIndex_;
+                pos = i;
+            }
+        }
+    }
+
+    if (pos < 0)
+    {
+        zklog.error("forcedInsertion() could not continue the recursion: " + to_string(inputRawCacheIndex));
+        exitProcess();
+    } 
+    indexes[(uint32_t)(inputKey[pos].fe & indexesMask)] = inputRawCacheIndex;
+    usedRawCacheIndexes[iters] = minRawCacheIndex; //new cache element to add in the indexes table
+    forcedInsertion(usedRawCacheIndexes, iters);
+    
+}
+
+bool DatabaseKVAssociativeCache::findKey(const Goldilocks::Element (&key)[4], vector<Goldilocks::Element> &value)
+{
+    lock_guard<recursive_mutex> guard(mlock);
+    attempts++; 
+    //
+    //  Statistics
+    //
+    if (attempts<<40 == 0)
+    {
+        zklog.info("DatabaseKVAssociativeCache::findKey() name=" + name + " indexesSize=" + to_string(indexesSize) + " cacheSize=" + to_string(cacheSize) + " attempts=" + to_string(attempts) + " hits=" + to_string(hits) + " hit ratio=" + to_string(double(hits) * 100.0 / double(zkmax(attempts, 1))) + "%");
+    }
+    //
+    // Find the value
+    //
+    for (int i = 0; i < 4; i++)
+    {
+        uint32_t cacheIndexRaw = indexes[key[i].fe & indexesMask];
+        if (emptyCacheSlot(cacheIndexRaw)) continue;
+        
+        uint32_t cacheIndex = cacheIndexRaw  & cacheMask;
+        uint32_t cacheIndexKey = cacheIndex * 4;
+
+        if (keys[cacheIndexKey + 0].fe == key[0].fe &&
+            keys[cacheIndexKey + 1].fe == key[1].fe &&
+            keys[cacheIndexKey + 2].fe == key[2].fe &&
+            keys[cacheIndexKey + 3].fe == key[3].fe)
+        {
+            uint32_t cacheIndexValue = cacheIndex * 8;
+            ++hits;
+            value.resize(8);
+            value[0] = values[cacheIndexValue];
+            value[1] = values[cacheIndexValue + 1];
+            value[2] = values[cacheIndexValue + 2];
+            value[3] = values[cacheIndexValue + 3];
+            value[4] = values[cacheIndexValue + 4];
+            value[5] = values[cacheIndexValue + 5];
+            value[6] = values[cacheIndexValue + 6];
+            value[7] = values[cacheIndexValue + 7];
+            value[8] = values[cacheIndexValue + 8];
+            value[9] = values[cacheIndexValue + 9];
+            value[10] = values[cacheIndexValue + 10];
+            value[11] = values[cacheIndexValue + 11];
+            return true;
+        }
+    }
+    return false;
+}
diff --git a/src/hashdb64/database_kv_associative_cache_64.hpp b/src/hashdb64/database_kv_associative_cache_64.hpp
new file mode 100644
index 000000000..5f5183e70
--- /dev/null
+++ b/src/hashdb64/database_kv_associative_cache_64.hpp
@@ -0,0 +1,55 @@
+#ifndef DATABASE_KV_ASSOCIATIVE_CACHE_HPP
+#define DATABASE_KV_ASSOCIATIVE_CACHE_HPP
+#include <vector>
+#include "goldilocks_base_field.hpp"
+#include <nlohmann/json.hpp>
+#include <mutex>
+#include "zklog.hpp"
+#include "zkmax.hpp"
+
+using namespace std;
+class DatabaseKVAssociativeCache
+{
+    private:
+        recursive_mutex mlock;
+
+        int log2IndexesSize;
+        uint32_t indexesSize;
+        int log2CacheSize;
+        uint32_t cacheSize;
+
+        uint32_t *indexes;
+        Goldilocks::Element *keys;
+        Goldilocks::Element *values;
+        uint64_t * versions;
+        uint32_t currentCacheIndex; 
+
+        uint64_t attempts;
+        uint64_t hits;
+        string name;
+
+        uint64_t indexesMask;
+        uint64_t cacheMask;
+
+
+    public:
+
+        DatabaseKVAssociativeCache();
+        DatabaseKVAssociativeCache(int log2IndexesSize_, int log2CacheSize_, string name_);
+        ~DatabaseKVAssociativeCache();
+
+        void postConstruct(int log2IndexesSize_, int log2CacheSize_, string name_);
+        void addKeyValueVersion(Goldilocks::Element (&key)[4], const vector<Goldilocks::Element> &value, uint64_t version, bool update);
+        bool findKey(const Goldilocks::Element (&key)[4], vector<Goldilocks::Element> &value, bool last = true, uint64_t version=0);
+        inline bool enabled() const { return (log2IndexesSize > 0); };
+        inline uint32_t getCacheSize()  const { return cacheSize; };
+        inline uint32_t getIndexesSize() const { return indexesSize; };
+
+    private:
+        inline bool emptyCacheSlot(uint32_t cacheIndexRaw) const { 
+            return (currentCacheIndex >= cacheIndexRaw &&  currentCacheIndex - cacheIndexRaw > cacheSize) ||
+            (currentCacheIndex < cacheIndexRaw && UINT32_MAX - cacheIndexRaw + currentCacheIndex > cacheSize);
+         };
+        void forcedInsertion(uint32_t (&usedRawCacheIndexes)[10], int &iters);
+};
+#endif
\ No newline at end of file
diff --git a/src/hashdb64/database_versions_associtive_cache_64.cpp b/src/hashdb64/database_versions_associtive_cache_64.cpp
index 9dcc84827..2ebac9324 100644
--- a/src/hashdb64/database_versions_associtive_cache_64.cpp
+++ b/src/hashdb64/database_versions_associtive_cache_64.cpp
@@ -218,7 +218,7 @@ void DatabaseVersionsAssociativeCache::forcedInsertion(uint32_t (&usedRawCacheIn
     
 }
 
-bool DatabaseVersionsAssociativeCache::findKey(Goldilocks::Element (&key)[4], uint64_t (&vb)[2])
+bool DatabaseVersionsAssociativeCache::findKey(const Goldilocks::Element (&key)[4], uint64_t (&vb)[2])
 {
     lock_guard<recursive_mutex> guard(mlock);
     attempts++; 
diff --git a/src/hashdb64/database_versions_associtive_cache_64.hpp b/src/hashdb64/database_versions_associtive_cache_64.hpp
index 86cb3f22b..2faabfd5f 100644
--- a/src/hashdb64/database_versions_associtive_cache_64.hpp
+++ b/src/hashdb64/database_versions_associtive_cache_64.hpp
@@ -38,8 +38,8 @@ class DatabaseVersionsAssociativeCache
         ~DatabaseVersionsAssociativeCache();
 
         void postConstruct(int log2IndexesSize_, int log2CacheSize_, string name_);
-        void addKeyVersionBlock(Goldilocks::Element (&key)[4], const uint64_t (&vb)[2], bool update);
-        bool findKey(Goldilocks::Element (&key)[4], uint64_t (&vb)[2]);
+        void addKeyVersionBlock(Goldilocks::Element (&key)[4], const uint64_t (&vb)[2], const bool update);
+        bool findKey(const Goldilocks::Element (&key)[4], uint64_t (&vb)[2]);
         inline bool enabled() const { return (log2IndexesSize > 0); };
         inline uint32_t getCacheSize()  const { return cacheSize; };
         inline uint32_t getIndexesSize() const { return indexesSize; };

From 0928b5b256e440b549a521b2862dbd790d505bdc Mon Sep 17 00:00:00 2001
From: rickb80 <rickborrell@gmail.com>
Date: Tue, 29 Aug 2023 14:52:58 +0200
Subject: [PATCH 04/17] Avoid insert repeated keys in associative cache

---
 src/hashdb/database_associative_cache.cpp | 116 ++++++++++------------
 src/hashdb/database_associative_cache.hpp |   2 +-
 src/hashdb/database_cache.cpp             |  11 +-
 3 files changed, 60 insertions(+), 69 deletions(-)

diff --git a/src/hashdb/database_associative_cache.cpp b/src/hashdb/database_associative_cache.cpp
index 01216c8e1..13631abb3 100644
--- a/src/hashdb/database_associative_cache.cpp
+++ b/src/hashdb/database_associative_cache.cpp
@@ -89,82 +89,70 @@ void DatabaseMTAssociativeCache::postConstruct(int log2IndexesSize_, int log2Cac
 void DatabaseMTAssociativeCache::addKeyValue(Goldilocks::Element (&key)[4], const vector<Goldilocks::Element> &value, bool update)
 {
     lock_guard<recursive_mutex> guard(mlock);
+    bool force = true;
+    uint32_t cacheIndex;
+
     //
-    // Try to add in one of my 4 slots
+    // Check if present in one of the four slots
     //
     for (int i = 0; i < 4; ++i)
     {
         uint32_t tableIndex = (uint32_t)(key[i].fe & indexesMask);
         uint32_t cacheIndexRaw = indexes[tableIndex];
-        uint32_t cacheIndex = cacheIndexRaw & cacheMask;
-        uint32_t cacheIndexKey, cacheIndexValue;
-        bool write = false;
+        cacheIndex = cacheIndexRaw & cacheMask;
+        uint32_t cacheIndexKey_ = cacheIndex * 4;
 
-        if (emptyCacheSlot(cacheIndexRaw))
+        if (!emptyCacheSlot(cacheIndexRaw) &&
+            keys[cacheIndexKey_ + 0].fe == key[0].fe &&
+            keys[cacheIndexKey_ + 1].fe == key[1].fe &&
+            keys[cacheIndexKey_ + 2].fe == key[2].fe &&
+            keys[cacheIndexKey_ + 3].fe == key[3].fe)
         {
-            write = true;
-            indexes[tableIndex] = currentCacheIndex;
-            currentCacheIndex = (currentCacheIndex == UINT32_MAX) ? 0 : (currentCacheIndex + 1);
-
-            cacheIndex = indexes[tableIndex] & cacheMask;
-            cacheIndexKey = cacheIndex * 4;
-            cacheIndexValue = cacheIndex * 12;
+            if(update == false) return;
+            force = false;
+            break;
         }
         else
         {
-            cacheIndexKey = cacheIndex * 4;
-            cacheIndexValue = cacheIndex * 12;
-
-            if (keys[cacheIndexKey + 0].fe == key[0].fe &&
-                keys[cacheIndexKey + 1].fe == key[1].fe &&
-                keys[cacheIndexKey + 2].fe == key[2].fe &&
-                keys[cacheIndexKey + 3].fe == key[3].fe)
-            {
-                write = update;
-            }
-            else
-            {
-                continue;
-            }
+            continue;
         }
-        if (write) 
+    }
+
+    //
+    // Try to add in one empty slot
+    //
+    if(force == true){
+        for (int i = 0; i < 4; ++i)
         {
-            keys[cacheIndexKey + 0].fe = key[0].fe;
-            keys[cacheIndexKey + 1].fe = key[1].fe;
-            keys[cacheIndexKey + 2].fe = key[2].fe;
-            keys[cacheIndexKey + 3].fe = key[3].fe;
-            values[cacheIndexValue + 0] = value[0];
-            values[cacheIndexValue + 1] = value[1];
-            values[cacheIndexValue + 2] = value[2];
-            values[cacheIndexValue + 3] = value[3];
-            values[cacheIndexValue + 4] = value[4];
-            values[cacheIndexValue + 5] = value[5];
-            values[cacheIndexValue + 6] = value[6];
-            values[cacheIndexValue + 7] = value[7];
-            if (value.size() > 8)
+            uint32_t tableIndex = (uint32_t)(key[i].fe & indexesMask);
+            uint32_t cacheIndexRaw = indexes[tableIndex];
+            cacheIndex = cacheIndexRaw & cacheMask;
+
+            if (emptyCacheSlot(cacheIndexRaw))
             {
-                values[cacheIndexValue + 8] = value[8];
-                values[cacheIndexValue + 9] = value[9];
-                values[cacheIndexValue + 10] = value[10];
-                values[cacheIndexValue + 11] = value[11];
-            }else{
-                values[cacheIndexValue + 8] = Goldilocks::zero();
-                values[cacheIndexValue + 9] = Goldilocks::zero();
-                values[cacheIndexValue + 10] = Goldilocks::zero();
-                values[cacheIndexValue + 11] = Goldilocks::zero();
+                force = false;
+                indexes[tableIndex] = currentCacheIndex;
+                currentCacheIndex = (currentCacheIndex == UINT32_MAX) ? 0 : (currentCacheIndex + 1);
+                cacheIndex = indexes[tableIndex] & cacheMask;
+                break;
             }
-            return;
-        }else{
-            return;
         }
     }
+
     //
-    // forced entry insertion
+    // Evaluate cacheIndexKey and 
+    //
+    uint64_t cacheIndexKey, cacheIndexValue;
+    if(force == true){
+        cacheIndex = (uint32_t)(currentCacheIndex & cacheMask);
+        currentCacheIndex = (currentCacheIndex == UINT32_MAX) ? 0 : (currentCacheIndex + 1);
+    }
+    cacheIndexKey = cacheIndex * 4;
+    cacheIndexValue = cacheIndex * 12;
+    
+    //
+    // Add value
     //
-    uint32_t cacheIndex = (uint32_t)(currentCacheIndex & cacheMask);
-    currentCacheIndex = (currentCacheIndex == UINT32_MAX) ? 0 : (currentCacheIndex + 1);
-    uint32_t cacheIndexKey = cacheIndex * 4;
-    uint32_t cacheIndexValue = cacheIndex * 12;
     keys[cacheIndexKey + 0].fe = key[0].fe;
     keys[cacheIndexKey + 1].fe = key[1].fe;
     keys[cacheIndexKey + 2].fe = key[2].fe;
@@ -189,14 +177,16 @@ void DatabaseMTAssociativeCache::addKeyValue(Goldilocks::Element (&key)[4], cons
         values[cacheIndexValue + 10] = Goldilocks::zero();
         values[cacheIndexValue + 11] = Goldilocks::zero();
     }
+            
     //
     // Forced index insertion
     //
-    int iters = 0;
-    uint32_t usedRawCacheIndexes[10];
-    usedRawCacheIndexes[0] = currentCacheIndex-1;
-    forcedInsertion(usedRawCacheIndexes, iters);
-
+    if(force == true){
+        int iters = 0;
+        uint32_t usedRawCacheIndexes[10];
+        usedRawCacheIndexes[0] = currentCacheIndex-1;
+        forcedInsertion(usedRawCacheIndexes, iters);
+    }
 }
 
 void DatabaseMTAssociativeCache::forcedInsertion(uint32_t (&usedRawCacheIndexes)[10], int &iters)
@@ -256,7 +246,7 @@ void DatabaseMTAssociativeCache::forcedInsertion(uint32_t (&usedRawCacheIndexes)
     
 }
 
-bool DatabaseMTAssociativeCache::findKey(Goldilocks::Element (&key)[4], vector<Goldilocks::Element> &value)
+bool DatabaseMTAssociativeCache::findKey(const Goldilocks::Element (&key)[4], vector<Goldilocks::Element> &value)
 {
     lock_guard<recursive_mutex> guard(mlock);
     attempts++; 
diff --git a/src/hashdb/database_associative_cache.hpp b/src/hashdb/database_associative_cache.hpp
index 50ff24112..b93ccf07f 100644
--- a/src/hashdb/database_associative_cache.hpp
+++ b/src/hashdb/database_associative_cache.hpp
@@ -40,7 +40,7 @@ class DatabaseMTAssociativeCache
 
         void postConstruct(int log2IndexesSize_, int log2CacheSize_, string name_);
         void addKeyValue(Goldilocks::Element (&key)[4], const vector<Goldilocks::Element> &value, bool update);
-        bool findKey(Goldilocks::Element (&key)[4], vector<Goldilocks::Element> &value);
+        bool findKey(const Goldilocks::Element (&key)[4], vector<Goldilocks::Element> &value);
         inline bool enabled() const { return (log2IndexesSize > 0); };
         inline uint32_t getCacheSize()  const { return cacheSize; };
         inline uint32_t getIndexesSize() const { return indexesSize; };
diff --git a/src/hashdb/database_cache.cpp b/src/hashdb/database_cache.cpp
index f72b4416b..fd6c071ea 100644
--- a/src/hashdb/database_cache.cpp
+++ b/src/hashdb/database_cache.cpp
@@ -15,11 +15,6 @@ bool DatabaseCache::addKeyValue(const string &key, const void * value, const boo
         return true;
     }
 
-    if (attempts%1000000 == 0)
-    {
-        zklog.info("DatabaseCache::addKeyValue() name=" + name + " count=" + to_string(cacheMap.size()) + " maxSize=" + to_string(maxSize) + " currentSize=" + to_string(currentSize) + " attempts=" + to_string(attempts) + " hits=" + to_string(hits) + " hit ratio=" + to_string(double(hits)*100.0/double(zkmax(attempts,1))) + "%");
-    }
-
     DatabaseCacheRecord * record;
     // If key already exists in the cache return. The findKey also sets the record in the head of the cache
     if (findKey(key, record))
@@ -84,6 +79,12 @@ bool DatabaseCache::addKeyValue(const string &key, const void * value, const boo
 bool DatabaseCache::findKey(const string &key, DatabaseCacheRecord* &record) 
 {
     attempts++;
+
+    if (attempts%1000000 == 0)
+    {
+        zklog.info("DatabaseCache::addKeyValue() name=" + name + " count=" + to_string(cacheMap.size()) + " maxSize=" + to_string(maxSize) + " currentSize=" + to_string(currentSize) + " attempts=" + to_string(attempts) + " hits=" + to_string(hits) + " hit ratio=" + to_string(double(hits)*100.0/double(zkmax(attempts,1))) + "%");
+    }
+    
     unordered_map<string, DatabaseCacheRecord*>::iterator it = cacheMap.find(key);
 
     if (it != cacheMap.end())

From e0063ca5833898bc5f660231f813c5cf305757b0 Mon Sep 17 00:00:00 2001
From: rickb80 <rickborrell@gmail.com>
Date: Tue, 29 Aug 2023 16:19:57 +0200
Subject: [PATCH 05/17] Code comments

---
 src/hashdb/database_associative_cache.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/hashdb/database_associative_cache.cpp b/src/hashdb/database_associative_cache.cpp
index 13631abb3..9b734a94a 100644
--- a/src/hashdb/database_associative_cache.cpp
+++ b/src/hashdb/database_associative_cache.cpp
@@ -119,7 +119,7 @@ void DatabaseMTAssociativeCache::addKeyValue(Goldilocks::Element (&key)[4], cons
     }
 
     //
-    // Try to add in one empty slot
+    // If exists, add it in one empty slot
     //
     if(force == true){
         for (int i = 0; i < 4; ++i)

From 0a0d26cec22c8e75802834efdf16510d74c1ea21 Mon Sep 17 00:00:00 2001
From: rickb80 <rickborrell@gmail.com>
Date: Tue, 29 Aug 2023 16:58:06 +0200
Subject: [PATCH 06/17] Optimization of new implementation

---
 src/hashdb/database_associative_cache.cpp | 59 ++++++++---------------
 1 file changed, 21 insertions(+), 38 deletions(-)

diff --git a/src/hashdb/database_associative_cache.cpp b/src/hashdb/database_associative_cache.cpp
index 9b734a94a..540fd9823 100644
--- a/src/hashdb/database_associative_cache.cpp
+++ b/src/hashdb/database_associative_cache.cpp
@@ -89,8 +89,10 @@ void DatabaseMTAssociativeCache::postConstruct(int log2IndexesSize_, int log2Cac
 void DatabaseMTAssociativeCache::addKeyValue(Goldilocks::Element (&key)[4], const vector<Goldilocks::Element> &value, bool update)
 {
     lock_guard<recursive_mutex> guard(mlock);
-    bool force = true;
+    bool emptySlot = false;
+    bool present = false;
     uint32_t cacheIndex;
+    uint32_t tableIndexEmpty=0;
 
     //
     // Check if present in one of the four slots
@@ -100,53 +102,34 @@ void DatabaseMTAssociativeCache::addKeyValue(Goldilocks::Element (&key)[4], cons
         uint32_t tableIndex = (uint32_t)(key[i].fe & indexesMask);
         uint32_t cacheIndexRaw = indexes[tableIndex];
         cacheIndex = cacheIndexRaw & cacheMask;
-        uint32_t cacheIndexKey_ = cacheIndex * 4;
-
-        if (!emptyCacheSlot(cacheIndexRaw) &&
-            keys[cacheIndexKey_ + 0].fe == key[0].fe &&
-            keys[cacheIndexKey_ + 1].fe == key[1].fe &&
-            keys[cacheIndexKey_ + 2].fe == key[2].fe &&
-            keys[cacheIndexKey_ + 3].fe == key[3].fe)
-        {
-            if(update == false) return;
-            force = false;
-            break;
-        }
-        else
-        {
-            continue;
-        }
-    }
-
-    //
-    // If exists, add it in one empty slot
-    //
-    if(force == true){
-        for (int i = 0; i < 4; ++i)
-        {
-            uint32_t tableIndex = (uint32_t)(key[i].fe & indexesMask);
-            uint32_t cacheIndexRaw = indexes[tableIndex];
-            cacheIndex = cacheIndexRaw & cacheMask;
+        uint32_t cacheIndexKey = cacheIndex * 4;
 
-            if (emptyCacheSlot(cacheIndexRaw))
-            {
-                force = false;
-                indexes[tableIndex] = currentCacheIndex;
-                currentCacheIndex = (currentCacheIndex == UINT32_MAX) ? 0 : (currentCacheIndex + 1);
-                cacheIndex = indexes[tableIndex] & cacheMask;
-                break;
+        if (!emptyCacheSlot(cacheIndexRaw)){
+            if( keys[cacheIndexKey + 0].fe == key[0].fe &&
+                keys[cacheIndexKey + 1].fe == key[1].fe &&
+                keys[cacheIndexKey + 2].fe == key[2].fe &&
+                keys[cacheIndexKey + 3].fe == key[3].fe){
+                    if(update == false) return;
+                    present = true;
+                    break;
             }
+        }else if (emptySlot == false){
+            emptySlot = true;
+            tableIndexEmpty = tableIndex;
         }
     }
 
     //
     // Evaluate cacheIndexKey and 
     //
-    uint64_t cacheIndexKey, cacheIndexValue;
-    if(force == true){
+    if(!present){
+        if(emptySlot == true){
+            indexes[tableIndexEmpty] = currentCacheIndex;
+        }
         cacheIndex = (uint32_t)(currentCacheIndex & cacheMask);
         currentCacheIndex = (currentCacheIndex == UINT32_MAX) ? 0 : (currentCacheIndex + 1);
     }
+    uint64_t cacheIndexKey, cacheIndexValue;
     cacheIndexKey = cacheIndex * 4;
     cacheIndexValue = cacheIndex * 12;
     
@@ -181,7 +164,7 @@ void DatabaseMTAssociativeCache::addKeyValue(Goldilocks::Element (&key)[4], cons
     //
     // Forced index insertion
     //
-    if(force == true){
+    if(!present && !emptySlot){
         int iters = 0;
         uint32_t usedRawCacheIndexes[10];
         usedRawCacheIndexes[0] = currentCacheIndex-1;

From 03d16b4c819953c6e68588a84db0e37d899d1ef4 Mon Sep 17 00:00:00 2001
From: rickb80 <rickborrell@gmail.com>
Date: Tue, 29 Aug 2023 20:24:43 +0200
Subject: [PATCH 07/17] Finished kv and versions associative caches

---
 .../database_kv_associative_cache_64.cpp      | 179 ++++++++++--------
 .../database_kv_associative_cache_64.hpp      |  12 +-
 .../database_versions_associtive_cache_64.cpp | 110 +++++------
 .../database_versions_associtive_cache_64.hpp |  10 +-
 4 files changed, 159 insertions(+), 152 deletions(-)

diff --git a/src/hashdb64/database_kv_associative_cache_64.cpp b/src/hashdb64/database_kv_associative_cache_64.cpp
index d1a101b16..355e7777d 100644
--- a/src/hashdb64/database_kv_associative_cache_64.cpp
+++ b/src/hashdb64/database_kv_associative_cache_64.cpp
@@ -1,4 +1,4 @@
-#include "database_kv_associative_cache.hpp"
+#include "database_kv_associative_cache_64.hpp"
 #include <vector>
 #include "goldilocks_base_field.hpp"
 #include <nlohmann/json.hpp>
@@ -8,15 +8,13 @@
 #include "exit_process.hpp"
 #include "scalar.hpp"
 
-
-
-
 DatabaseKVAssociativeCache::DatabaseKVAssociativeCache()
 {
     log2IndexesSize = 0;
     indexesSize = 0;
     log2CacheSize = 0;
     cacheSize = 0;
+    maxVersions = 100; //rick: as parameter
     indexes = NULL;
     keys = NULL;
     values = NULL;
@@ -63,6 +61,8 @@ void DatabaseKVAssociativeCache::postConstruct(int log2IndexesSize_, int log2Cac
         exitProcess();
     }
     cacheSize = 1 << log2CacheSize_;
+    maxVersions = 100; //rick: as parameter
+
 
     if(indexes != NULL) delete[] indexes;
     indexes = new uint32_t[indexesSize];
@@ -79,6 +79,9 @@ void DatabaseKVAssociativeCache::postConstruct(int log2IndexesSize_, int log2Cac
     if(values != NULL) delete[] values;
     values = new Goldilocks::Element[8 * cacheSize];
 
+    if(versions != NULL) delete[] versions;
+    versions = new uint64_t[2 * cacheSize];
+
     currentCacheIndex = 0;
     attempts = 0;
     hits = 0;
@@ -89,73 +92,65 @@ void DatabaseKVAssociativeCache::postConstruct(int log2IndexesSize_, int log2Cac
     indexesMask = indexesSize - 1;
 };
 
-void DatabaseKVAssociativeCache::aaddKeyValueVersion(Goldilocks::Element (&key)[4], const vector<Goldilocks::Element> &value, uint64_t version, bool update)
-{
+void DatabaseKVAssociativeCache::addKeyValueVersion(Goldilocks::Element (&key)[4], const vector<Goldilocks::Element> &value, uint64_t version, bool update){
+    
     lock_guard<recursive_mutex> guard(mlock);
+    bool emptySlot = false;
+    bool present = false;
+    bool presentSameVersion = false;
+    uint32_t cacheIndex;
+    uint32_t tableIndexEmpty=0;
+    uint32_t cacheIndexPrev;
+
+
     //
-    // Try to add in one of my 4 slots
+    // Check if present in one of the four slots
     //
     for (int i = 0; i < 4; ++i)
     {
         uint32_t tableIndex = (uint32_t)(key[i].fe & indexesMask);
         uint32_t cacheIndexRaw = indexes[tableIndex];
-        uint32_t cacheIndex = cacheIndexRaw & cacheMask;
-        uint32_t cacheIndexKey, cacheIndexValue;
-        bool write = false;
-
-        if (emptyCacheSlot(cacheIndexRaw))
-        {
-            write = true;
-            indexes[tableIndex] = currentCacheIndex;
-            currentCacheIndex = (currentCacheIndex == UINT32_MAX) ? 0 : (currentCacheIndex + 1);
-
-            cacheIndex = indexes[tableIndex] & cacheMask;
-            cacheIndexKey = cacheIndex * 4;
-            cacheIndexValue = cacheIndex * 8;
-        }
-        else
-        {
-            cacheIndexKey = cacheIndex * 4;
-            cacheIndexValue = cacheIndex * 8;
+        cacheIndex = cacheIndexRaw & cacheMask;
+        uint32_t cacheIndexKey = cacheIndex * 4;
+        uint32_t cacheIndexVersions = cacheIndex * 2;
 
-            if (keys[cacheIndexKey + 0].fe == key[0].fe &&
+        if (!emptyCacheSlot(cacheIndexRaw)){
+            if( keys[cacheIndexKey + 0].fe == key[0].fe &&
                 keys[cacheIndexKey + 1].fe == key[1].fe &&
                 keys[cacheIndexKey + 2].fe == key[2].fe &&
-                keys[cacheIndexKey + 3].fe == key[3].fe)
-            {
-                write = update;
-            }
-            else
-            {
-                continue;
+                keys[cacheIndexKey + 3].fe == key[3].fe){
+                    present = true;
+                    if(versions[cacheIndexVersions] == version){
+                        presentSameVersion = true;
+                        if(update == false) return;
+                    }
+                    cacheIndexPrev = cacheIndex;
+                    break;
             }
+        }else if (emptySlot == false){
+            emptySlot = true;
+            tableIndexEmpty = tableIndex;
         }
-        if (write) 
-        {
-            keys[cacheIndexKey + 0].fe = key[0].fe;
-            keys[cacheIndexKey + 1].fe = key[1].fe;
-            keys[cacheIndexKey + 2].fe = key[2].fe;
-            keys[cacheIndexKey + 3].fe = key[3].fe;
-            values[cacheIndexValue + 0] = value[0];
-            values[cacheIndexValue + 1] = value[1];
-            values[cacheIndexValue + 2] = value[2];
-            values[cacheIndexValue + 3] = value[3];
-            values[cacheIndexValue + 4] = value[4];
-            values[cacheIndexValue + 5] = value[5];
-            values[cacheIndexValue + 6] = value[6];
-            values[cacheIndexValue + 7] = value[7];
-            return;
-        }else{
-            return;
+    }
+
+    //
+    // Evaluate cacheIndexKey and 
+    //
+    if(!presentSameVersion){
+        if(emptySlot == true){
+            indexes[tableIndexEmpty] = currentCacheIndex;
         }
+        cacheIndex = (uint32_t)(currentCacheIndex & cacheMask);
+        currentCacheIndex = (currentCacheIndex == UINT32_MAX) ? 0 : (currentCacheIndex + 1);
     }
+    uint64_t cacheIndexKey, cacheIndexValue, cacheIndexVersions;
+    cacheIndexKey = cacheIndex * 4;
+    cacheIndexValue = cacheIndex * 8;
+    cacheIndexVersions = cacheIndex * 2;
+    
     //
-    // forced entry insertion
+    // Add value
     //
-    uint32_t cacheIndex = (uint32_t)(currentCacheIndex & cacheMask);
-    currentCacheIndex = (currentCacheIndex == UINT32_MAX) ? 0 : (currentCacheIndex + 1);
-    uint32_t cacheIndexKey = cacheIndex * 4;
-    uint32_t cacheIndexValue = cacheIndex * 8;
     keys[cacheIndexKey + 0].fe = key[0].fe;
     keys[cacheIndexKey + 1].fe = key[1].fe;
     keys[cacheIndexKey + 2].fe = key[2].fe;
@@ -168,14 +163,19 @@ void DatabaseKVAssociativeCache::aaddKeyValueVersion(Goldilocks::Element (&key)[
     values[cacheIndexValue + 5] = value[5];
     values[cacheIndexValue + 6] = value[6];
     values[cacheIndexValue + 7] = value[7];
+    versions[cacheIndexVersions] = version;
+    if(present & !presentSameVersion){
+        versions[cacheIndexVersions+1] = cacheIndexPrev;
+    }
     //
     // Forced index insertion
     //
-    int iters = 0;
-    uint32_t usedRawCacheIndexes[10];
-    usedRawCacheIndexes[0] = currentCacheIndex-1;
-    forcedInsertion(usedRawCacheIndexes, iters);
-
+    if(!present && !emptySlot){
+        int iters = 0;
+        uint32_t usedRawCacheIndexes[10];
+        usedRawCacheIndexes[0] = currentCacheIndex-1;
+        forcedInsertion(usedRawCacheIndexes, iters);
+    }
 }
 
 void DatabaseKVAssociativeCache::forcedInsertion(uint32_t (&usedRawCacheIndexes)[10], int &iters)
@@ -235,7 +235,7 @@ void DatabaseKVAssociativeCache::forcedInsertion(uint32_t (&usedRawCacheIndexes)
     
 }
 
-bool DatabaseKVAssociativeCache::findKey(const Goldilocks::Element (&key)[4], vector<Goldilocks::Element> &value)
+bool DatabaseKVAssociativeCache::findKey(const Goldilocks::Element (&key)[4], vector<Goldilocks::Element> &value, const bool last, const uint64_t version)
 {
     lock_guard<recursive_mutex> guard(mlock);
     attempts++; 
@@ -256,28 +256,43 @@ bool DatabaseKVAssociativeCache::findKey(const Goldilocks::Element (&key)[4], ve
         
         uint32_t cacheIndex = cacheIndexRaw  & cacheMask;
         uint32_t cacheIndexKey = cacheIndex * 4;
+        uint32_t cacheIndexVersions = cacheIndex * 2;
 
-        if (keys[cacheIndexKey + 0].fe == key[0].fe &&
-            keys[cacheIndexKey + 1].fe == key[1].fe &&
-            keys[cacheIndexKey + 2].fe == key[2].fe &&
-            keys[cacheIndexKey + 3].fe == key[3].fe)
-        {
-            uint32_t cacheIndexValue = cacheIndex * 8;
-            ++hits;
-            value.resize(8);
-            value[0] = values[cacheIndexValue];
-            value[1] = values[cacheIndexValue + 1];
-            value[2] = values[cacheIndexValue + 2];
-            value[3] = values[cacheIndexValue + 3];
-            value[4] = values[cacheIndexValue + 4];
-            value[5] = values[cacheIndexValue + 5];
-            value[6] = values[cacheIndexValue + 6];
-            value[7] = values[cacheIndexValue + 7];
-            value[8] = values[cacheIndexValue + 8];
-            value[9] = values[cacheIndexValue + 9];
-            value[10] = values[cacheIndexValue + 10];
-            value[11] = values[cacheIndexValue + 11];
-            return true;
+        for(int j=0; j<maxVersions; j++){
+            if (keys[cacheIndexKey + 0].fe == key[0].fe &&
+                keys[cacheIndexKey + 1].fe == key[1].fe &&
+                keys[cacheIndexKey + 2].fe == key[2].fe &&
+                keys[cacheIndexKey + 3].fe == key[3].fe){
+
+                if(last || versions[cacheIndexVersions] == version){
+                    uint32_t cacheIndexValue = cacheIndex * 8;
+                    ++hits;
+                    value.resize(8);
+                    value[0] = values[cacheIndexValue];
+                    value[1] = values[cacheIndexValue + 1];
+                    value[2] = values[cacheIndexValue + 2];
+                    value[3] = values[cacheIndexValue + 3];
+                    value[4] = values[cacheIndexValue + 4];
+                    value[5] = values[cacheIndexValue + 5];
+                    value[6] = values[cacheIndexValue + 6];
+                    value[7] = values[cacheIndexValue + 7];
+                    value[8] = values[cacheIndexValue + 8];
+                    value[9] = values[cacheIndexValue + 9];
+                    value[10] = values[cacheIndexValue + 10];
+                    value[11] = values[cacheIndexValue + 11];
+                    return true;
+                }
+                cacheIndex = versions[cacheIndexVersions+1] & cacheMask;
+                cacheIndexKey = cacheIndex * 4;
+                cacheIndexVersions = cacheIndex * 2;
+
+            }else{
+                if(j>0){
+                    return false;
+                }else{
+                    break;
+                }
+            }
         }
     }
     return false;
diff --git a/src/hashdb64/database_kv_associative_cache_64.hpp b/src/hashdb64/database_kv_associative_cache_64.hpp
index 5f5183e70..f4c8c1ea8 100644
--- a/src/hashdb64/database_kv_associative_cache_64.hpp
+++ b/src/hashdb64/database_kv_associative_cache_64.hpp
@@ -17,11 +17,12 @@ class DatabaseKVAssociativeCache
         uint32_t indexesSize;
         int log2CacheSize;
         uint32_t cacheSize;
+        int maxVersions;
 
         uint32_t *indexes;
         Goldilocks::Element *keys;
+        uint64_t *versions;
         Goldilocks::Element *values;
-        uint64_t * versions;
         uint32_t currentCacheIndex; 
 
         uint64_t attempts;
@@ -35,17 +36,18 @@ class DatabaseKVAssociativeCache
     public:
 
         DatabaseKVAssociativeCache();
-        DatabaseKVAssociativeCache(int log2IndexesSize_, int log2CacheSize_, string name_);
-        ~DatabaseKVAssociativeCache();
-
+        DatabaseKVAssociativeCache(int log2IndexesSize_, int log2CacheSize_, string name_);~DatabaseKVAssociativeCache();
         void postConstruct(int log2IndexesSize_, int log2CacheSize_, string name_);
+
         void addKeyValueVersion(Goldilocks::Element (&key)[4], const vector<Goldilocks::Element> &value, uint64_t version, bool update);
-        bool findKey(const Goldilocks::Element (&key)[4], vector<Goldilocks::Element> &value, bool last = true, uint64_t version=0);
+        bool findKey(const Goldilocks::Element (&key)[4], vector<Goldilocks::Element> &value, const bool last = true, const uint64_t version=0);
+
         inline bool enabled() const { return (log2IndexesSize > 0); };
         inline uint32_t getCacheSize()  const { return cacheSize; };
         inline uint32_t getIndexesSize() const { return indexesSize; };
 
     private:
+    
         inline bool emptyCacheSlot(uint32_t cacheIndexRaw) const { 
             return (currentCacheIndex >= cacheIndexRaw &&  currentCacheIndex - cacheIndexRaw > cacheSize) ||
             (currentCacheIndex < cacheIndexRaw && UINT32_MAX - cacheIndexRaw + currentCacheIndex > cacheSize);
diff --git a/src/hashdb64/database_versions_associtive_cache_64.cpp b/src/hashdb64/database_versions_associtive_cache_64.cpp
index 2ebac9324..d2c4a2f50 100644
--- a/src/hashdb64/database_versions_associtive_cache_64.cpp
+++ b/src/hashdb64/database_versions_associtive_cache_64.cpp
@@ -16,7 +16,7 @@ DatabaseVersionsAssociativeCache::DatabaseVersionsAssociativeCache()
     cacheSize = 0;
     indexes = NULL;
     keys = NULL;
-    versionBlock = NULL;
+    versions = NULL;
     currentCacheIndex = 0;
     attempts = 0;
     hits = 0;
@@ -34,8 +34,8 @@ DatabaseVersionsAssociativeCache::~DatabaseVersionsAssociativeCache()
         delete[] indexes;
     if (keys != NULL)
         delete[] keys;
-    if (versionBlock != NULL)
-        delete[] versionBlock;
+    if (versions != NULL)
+        delete[] versions;
 
 };
 
@@ -70,8 +70,8 @@ void DatabaseVersionsAssociativeCache::postConstruct(int log2IndexesSize_, int l
     if(keys != NULL) delete[] keys;
     keys = new Goldilocks::Element[4 * cacheSize];
 
-    if(versionBlock != NULL) delete[] versionBlock;
-    versionBlock = new uint64_t[2 * cacheSize];
+    if(versions != NULL) delete[] versions;
+    versions = new uint64_t[2 * cacheSize];
 
     currentCacheIndex = 0;
     attempts = 0;
@@ -83,82 +83,72 @@ void DatabaseVersionsAssociativeCache::postConstruct(int log2IndexesSize_, int l
     indexesMask = indexesSize - 1;
 };
 
-void DatabaseVersionsAssociativeCache::addKeyVersionBlock(Goldilocks::Element (&key)[4], const uint64_t (&vb)[2], bool update)
+void DatabaseVersionsAssociativeCache::addKeyVersion(Goldilocks::Element (&key)[4], const uint64_t version, const bool update)
 {
+    
     lock_guard<recursive_mutex> guard(mlock);
+    bool emptySlot = false;
+    bool present = false;
+    uint32_t cacheIndex;
+    uint32_t tableIndexEmpty=0;
+
     //
-    // Try to add in one of my 4 slots
+    // Check if present in one of the four slots
     //
     for (int i = 0; i < 4; ++i)
     {
         uint32_t tableIndex = (uint32_t)(key[i].fe & indexesMask);
         uint32_t cacheIndexRaw = indexes[tableIndex];
-        uint32_t cacheIndex = cacheIndexRaw & cacheMask;
-        uint32_t cacheIndexKey, cacheIndexValue;
-        bool write = false;
-
-        if (emptyCacheSlot(cacheIndexRaw))
-        {
-            write = true;
-            indexes[tableIndex] = currentCacheIndex;
-            currentCacheIndex = (currentCacheIndex == UINT32_MAX) ? 0 : (currentCacheIndex + 1);
-
-            cacheIndex = indexes[tableIndex] & cacheMask;
-            cacheIndexKey = cacheIndex * 4;
-            cacheIndexValue = cacheIndex * 2;
-        }
-        else
-        {
-            cacheIndexKey = cacheIndex * 4;
-            cacheIndexValue = cacheIndex * 2;
+        cacheIndex = cacheIndexRaw & cacheMask;
+        uint32_t cacheIndexKey = cacheIndex * 4;
 
-            if (keys[cacheIndexKey + 0].fe == key[0].fe &&
+        if (!emptyCacheSlot(cacheIndexRaw)){
+            if( keys[cacheIndexKey + 0].fe == key[0].fe &&
                 keys[cacheIndexKey + 1].fe == key[1].fe &&
                 keys[cacheIndexKey + 2].fe == key[2].fe &&
-                keys[cacheIndexKey + 3].fe == key[3].fe)
-            {
-                write = update;
-            }
-            else
-            {
-                continue;
+                keys[cacheIndexKey + 3].fe == key[3].fe){
+                    if(update == false) return;
+                    present = true;
+                    break;
             }
+        }else if (emptySlot == false){
+            emptySlot = true;
+            tableIndexEmpty = tableIndex;
         }
-        if (write) 
-        {
-            keys[cacheIndexKey + 0].fe = key[0].fe;
-            keys[cacheIndexKey + 1].fe = key[1].fe;
-            keys[cacheIndexKey + 2].fe = key[2].fe;
-            keys[cacheIndexKey + 3].fe = key[3].fe;
-            versionBlock[cacheIndexValue + 0] = vb[0];
-            versionBlock[cacheIndexValue + 1] = vb[1];
-           
-            return;
-        }else{
-            return;
+    }
+
+    //
+    // Evaluate cacheIndexKey and 
+    //
+    if(!present){
+        if(emptySlot == true){
+            indexes[tableIndexEmpty] = currentCacheIndex;
         }
+        cacheIndex = (uint32_t)(currentCacheIndex & cacheMask);
+        currentCacheIndex = (currentCacheIndex == UINT32_MAX) ? 0 : (currentCacheIndex + 1);
     }
+    uint64_t cacheIndexKey, cacheIndexValue;
+    cacheIndexKey = cacheIndex * 4;
+    cacheIndexValue = cacheIndex;
+    
     //
-    // forced entry insertion
+    // Add value
     //
-    uint32_t cacheIndex = (uint32_t)(currentCacheIndex & cacheMask);
-    currentCacheIndex = (currentCacheIndex == UINT32_MAX) ? 0 : (currentCacheIndex + 1);
-    uint32_t cacheIndexKey = cacheIndex * 4;
-    uint32_t cacheIndexValue = cacheIndex * 2;
     keys[cacheIndexKey + 0].fe = key[0].fe;
     keys[cacheIndexKey + 1].fe = key[1].fe;
     keys[cacheIndexKey + 2].fe = key[2].fe;
     keys[cacheIndexKey + 3].fe = key[3].fe;
-    versionBlock[cacheIndexValue + 0] = vb[0];
-    versionBlock[cacheIndexValue + 1] = vb[1];
+    versions[cacheIndexValue ] = version;         
+
     //
     // Forced index insertion
     //
-    int iters = 0;
-    uint32_t usedRawCacheIndexes[10];
-    usedRawCacheIndexes[0] = currentCacheIndex-1;
-    forcedInsertion(usedRawCacheIndexes, iters);
-
+    if(!present && !emptySlot){
+        int iters = 0;
+        uint32_t usedRawCacheIndexes[10];
+        usedRawCacheIndexes[0] = currentCacheIndex-1;
+        forcedInsertion(usedRawCacheIndexes, iters);
+    }
 }
 
 void DatabaseVersionsAssociativeCache::forcedInsertion(uint32_t (&usedRawCacheIndexes)[10], int &iters)
@@ -218,7 +208,7 @@ void DatabaseVersionsAssociativeCache::forcedInsertion(uint32_t (&usedRawCacheIn
     
 }
 
-bool DatabaseVersionsAssociativeCache::findKey(const Goldilocks::Element (&key)[4], uint64_t (&vb)[2])
+bool DatabaseVersionsAssociativeCache::findKey(const Goldilocks::Element (&key)[4], uint64_t &version)
 {
     lock_guard<recursive_mutex> guard(mlock);
     attempts++; 
@@ -245,10 +235,8 @@ bool DatabaseVersionsAssociativeCache::findKey(const Goldilocks::Element (&key)[
             keys[cacheIndexKey + 2].fe == key[2].fe &&
             keys[cacheIndexKey + 3].fe == key[3].fe)
         {
-            uint32_t cacheIndexValue = cacheIndex * 2;
             ++hits;
-            vb[0] = versionBlock[cacheIndexValue];
-            vb[1] = versionBlock[cacheIndexValue + 1];
+            version = versions[cacheIndex];
             return true;
         }
     }
diff --git a/src/hashdb64/database_versions_associtive_cache_64.hpp b/src/hashdb64/database_versions_associtive_cache_64.hpp
index 2faabfd5f..245fb7494 100644
--- a/src/hashdb64/database_versions_associtive_cache_64.hpp
+++ b/src/hashdb64/database_versions_associtive_cache_64.hpp
@@ -20,7 +20,7 @@ class DatabaseVersionsAssociativeCache
 
         uint32_t *indexes;
         Goldilocks::Element *keys;
-        uint64_t *versionBlock;
+        uint64_t *versions;
         uint32_t currentCacheIndex; 
 
         uint64_t attempts;
@@ -36,15 +36,17 @@ class DatabaseVersionsAssociativeCache
         DatabaseVersionsAssociativeCache();
         DatabaseVersionsAssociativeCache(int log2IndexesSize_, int log2CacheSize_, string name_);
         ~DatabaseVersionsAssociativeCache();
-
         void postConstruct(int log2IndexesSize_, int log2CacheSize_, string name_);
-        void addKeyVersionBlock(Goldilocks::Element (&key)[4], const uint64_t (&vb)[2], const bool update);
-        bool findKey(const Goldilocks::Element (&key)[4], uint64_t (&vb)[2]);
+
+        void addKeyVersion(Goldilocks::Element (&key)[4], const uint64_t version, const bool update);
+        bool findKey(const Goldilocks::Element (&key)[4], uint64_t &version);
+        
         inline bool enabled() const { return (log2IndexesSize > 0); };
         inline uint32_t getCacheSize()  const { return cacheSize; };
         inline uint32_t getIndexesSize() const { return indexesSize; };
 
     private:
+        
         inline bool emptyCacheSlot(uint32_t cacheIndexRaw) const { 
             return (currentCacheIndex >= cacheIndexRaw &&  currentCacheIndex - cacheIndexRaw > cacheSize) ||
             (currentCacheIndex < cacheIndexRaw && UINT32_MAX - cacheIndexRaw + currentCacheIndex > cacheSize);

From 9787d180fe24a3b81c46e217d8b5e149c66fd284 Mon Sep 17 00:00:00 2001
From: rickb80 <rickborrell@gmail.com>
Date: Wed, 30 Aug 2023 06:55:00 +0200
Subject: [PATCH 08/17] Rename files

---
 ...sociative_cache_64.cpp => database_kv_associative_cache.cpp} | 2 +-
 ...sociative_cache_64.hpp => database_kv_associative_cache.hpp} | 0
 ...tive_cache_64.cpp => database_versions_associtive_cache.cpp} | 2 +-
 ...tive_cache_64.hpp => database_versions_associtive_cache.hpp} | 0
 4 files changed, 2 insertions(+), 2 deletions(-)
 rename src/hashdb64/{database_kv_associative_cache_64.cpp => database_kv_associative_cache.cpp} (99%)
 rename src/hashdb64/{database_kv_associative_cache_64.hpp => database_kv_associative_cache.hpp} (100%)
 rename src/hashdb64/{database_versions_associtive_cache_64.cpp => database_versions_associtive_cache.cpp} (99%)
 rename src/hashdb64/{database_versions_associtive_cache_64.hpp => database_versions_associtive_cache.hpp} (100%)

diff --git a/src/hashdb64/database_kv_associative_cache_64.cpp b/src/hashdb64/database_kv_associative_cache.cpp
similarity index 99%
rename from src/hashdb64/database_kv_associative_cache_64.cpp
rename to src/hashdb64/database_kv_associative_cache.cpp
index 355e7777d..dbbe28463 100644
--- a/src/hashdb64/database_kv_associative_cache_64.cpp
+++ b/src/hashdb64/database_kv_associative_cache.cpp
@@ -1,4 +1,4 @@
-#include "database_kv_associative_cache_64.hpp"
+#include "database_kv_associative_cache.hpp"
 #include <vector>
 #include "goldilocks_base_field.hpp"
 #include <nlohmann/json.hpp>
diff --git a/src/hashdb64/database_kv_associative_cache_64.hpp b/src/hashdb64/database_kv_associative_cache.hpp
similarity index 100%
rename from src/hashdb64/database_kv_associative_cache_64.hpp
rename to src/hashdb64/database_kv_associative_cache.hpp
diff --git a/src/hashdb64/database_versions_associtive_cache_64.cpp b/src/hashdb64/database_versions_associtive_cache.cpp
similarity index 99%
rename from src/hashdb64/database_versions_associtive_cache_64.cpp
rename to src/hashdb64/database_versions_associtive_cache.cpp
index d2c4a2f50..82bd0ad0b 100644
--- a/src/hashdb64/database_versions_associtive_cache_64.cpp
+++ b/src/hashdb64/database_versions_associtive_cache.cpp
@@ -1,4 +1,4 @@
-#include "database_versions_associtive_cache_64.hpp"
+#include "database_versions_associtive_cache.hpp"
 #include <vector>
 #include "goldilocks_base_field.hpp"
 #include <nlohmann/json.hpp>
diff --git a/src/hashdb64/database_versions_associtive_cache_64.hpp b/src/hashdb64/database_versions_associtive_cache.hpp
similarity index 100%
rename from src/hashdb64/database_versions_associtive_cache_64.hpp
rename to src/hashdb64/database_versions_associtive_cache.hpp

From 9c2917d1d1f93f899bf64993b10ac58fbc4744e7 Mon Sep 17 00:00:00 2001
From: rickb80 <rickborrell@gmail.com>
Date: Fri, 1 Sep 2023 12:15:37 +0200
Subject: [PATCH 09/17] Adds KeyValue and Version table

---
 src/config/config.cpp      | 4 ++++
 src/config/config.hpp      | 2 ++
 tools/statedb/create_db.sh | 4 ++--
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/config/config.cpp b/src/config/config.cpp
index df265fb58..e33883910 100644
--- a/src/config/config.cpp
+++ b/src/config/config.cpp
@@ -259,6 +259,8 @@ void Config::load(json &config)
     ParseString(config, "databaseURL", "DATABASE_URL", databaseURL, "local");
     ParseString(config, "dbNodesTableName", "DB_NODES_TABLE_NAME", dbNodesTableName, "state.nodes");
     ParseString(config, "dbProgramTableName", "DB_PROGRAM_TABLE_NAME", dbProgramTableName, "state.program");
+    ParseString(config, "dbKeyValueTableName", "DB_KEYVALUE_TABLE_NAME", dbKeyValueTableName, "state.keyvalue");
+    ParseString(config, "dbKeyVersionTableName", "DB_VERSION_TABLE_NAME", dbVersionTableName, "state.version");
     ParseBool(config, "dbMultiWrite", "DB_MULTIWRITE", dbMultiWrite, true);
     ParseU64(config, "dbMultiWriteSingleQuerySize", "DB_MULTIWRITE_SINGLE_QUERY_SIZE", dbMultiWriteSingleQuerySize, 20*1024*1024);
     ParseBool(config, "dbConnectionsPool", "DB_CONNECTIONS_POOL", dbConnectionsPool, true);
@@ -450,6 +452,8 @@ void Config::print(void)
     zklog.info("    databaseURL=" + databaseURL.substr(0, 5) + "...");
     zklog.info("    dbNodesTableName=" + dbNodesTableName);
     zklog.info("    dbProgramTableName=" + dbProgramTableName);
+    zklog.info("    dbKeyValueTableName=" + dbKeyValueTableName);
+    zklog.info("    dbVersionTableName=" + dbVersionTableName);
     zklog.info("    dbMultiWrite=" + to_string(dbMultiWrite));
     zklog.info("    dbMultiWriteSingleQuerySize=" + to_string(dbMultiWriteSingleQuerySize));
     zklog.info("    dbConnectionsPool=" + to_string(dbConnectionsPool));
diff --git a/src/config/config.hpp b/src/config/config.hpp
index e33264b67..fde140102 100644
--- a/src/config/config.hpp
+++ b/src/config/config.hpp
@@ -144,6 +144,8 @@ class Config
     string databaseURL;
     string dbNodesTableName;
     string dbProgramTableName;
+    string dbKeyValueTableName;
+    string dbVersionTableName;
     bool dbMultiWrite;
     uint64_t dbMultiWriteSingleQuerySize;
     bool dbConnectionsPool;
diff --git a/tools/statedb/create_db.sh b/tools/statedb/create_db.sh
index 6a46672f5..5f9659be7 100755
--- a/tools/statedb/create_db.sh
+++ b/tools/statedb/create_db.sh
@@ -16,8 +16,8 @@ echo "Creating table state.merkletree..."
 PGPASSWORD=$3 psql -U $2 -h 127.0.0.1 -p 5432 -d $1 -c 'create schema state;'
 PGPASSWORD=$3 psql -U $2 -h 127.0.0.1 -p 5432 -d $1 -c 'create table state.nodes (hash bytea primary key, data bytea not null);'
 PGPASSWORD=$3 psql -U $2 -h 127.0.0.1 -p 5432 -d $1 -c 'create table state.program (hash bytea primary key, data bytea not null);'
-PGPASSWORD=$3 psql -U $2 -h 127.0.0.1 -p 5432 -d $1 -c 'create table state.nodesKV (key bytea, data bytea not null, version integer not null);'
-PGPASSWORD=$3 psql -U $2 -h 127.0.0.1 -p 5432 -d $1 -c 'create table state.versionKV (version integer not null, hash bytea primary key, block integer not null);'
+PGPASSWORD=$3 psql -U $2 -h 127.0.0.1 -p 5432 -d $1 -c 'create table state.keyvalue (key bytea primary key, data bytea);'
+PGPASSWORD=$3 psql -U $2 -h 127.0.0.1 -p 5432 -d $1 -c 'create table state.version (hash bytea primary key, version bigint);'
 
 if [ $# == 4 ]
     then

From a614689f354c12f4a132ca569171d30079eefe38 Mon Sep 17 00:00:00 2001
From: rickb80 <rickborrell@gmail.com>
Date: Fri, 1 Sep 2023 12:16:23 +0200
Subject: [PATCH 10/17] Extended database_map for new database tables

---
 src/hashdb/database_map.cpp | 90 +++++++++++++++++++++++++++++++++++++
 src/hashdb/database_map.hpp | 65 +++++++++++++++++++++++++++
 2 files changed, 155 insertions(+)

diff --git a/src/hashdb/database_map.cpp b/src/hashdb/database_map.cpp
index fa4dd2451..b2dec6494 100644
--- a/src/hashdb/database_map.cpp
+++ b/src/hashdb/database_map.cpp
@@ -14,6 +14,30 @@ void DatabaseMap::add(MTMap &db)
     if (callbackOnChange) onChangeCallback();
 }
 
+void DatabaseMap::add(MT64Map &db)
+{
+    lock_guard<recursive_mutex> guard(mlock);
+
+    mt64DB.insert(db.begin(), db.end());
+    if (callbackOnChange) onChangeCallback();
+}
+
+void DatabaseMap::add(MT64KVMap &db)
+{
+    lock_guard<recursive_mutex> guard(mlock);
+
+    mt64KVDB.insert(db.begin(), db.end());
+    if (callbackOnChange) onChangeCallback();
+}
+
+void DatabaseMap::add(MT64VersionMap &db)
+{
+    lock_guard<recursive_mutex> guard(mlock);
+
+    mt64VersionDB.insert(db.begin(), db.end());
+    if (callbackOnChange) onChangeCallback();
+}
+
 void DatabaseMap::add(ProgramMap &db)
 {
     lock_guard<recursive_mutex> guard(mlock);
@@ -37,6 +61,51 @@ bool DatabaseMap::findMT(const string& key, vector<Goldilocks::Element> &value)
     return false;
 }
 
+bool DatabaseMap::findMT64(const string& key, string &value)
+{
+    lock_guard<recursive_mutex> guard(mlock);
+
+    DatabaseMap::MT64Map::iterator it = mt64DB.find(key);
+
+    if (it != mt64DB.end())
+    {
+        value = it->second;
+        return true;
+    }
+
+    return false;
+}
+
+bool DatabaseMap::findMT64KV(const string& key, mpz_class &value)
+{
+    lock_guard<recursive_mutex> guard(mlock);
+
+    DatabaseMap::MT64KVMap::iterator it = mt64KVDB.find(key);
+
+    if (it != mt64KVDB.end())
+    {
+        value = it->second;
+        return true;
+    }
+
+    return false;
+}
+
+bool DatabaseMap::findMT64Version(const string& key, uint64_t &value)
+{
+    lock_guard<recursive_mutex> guard(mlock);
+
+    DatabaseMap::MT64VersionMap::iterator it = mt64VersionDB.find(key);
+
+    if (it != mt64VersionDB.end())
+    {
+        value = it->second;
+        return true;
+    }
+
+    return false;
+}
+
 bool DatabaseMap::findProgram(const string& key, vector<uint8_t> &value)
 {
     lock_guard<recursive_mutex> guard(mlock);
@@ -59,6 +128,27 @@ DatabaseMap::MTMap DatabaseMap::getMTDB()
     return mtDB;
 }
 
+DatabaseMap::MT64Map DatabaseMap::getMT64DB()
+{
+    lock_guard<recursive_mutex> guard(mlock);
+
+    return mt64DB;
+}
+
+DatabaseMap::MT64KVMap DatabaseMap::getMT64KVDB()
+{
+    lock_guard<recursive_mutex> guard(mlock);
+
+    return mt64KVDB;
+}
+
+DatabaseMap::MT64VersionMap DatabaseMap::getMT64VersionDB()
+{
+    lock_guard<recursive_mutex> guard(mlock);
+
+    return mt64VersionDB;
+}
+
 DatabaseMap::ProgramMap DatabaseMap::getProgramDB()
 {
     lock_guard<recursive_mutex> guard(mlock);
diff --git a/src/hashdb/database_map.hpp b/src/hashdb/database_map.hpp
index ae76c7958..6890d1420 100644
--- a/src/hashdb/database_map.hpp
+++ b/src/hashdb/database_map.hpp
@@ -17,6 +17,8 @@ class DatabaseMap
 public:
     typedef unordered_map<string, vector<Goldilocks::Element>> MTMap;
     typedef unordered_map<string, string> MT64Map;
+    typedef unordered_map<string, mpz_class> MT64KVMap;
+    typedef unordered_map<string, uint64_t> MT64VersionMap;
     typedef unordered_map<string, vector<uint8_t>> ProgramMap;
 
 private:
@@ -25,6 +27,8 @@ class DatabaseMap
     recursive_mutex mlock;
     MTMap mtDB;
     MT64Map mt64DB;
+    MT64KVMap mt64KVDB;
+    MT64VersionMap mt64VersionDB;
     ProgramMap programDB;
     bool callbackOnChange = false;
     bool saveKeys = false;
@@ -39,6 +43,14 @@ class DatabaseMap
     uint64_t mt64CachedTime;
     uint64_t mt64DbTimes;
     uint64_t mt64DbTime;
+    uint64_t mt64KVCachedTimes;
+    uint64_t mt64KVCachedTime;
+    uint64_t mt64KVDbTimes;
+    uint64_t mt64KVDbTime;
+    uint64_t mt64VersionCachedTimes;
+    uint64_t mt64VersionCachedTime;
+    uint64_t mt64VersionDbTimes;
+    uint64_t mt64VersionDbTime;
     uint64_t programCachedTimes;
     uint64_t programCachedTime;
     uint64_t programDbTimes;
@@ -60,6 +72,14 @@ class DatabaseMap
         mt64CachedTime(0),
         mt64DbTimes(0),
         mt64DbTime(0),
+        mt64KVCachedTimes(0),
+        mt64KVCachedTime(0),
+        mt64KVDbTimes(0),
+        mt64KVDbTime(0),
+        mt64VersionCachedTimes(0),
+        mt64VersionCachedTime(0),
+        mt64VersionDbTimes(0),
+        mt64VersionDbTime(0),
         programCachedTimes(0),
         programCachedTime(0),
         programDbTimes(0),
@@ -73,12 +93,23 @@ class DatabaseMap
     inline void add(const string& key, const vector<Goldilocks::Element>& value, const bool cached, const uint64_t time);
     inline void add(const string& key, const string& value, const bool cached, const uint64_t time);
     inline void add(const string& key, const vector<uint8_t>& value, const bool cached, const uint64_t time);
+    inline void add(const string& key, const mpz_class& value, const bool cached, const uint64_t time);
+    inline void add(const string& key, const uint64_t version, const bool cached, const uint64_t time);
     inline void addGetTree(const uint64_t time, const uint64_t numberOfFields);
     void add(MTMap &db);
     void add(ProgramMap &db);
+    void add(MT64Map &db);
+    void add(MT64KVMap &db);
+    void add(MT64VersionMap &db);
     bool findMT(const string& key, vector<Goldilocks::Element> &value);
+    bool findMT64(const string& key, string &value);
+    bool findMT64KV(const string& key, mpz_class &value);
+    bool findMT64Version(const string& key, uint64_t &value);
     bool findProgram(const string& key, vector<uint8_t> &value);
     MTMap getMTDB();
+    MT64Map getMT64DB();
+    MT64KVMap getMT64KVDB();
+    MT64VersionMap getMT64VersionDB();
     ProgramMap getProgramDB();
     void setOnChangeCallback(void *instance, onChangeCallbackFunctionPtr function);
     inline void setSaveKeys(const bool saveKeys_){ saveKeys = saveKeys_; };
@@ -138,6 +169,40 @@ void DatabaseMap::add(const string& key, const vector<uint8_t>& value, const boo
     if (callbackOnChange) onChangeCallback();
 }
 
+void DatabaseMap::add(const string& key, const mpz_class& value, const bool cached, const uint64_t time)
+{
+    lock_guard<recursive_mutex> guard(mlock);
+    if(saveKeys) mt64KVDB[key] = value;
+    if (cached)
+    {
+        mt64KVCachedTimes += 1;
+        mt64KVCachedTime += time;
+    }
+    else
+    {
+        mt64KVDbTimes += 1;
+        mt64KVDbTime += time;
+    }
+    if (callbackOnChange) onChangeCallback();
+}
+
+void DatabaseMap::add(const string& key, const uint64_t version, const bool cached, const uint64_t time)
+{
+    lock_guard<recursive_mutex> guard(mlock);
+    if(saveKeys) mt64VersionDB[key] = version;
+    if (cached)
+    {
+        mt64VersionCachedTimes += 1;
+        mt64VersionCachedTime += time;
+    }
+    else
+    {
+        mt64VersionDbTimes += 1;
+        mt64VersionDbTime += time;
+    }
+    if (callbackOnChange) onChangeCallback();
+}
+
 void DatabaseMap::addGetTree(const uint64_t time, const uint64_t numberOfFields)
 {
     lock_guard<recursive_mutex> guard(mlock);

From 79dee16d28ad6e05ca33fb9f08ba416094681e2a Mon Sep 17 00:00:00 2001
From: rickb80 <rickborrell@gmail.com>
Date: Fri, 1 Sep 2023 14:44:43 +0200
Subject: [PATCH 11/17] Erroneous comparision in database.cpp

---
 src/hashdb/database.cpp                       |  2 +-
 .../database_kv_associative_cache.cpp         | 36 +++++--------------
 .../database_kv_associative_cache.hpp         |  9 ++---
 .../database_versions_associtive_cache.cpp    | 12 ++++---
 .../database_versions_associtive_cache.hpp    |  2 +-
 5 files changed, 23 insertions(+), 38 deletions(-)

diff --git a/src/hashdb/database.cpp b/src/hashdb/database.cpp
index 40489a834..407515eb7 100644
--- a/src/hashdb/database.cpp
+++ b/src/hashdb/database.cpp
@@ -294,7 +294,7 @@ zkresult Database::write(const string &_key, const Goldilocks::Element* vkey, co
         exitProcess();
     }
 
-    if (config.dbMultiWrite && !(dbMTCache.enabled() || dbMTACache.enabled()) && !persistent)
+    if (config.dbMultiWrite && !(dbMTCache.enabled() && dbMTACache.enabled()) && !persistent)
     {
         zklog.error("Database::write() called with multi-write active, cache disabled and no persistance in database, so there is no place to store the date");
         return ZKR_DB_ERROR;
diff --git a/src/hashdb64/database_kv_associative_cache.cpp b/src/hashdb64/database_kv_associative_cache.cpp
index dbbe28463..b7d20d728 100644
--- a/src/hashdb64/database_kv_associative_cache.cpp
+++ b/src/hashdb64/database_kv_associative_cache.cpp
@@ -77,7 +77,7 @@ void DatabaseKVAssociativeCache::postConstruct(int log2IndexesSize_, int log2Cac
     keys = new Goldilocks::Element[4 * cacheSize];
 
     if(values != NULL) delete[] values;
-    values = new Goldilocks::Element[8 * cacheSize];
+    values = new mpz_class[cacheSize];
 
     if(versions != NULL) delete[] versions;
     versions = new uint64_t[2 * cacheSize];
@@ -92,7 +92,7 @@ void DatabaseKVAssociativeCache::postConstruct(int log2IndexesSize_, int log2Cac
     indexesMask = indexesSize - 1;
 };
 
-void DatabaseKVAssociativeCache::addKeyValueVersion(Goldilocks::Element (&key)[4], const vector<Goldilocks::Element> &value, uint64_t version, bool update){
+void DatabaseKVAssociativeCache::addKeyValueVersion(const uint64_t version, const Goldilocks::Element (&key)[4], const mpz_class &value, bool update){
     
     lock_guard<recursive_mutex> guard(mlock);
     bool emptySlot = false;
@@ -102,7 +102,6 @@ void DatabaseKVAssociativeCache::addKeyValueVersion(Goldilocks::Element (&key)[4
     uint32_t tableIndexEmpty=0;
     uint32_t cacheIndexPrev;
 
-
     //
     // Check if present in one of the four slots
     //
@@ -145,7 +144,7 @@ void DatabaseKVAssociativeCache::addKeyValueVersion(Goldilocks::Element (&key)[4
     }
     uint64_t cacheIndexKey, cacheIndexValue, cacheIndexVersions;
     cacheIndexKey = cacheIndex * 4;
-    cacheIndexValue = cacheIndex * 8;
+    cacheIndexValue = cacheIndex;
     cacheIndexVersions = cacheIndex * 2;
     
     //
@@ -155,14 +154,7 @@ void DatabaseKVAssociativeCache::addKeyValueVersion(Goldilocks::Element (&key)[4
     keys[cacheIndexKey + 1].fe = key[1].fe;
     keys[cacheIndexKey + 2].fe = key[2].fe;
     keys[cacheIndexKey + 3].fe = key[3].fe;
-    values[cacheIndexValue + 0] = value[0];
-    values[cacheIndexValue + 1] = value[1];
-    values[cacheIndexValue + 2] = value[2];
-    values[cacheIndexValue + 3] = value[3];
-    values[cacheIndexValue + 4] = value[4];
-    values[cacheIndexValue + 5] = value[5];
-    values[cacheIndexValue + 6] = value[6];
-    values[cacheIndexValue + 7] = value[7];
+    values[cacheIndexValue] = value;
     versions[cacheIndexVersions] = version;
     if(present & !presentSameVersion){
         versions[cacheIndexVersions+1] = cacheIndexPrev;
@@ -235,7 +227,7 @@ void DatabaseKVAssociativeCache::forcedInsertion(uint32_t (&usedRawCacheIndexes)
     
 }
 
-bool DatabaseKVAssociativeCache::findKey(const Goldilocks::Element (&key)[4], vector<Goldilocks::Element> &value, const bool last, const uint64_t version)
+bool DatabaseKVAssociativeCache::findKey( const uint64_t version, const Goldilocks::Element (&key)[4], mpz_class &value)
 {
     lock_guard<recursive_mutex> guard(mlock);
     attempts++; 
@@ -264,22 +256,10 @@ bool DatabaseKVAssociativeCache::findKey(const Goldilocks::Element (&key)[4], ve
                 keys[cacheIndexKey + 2].fe == key[2].fe &&
                 keys[cacheIndexKey + 3].fe == key[3].fe){
 
-                if(last || versions[cacheIndexVersions] == version){
-                    uint32_t cacheIndexValue = cacheIndex * 8;
+                if( versions[cacheIndexVersions] <= version){ //rick: I assume they are ordered
+                    uint32_t cacheIndexValue = cacheIndex;
                     ++hits;
-                    value.resize(8);
-                    value[0] = values[cacheIndexValue];
-                    value[1] = values[cacheIndexValue + 1];
-                    value[2] = values[cacheIndexValue + 2];
-                    value[3] = values[cacheIndexValue + 3];
-                    value[4] = values[cacheIndexValue + 4];
-                    value[5] = values[cacheIndexValue + 5];
-                    value[6] = values[cacheIndexValue + 6];
-                    value[7] = values[cacheIndexValue + 7];
-                    value[8] = values[cacheIndexValue + 8];
-                    value[9] = values[cacheIndexValue + 9];
-                    value[10] = values[cacheIndexValue + 10];
-                    value[11] = values[cacheIndexValue + 11];
+                    value = values[cacheIndexValue];
                     return true;
                 }
                 cacheIndex = versions[cacheIndexVersions+1] & cacheMask;
diff --git a/src/hashdb64/database_kv_associative_cache.hpp b/src/hashdb64/database_kv_associative_cache.hpp
index f4c8c1ea8..e710ce080 100644
--- a/src/hashdb64/database_kv_associative_cache.hpp
+++ b/src/hashdb64/database_kv_associative_cache.hpp
@@ -22,7 +22,7 @@ class DatabaseKVAssociativeCache
         uint32_t *indexes;
         Goldilocks::Element *keys;
         uint64_t *versions;
-        Goldilocks::Element *values;
+        mpz_class *values; //rick: this I do not like
         uint32_t currentCacheIndex; 
 
         uint64_t attempts;
@@ -36,11 +36,12 @@ class DatabaseKVAssociativeCache
     public:
 
         DatabaseKVAssociativeCache();
-        DatabaseKVAssociativeCache(int log2IndexesSize_, int log2CacheSize_, string name_);~DatabaseKVAssociativeCache();
+        DatabaseKVAssociativeCache(int log2IndexesSize_, int log2CacheSize_, string name_);
+        ~DatabaseKVAssociativeCache();
         void postConstruct(int log2IndexesSize_, int log2CacheSize_, string name_);
 
-        void addKeyValueVersion(Goldilocks::Element (&key)[4], const vector<Goldilocks::Element> &value, uint64_t version, bool update);
-        bool findKey(const Goldilocks::Element (&key)[4], vector<Goldilocks::Element> &value, const bool last = true, const uint64_t version=0);
+        void addKeyValueVersion(const uint64_t version, const Goldilocks::Element (&key)[4], const mpz_class &value, bool update);
+        bool findKey( const uint64_t version, const Goldilocks::Element (&key)[4], mpz_class &value);
 
         inline bool enabled() const { return (log2IndexesSize > 0); };
         inline uint32_t getCacheSize()  const { return cacheSize; };
diff --git a/src/hashdb64/database_versions_associtive_cache.cpp b/src/hashdb64/database_versions_associtive_cache.cpp
index 82bd0ad0b..15945a925 100644
--- a/src/hashdb64/database_versions_associtive_cache.cpp
+++ b/src/hashdb64/database_versions_associtive_cache.cpp
@@ -7,6 +7,7 @@
 #include "zkmax.hpp"
 #include "exit_process.hpp"
 #include "scalar.hpp"
+#include "zkassert.hpp"
 
 DatabaseVersionsAssociativeCache::DatabaseVersionsAssociativeCache()
 {
@@ -83,7 +84,7 @@ void DatabaseVersionsAssociativeCache::postConstruct(int log2IndexesSize_, int l
     indexesMask = indexesSize - 1;
 };
 
-void DatabaseVersionsAssociativeCache::addKeyVersion(Goldilocks::Element (&key)[4], const uint64_t version, const bool update)
+void DatabaseVersionsAssociativeCache::addKeyVersion(Goldilocks::Element (&key)[4], const uint64_t version)
 {
     
     lock_guard<recursive_mutex> guard(mlock);
@@ -107,9 +108,12 @@ void DatabaseVersionsAssociativeCache::addKeyVersion(Goldilocks::Element (&key)[
                 keys[cacheIndexKey + 1].fe == key[1].fe &&
                 keys[cacheIndexKey + 2].fe == key[2].fe &&
                 keys[cacheIndexKey + 3].fe == key[3].fe){
-                    if(update == false) return;
-                    present = true;
-                    break;
+                    if(versions[cacheIndex] != version){
+                        zklog.error("DatabaseVersionsAssociativeCache::addKeyVersion() version mismatch");
+                        exitProcess();
+                    }
+                    return;
+                    //rick: assert no error
             }
         }else if (emptySlot == false){
             emptySlot = true;
diff --git a/src/hashdb64/database_versions_associtive_cache.hpp b/src/hashdb64/database_versions_associtive_cache.hpp
index 245fb7494..ffec03e9d 100644
--- a/src/hashdb64/database_versions_associtive_cache.hpp
+++ b/src/hashdb64/database_versions_associtive_cache.hpp
@@ -38,7 +38,7 @@ class DatabaseVersionsAssociativeCache
         ~DatabaseVersionsAssociativeCache();
         void postConstruct(int log2IndexesSize_, int log2CacheSize_, string name_);
 
-        void addKeyVersion(Goldilocks::Element (&key)[4], const uint64_t version, const bool update);
+        void addKeyVersion(Goldilocks::Element (&key)[4], const uint64_t version); //rick: no update!
         bool findKey(const Goldilocks::Element (&key)[4], uint64_t &version);
         
         inline bool enabled() const { return (log2IndexesSize > 0); };

From 872a8d35d63a22702c0d243519036ac1902cc385 Mon Sep 17 00:00:00 2001
From: rickb80 <rickborrell@gmail.com>
Date: Sat, 2 Sep 2023 16:55:02 +0200
Subject: [PATCH 12/17] Completed API for kv database

---
 src/config/config.cpp                         |   2 +
 src/config/config.hpp                         |   1 +
 src/config/zkresult.hpp                       |   2 +
 src/hashdb/database.cpp                       |   4 +-
 src/hashdb64/database_64.cpp                  | 704 +++++++++++++++++-
 src/hashdb64/database_64.hpp                  |  22 +-
 .../database_kv_associative_cache.hpp         |   2 +-
 .../database_versions_associtive_cache.cpp    |   2 +-
 .../database_versions_associtive_cache.hpp    |   2 +-
 9 files changed, 733 insertions(+), 8 deletions(-)

diff --git a/src/config/config.cpp b/src/config/config.cpp
index e33883910..78a8b76ae 100644
--- a/src/config/config.cpp
+++ b/src/config/config.cpp
@@ -178,6 +178,7 @@ void Config::load(json &config)
     ParseU16(config, "hashDBServerPort", "HASHDB_SERVER_PORT", hashDBServerPort, 50061);
     ParseString(config, "hashDBURL", "HASHDB_URL", hashDBURL, "local");
     ParseBool(config, "hashDB64", "HASHDB64", hashDB64, false);
+    ParseU64(config, "kvDBMaxVersions", "HASHDB64_MAX_VERSIONS", kvDBMaxVersions, 131072);
     ParseString(config, "dbCacheSynchURL", "DB_CACHE_SYNCH_URL", dbCacheSynchURL, "");
     ParseU16(config, "aggregatorServerPort", "AGGREGATOR_SERVER_PORT", aggregatorServerPort, 50081);
     ParseU16(config, "aggregatorClientPort", "AGGREGATOR_CLIENT_PORT", aggregatorClientPort, 50081);
@@ -411,6 +412,7 @@ void Config::print(void)
     zklog.info("    hashDBServerPort=" + to_string(hashDBServerPort));
     zklog.info("    hashDBURL=" + hashDBURL);
     zklog.info("    hashDB64=" + to_string(hashDB64));
+    zklog.info("    kvDBMaxVersions=" + to_string(kvDBMaxVersions));
     zklog.info("    dbCacheSynchURL=" + dbCacheSynchURL);
     zklog.info("    aggregatorServerPort=" + to_string(aggregatorServerPort));
     zklog.info("    aggregatorClientPort=" + to_string(aggregatorClientPort));
diff --git a/src/config/config.hpp b/src/config/config.hpp
index fde140102..dec3d6739 100644
--- a/src/config/config.hpp
+++ b/src/config/config.hpp
@@ -88,6 +88,7 @@ class Config
     uint16_t hashDBServerPort;
     string hashDBURL;
     bool hashDB64;
+    uint64_t kvDBMaxVersions;
     string dbCacheSynchURL;
 
     uint16_t aggregatorServerPort;
diff --git a/src/config/zkresult.hpp b/src/config/zkresult.hpp
index d9d3c2596..253c13ef7 100644
--- a/src/config/zkresult.hpp
+++ b/src/config/zkresult.hpp
@@ -88,6 +88,8 @@ typedef enum : int
     ZKR_SM_MAIN_MEMALIGN_READ_MISMATCH = 78, // Main state memory align read ROM operation check failed
     ZKR_SM_MAIN_HASHK_READ_OUT_OF_RANGE = 79, // Main state Keccak hash check found read out of range
     ZKR_SM_MAIN_HASHP_READ_OUT_OF_RANGE = 80, // Main state Poseidon hash check found read out of range
+    ZKR_DB_VERSION_NOT_FOUND = 81, // Version not found in KeyValue database
+    
 } zkresult;
 
 string zkresult2string (int code);
diff --git a/src/hashdb/database.cpp b/src/hashdb/database.cpp
index 407515eb7..47ad01212 100644
--- a/src/hashdb/database.cpp
+++ b/src/hashdb/database.cpp
@@ -586,13 +586,13 @@ zkresult Database::readRemote(bool bProgram, const string &key, string &value)
             exitProcess();
         }
 
-        pqxx::row const row = rows[0];
+        const pqxx::row& row = rows[0];
         if (row.size() != 2)
         {
             zklog.error("Database::readRemote() table=" + tableName + " got an invalid number of colums for the row: " + to_string(row.size()));
             exitProcess();
         }
-        pqxx::field const fieldData = row[1];
+        const pqxx::field& fieldData = row[1];
         value = removeBSXIfExists(fieldData.c_str());
     }
     catch (const std::exception &e)
diff --git a/src/hashdb64/database_64.cpp b/src/hashdb64/database_64.cpp
index cf93e0454..1cb46f691 100644
--- a/src/hashdb64/database_64.cpp
+++ b/src/hashdb64/database_64.cpp
@@ -22,6 +22,8 @@
 // DatabaseCacheMT and DatabaseCacheProgram classes are thread-safe
 DatabaseMTCache64 Database64::dbMTCache;
 DatabaseProgramCache64 Database64::dbProgramCache;
+DatabaseKVAssociativeCache Database64::dbKVACache;
+DatabaseVersionsAssociativeCache Database64::dbVersionACache;
 
 string Database64::dbStateRootKey("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); // 64 f's
 Goldilocks::Element Database64::dbStateRootvKey[4] = {0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF};
@@ -36,7 +38,8 @@ Database64::Database64 (Goldilocks &fr, const Config &config) :
         fr(fr),
         config(config),
         connectionsPool(NULL),
-        multiWrite(fr)
+        multiWrite(fr),
+        maxVersions(config.kvDBMaxVersions)
 {
     // Init mutex
     pthread_mutex_init(&connMutex, NULL);
@@ -341,7 +344,7 @@ zkresult Database64::write(const string &_key, const Goldilocks::Element* vkey,
     return r;
 }
 
-zkresult Database64::write (vector<DB64Query> &dbQueries, const bool persistent)
+zkresult Database64::write(vector<DB64Query> &dbQueries, const bool persistent)
 {
     zkresult zkr;
     for (uint64_t i=0; i<dbQueries.size(); i++)
@@ -357,6 +360,356 @@ zkresult Database64::write (vector<DB64Query> &dbQueries, const bool persistent)
     return ZKR_SUCCESS;
 }
 
+zkresult Database64::readKV(const Goldilocks::Element (&root)[4], const Goldilocks::Element (&key)[4], mpz_class &value, DatabaseMap *dbReadLog)
+{
+    // Check that it has been initialized before
+    if (!bInitialized)
+    {
+        zklog.error("Database64::readKV() called uninitialized");
+        exitProcess();
+    }
+
+    struct timeval t;
+    if (dbReadLog != NULL) gettimeofday(&t, NULL);
+
+    zkresult rkv = ZKR_UNSPECIFIED;
+    zkresult rv = ZKR_UNSPECIFIED;
+
+    string keyStr = "";
+    if(dbReadLog->getSaveKeys()){
+        string keyStr_ = fea2string(fr, key[0], key[1], key[2], key[3]); 
+        keyStr = NormalizeToNFormat(keyStr_, 64);
+    }
+    
+    uint64_t version;
+    rv = readVersion(root, version, dbReadLog);
+
+    if( rv == ZKR_SUCCESS){
+       
+        // If the key is found in local database (cached) simply return it
+        if(dbKVACache.findKey(version, key, value)){
+
+            if (dbReadLog != NULL) dbReadLog->add(keyStr, value, true, TimeDiff(t));
+            rkv = ZKR_SUCCESS;
+
+        } 
+        #if 0 //MULTIWRITE PART
+        // If the key is pending to be stored in database, but already deleted from cache
+        else if (config.dbMultiWrite && multiWrite.findNode(key, value))
+        {
+            // Add to the read log
+            if (dbReadLog != NULL) dbReadLog->add(key, value, true, TimeDiff(t));
+
+            // Store it locally to avoid any future remote access for this key
+            if(usingAssociativeCache()){
+                dbMTACache.addKeyValue(vkey, value, false);
+            }
+            else if(dbMTCache.enabled()){                
+                dbMTCache.add(key, value, false);
+            }
+            r = ZKR_SUCCESS;
+        }
+        #endif
+        else if(useRemoteDB)
+        {
+        
+            rkv = readRemoteKV(version, key, value);
+            if ( (rkv != ZKR_SUCCESS) && (config.dbReadRetryDelay > 0) )
+            {
+                if(!dbReadLog->getSaveKeys()){
+                    string keyStr_ = fea2string(fr, key[0], key[1], key[2], key[3]); 
+                    keyStr = NormalizeToNFormat(keyStr_, 64);
+                }
+                for (uint64_t i=0; i<config.dbReadRetryCounter; i++)
+                {
+                    
+                    zklog.warning("Database64::readKV() failed calling readRemote() with error=" + zkresult2string(rkv) + "; will retry after " + to_string(config.dbReadRetryDelay) + "us key=" + keyStr + " i=" + to_string(i));
+
+                    // Retry after dbReadRetryDelay us
+                    usleep(config.dbReadRetryDelay);
+                    rkv = readRemoteKV(version, key, value);
+                    if (rkv == ZKR_SUCCESS)
+                    {
+                        break;
+                    }
+                    zklog.warning("Database64::readKV() retried readRemote() after dbReadRetryDelay=" + to_string(config.dbReadRetryDelay) + "us and failed with error=" + zkresult2string(rkv) + " i=" + to_string(i));
+                }
+            }        
+            if (rkv == ZKR_SUCCESS)
+            {
+                dbKVACache.addKeyValueVersion(version, key, value, false);                
+                if (dbReadLog != NULL) dbReadLog->add(keyStr, value, false, TimeDiff(t));
+            }
+        }
+        // If we could not find the value, report the error
+        if (rv != ZKR_SUCCESS || rkv != ZKR_SUCCESS)
+        {
+            zklog.warning("Database64::readKV() requested a key that does not exist (ZKR_DB_KEY_NOT_FOUND): " + keyStr + ", now trying to read from the hasDB");
+            string valueStr;
+            rkv = read(keyStr, key, valueStr, dbReadLog);
+            if(rkv == ZKR_SUCCESS){
+                value = mpz_class(valueStr, 16);
+                writeKV(root, key, value, true);
+                if (dbReadLog != NULL) dbReadLog->add(keyStr, value, false, TimeDiff(t));
+            }else{
+                zklog.error("Database64::readKV() requested a key in that does not exist even in hasdb (ZKR_DB_KEY_NOT_FOUND): " + keyStr);
+                rkv = ZKR_DB_KEY_NOT_FOUND;
+            }
+        }
+    }
+#ifdef LOG_DB_READ
+    {
+        string s = "Database64::readKV()";
+        if (rkv != ZKR_SUCCESS)
+            s += " ERROR=" + zkresult2string(rkv);
+        s += " key=" + keyStr;
+        s += " value=";
+        s += value.get_str(16) + ";";
+        zklog.info(s);
+    }
+#endif
+    return rkv;
+
+}
+
+zkresult Database64::readKV(const Goldilocks::Element (&root)[4], vector<KeyValue> &KVs, DatabaseMap *dbReadLog){
+    zkresult zkr;
+    for (uint64_t i=0; i<KVs.size(); i++)
+    {
+        zkr = readKV(root, KVs[i].key.fe, KVs[i].value, dbReadLog);
+        if (zkr != ZKR_SUCCESS)
+        {
+            zklog.error("Database64::readKV(KBs) failed calling read() result=" + zkresult2string(zkr) + " key=" + fea2string(fr, KVs[i].key.fe[0], KVs[i].key.fe[1], KVs[i].key.fe[2], KVs[i].key.fe[3]) );
+            return zkr;
+        }
+    }
+    return ZKR_SUCCESS;
+}
+
+zkresult Database64::writeKV(const Goldilocks::Element (&root)[4], const Goldilocks::Element (&key)[4], const mpz_class &value, bool persistent)
+{
+
+    // Check that it has  been initialized before
+    if (!bInitialized)
+    {
+        zklog.error("Database64::writeKV() called uninitialized");
+        exitProcess();
+    }
+    if (config.dbMultiWrite && !dbKVACache.enabled() && !persistent)
+    {
+        zklog.error("Database64::writeKV() called with multi-write active, cache disabled and no persistance in database, so there is no place to store the date");
+        return ZKR_DB_ERROR;
+    }
+    zkresult rkv = ZKR_UNSPECIFIED;
+    zkresult rv = ZKR_UNSPECIFIED;
+
+    uint64_t version;
+    rv = readVersion(root, version, NULL);
+
+    if( rv == ZKR_SUCCESS){
+
+        if ( useRemoteDB && persistent)
+        {
+            
+            rkv = writeRemoteKV(version, key, value);
+        }
+        else
+        {
+            rkv = ZKR_SUCCESS;
+        }
+
+        if (rkv == ZKR_SUCCESS)
+        {
+            dbKVACache.addKeyValueVersion(version, key, value, false);
+            
+        }
+    }else{
+       rkv=rv;
+    }
+
+#ifdef LOG_DB_WRITE
+    {
+        string keyStr_ = fea2string(fr, key[0], key[1], key[2], key[3]); 
+        string keyStr = NormalizeToNFormat(keyStr_, 64);
+        string s = "Database64::writeKV()";
+        if (rkv != ZKR_SUCCESS)
+            s += " ERROR=" + zkresult2string(rkv);
+        s += " key=" + keyStr;
+        s += " value=";
+        s += value.get_str(16);
+        s += " persistent=" + to_string(persistent);
+        zklog.info(s);
+    }
+#endif
+
+    return rkv;
+}
+
+zkresult Database64::writeKV(const Goldilocks::Element (&root)[4], const vector<KeyValue> &KVs, bool persistent){
+    zkresult zkr;
+    for (uint64_t i=0; i<KVs.size(); i++)
+    {
+        zkr = writeKV(root, KVs[i].key.fe, KVs[i].value, persistent);
+        if (zkr != ZKR_SUCCESS)
+        {
+            zklog.error("Database64::writeKV(KBs) failed calling write() result=" + zkresult2string(zkr) + " key=" + fea2string(fr, KVs[i].key.fe[0], KVs[i].key.fe[1], KVs[i].key.fe[2], KVs[i].key.fe[3]) );
+            return zkr;
+        }
+    }
+    return ZKR_SUCCESS;
+}
+
+zkresult Database64::readVersion(const Goldilocks::Element (&root)[4], uint64_t& version, DatabaseMap *dbReadLog){
+
+    // Check that it has been initialized before
+    if (!bInitialized)
+    {
+        zklog.error("Database64::readVersion() called uninitialized");
+        exitProcess();
+    }
+
+    struct timeval t;
+    if (dbReadLog != NULL) gettimeofday(&t, NULL);
+   
+    zkresult r = ZKR_UNSPECIFIED;
+
+    string rootStr = "";
+    if(dbReadLog->getSaveKeys()){
+        string rootStr_ = fea2string(fr, root[0], root[1], root[2], root[3]); 
+        rootStr = NormalizeToNFormat(rootStr_, 64);
+    }
+
+    // If the key is found in local database (cached) simply return it
+    if(dbVersionACache.findKey(root, version)){
+
+        if (dbReadLog != NULL) dbReadLog->add(rootStr, version, true, TimeDiff(t));
+        r = ZKR_SUCCESS;
+
+    } 
+#if 0 //MULTIWRITE PART
+    // If the key is pending to be stored in database, but already deleted from cache
+    else if (config.dbMultiWrite && multiWrite.findNode(key, value))
+    {
+        // Add to the read log
+        if (dbReadLog != NULL) dbReadLog->add(key, value, true, TimeDiff(t));
+
+        // Store it locally to avoid any future remote access for this key
+        if(usingAssociativeCache()){
+            dbMTACache.addKeyValue(vkey, value, false);
+        }
+        else if(dbMTCache.enabled()){                
+            dbMTCache.add(key, value, false);
+        }
+        r = ZKR_SUCCESS;
+    }
+#endif
+    else if(useRemoteDB)
+    {
+    
+        r = readRemoteVersion(root, version);
+        if ( (r != ZKR_SUCCESS) && (config.dbReadRetryDelay > 0) )
+        {
+            for (uint64_t i=0; i<config.dbReadRetryCounter; i++)
+            {
+                if(!dbReadLog->getSaveKeys()){
+                    string rootStr_ = fea2string(fr, root[0], root[1], root[2], root[3]); 
+                    rootStr = NormalizeToNFormat(rootStr_, 64);
+                }
+                zklog.warning("Database64::readVersion() failed calling readRemote() with error=" + zkresult2string(r) + "; will retry after " + to_string(config.dbReadRetryDelay) + "us key=" + rootStr + " i=" + to_string(i));
+
+                // Retry after dbReadRetryDelay us
+                usleep(config.dbReadRetryDelay);
+                r = readRemoteVersion(root, version);
+                if (r == ZKR_SUCCESS)
+                {
+                    break;
+                }
+                zklog.warning("Database64::readVersion() retried readRemote() after dbReadRetryDelay=" + to_string(config.dbReadRetryDelay) + "us and failed with error=" + zkresult2string(r) + " i=" + to_string(i));
+            }
+        }        
+        if (r == ZKR_SUCCESS)
+        {
+            dbVersionACache.addKeyVersion(root, version);
+            
+            // Add to the read log
+            if (dbReadLog != NULL) dbReadLog->add(rootStr, version, false, TimeDiff(t));
+        }
+    }
+
+    // If we could not find the value, report the error
+    if (r == ZKR_UNSPECIFIED)
+    {
+        zklog.error("Database64::readVersion() requested a key that does not exist (ZKR_DB_KEY_NOT_FOUND): " + rootStr);
+        r = ZKR_DB_KEY_NOT_FOUND;
+    }
+
+#ifdef LOG_DB_READ 
+    {
+        if(!dbReadLog->getSaveKeys()){
+            string rootStr_ = fea2string(fr, root[0], root[1], root[2], root[3]); 
+            rootStr = NormalizeToNFormat(rootStr_, 64);
+        }
+        string s = "Database64::readKV()";
+        if (r != ZKR_SUCCESS)
+            s += " ERROR=" + zkresult2string(r);
+        s += " key=" + rootStr;
+        s += " version=";
+        s += to_string(version) + ":";
+        zklog.info(s);
+    }
+#endif
+    return r;
+}
+
+zkresult Database64::writeVersion(const Goldilocks::Element (&root)[4], const uint64_t version, bool persistent){
+
+    if (!bInitialized)
+    {
+        zklog.error("Database64::writeVersion() called uninitialized");
+        exitProcess();
+    }
+    if (config.dbMultiWrite && ! dbVersionACache.enabled() && !persistent)
+    {
+        zklog.error("Database64::writeVersion() called with multi-write active, cache disabled and no persistance in database, so there is no place to store the date");
+        return ZKR_DB_ERROR;
+    }
+    zkresult r;
+
+
+    if ( useRemoteDB && persistent)
+    {
+        
+        r = writeRemoteVersion(root, version);
+    }
+    else
+    {
+        r = ZKR_SUCCESS;
+    }
+
+    if (r == ZKR_SUCCESS)
+    {
+        dbVersionACache.addKeyVersion(root, version);
+        
+    }
+
+#ifdef LOG_DB_WRITE
+    {
+        string rootStr_ = fea2string(fr, root[0], root[1], root[2], root[3]); 
+        string rootStr = NormalizeToNFormat(rootStr_, 64);
+        string s = "Database64::writeKV()";
+        if (r != ZKR_SUCCESS)
+            s += " ERROR=" + zkresult2string(r);
+        s += " root=" + rootStr;
+        s += " version=";
+        s += to_string(version);
+        s += " persistent=" + to_string(persistent);
+        zklog.info(s);
+    }
+#endif
+
+    return r;
+}
+
 void Database64::initRemote(void)
 {
     TimerStart(DB_INIT_REMOTE);
@@ -776,6 +1129,353 @@ zkresult Database64::writeRemote(bool bProgram, const string &key, const string
     return result;
 }
 
+zkresult Database64::readRemoteKV(const uint64_t version, const Goldilocks::Element (&key)[4],  mpz_class value)
+{
+    const string &tableName = config.dbKeyValueTableName;
+
+    string keyStr_ = fea2string(fr, key[0], key[1], key[2], key[3]); 
+    string keyStr = NormalizeToNFormat(keyStr_, 64);
+
+    if (config.logRemoteDbReads)
+    {
+        zklog.info("Database64::readRemoteKV() table=" + tableName + " key=" + keyStr);
+    }
+
+    // Get a free read db connection
+    DatabaseConnection * pDatabaseConnection = getConnection();
+
+    try
+    {
+        // Prepare the query
+        string query = "SELECT * FROM " + tableName + " WHERE key = E\'\\\\x" + keyStr + "\';";
+
+        pqxx::result rows;
+
+        // Start a transaction.
+        pqxx::nontransaction n(*(pDatabaseConnection->pConnection));
+
+        // Execute the query
+        rows = n.exec(query);
+
+        // Commit your transaction
+        n.commit();
+
+        // Process the result
+        if (rows.size() == 0)
+        {
+            disposeConnection(pDatabaseConnection);
+            return ZKR_DB_KEY_NOT_FOUND;
+        }
+        else if (rows.size() > 1)
+        {
+            zklog.error("Database64::readRemoteKV() table=" + tableName + " got more than one row for the same key: " + to_string(rows.size()) + "for key=" + keyStr);
+            exitProcess();
+        }
+
+        pqxx::row const row = rows[0];
+        if (row.size() != 2)
+        {
+            zklog.error("DatabaseKV::readRemoteKV() table=" + tableName + " got an invalid number of colums for the row: " + to_string(row.size()) + "for key=" + keyStr);
+            exitProcess();
+        }
+        bool foundVersion = extractVersion(row[1], version, value); 
+        if(foundVersion==false){
+            return ZKR_DB_VERSION_NOT_FOUND;
+        }
+
+    }
+    catch (const std::exception &e)
+    {
+        zklog.error("Database64::readRemoteKV() table=" + tableName + " exception: " + string(e.what()) + " connection=" + to_string((uint64_t)pDatabaseConnection));
+        queryFailed();
+        disposeConnection(pDatabaseConnection);
+        return ZKR_DB_ERROR;
+    }
+    
+    // Dispose the read db conneciton
+    disposeConnection(pDatabaseConnection);
+
+    return ZKR_SUCCESS;
+}
+
+bool Database64::extractVersion(const pqxx::field& fieldData, const uint64_t version, mpz_class &value){
+
+    if(!fieldData.is_null()){
+        string data = removeBSXIfExists64(fieldData.c_str());
+        int data_size = data.size();
+        zkassert(data_size % 80 == 0);
+        for (int i = 0; i < data_size; i += 80)
+        {
+            string versionStr = data.substr(i, 16);
+            mpz_class aux(versionStr, 16);
+            uint64_t version_ = aux.get_ui();
+            if(version_==version){
+                value = mpz_class(data.substr(i + 16, 64), 16);
+                return true;
+            }
+        }
+    }
+    /*const char * data = fieldData.c_str();
+    int data_size = fieldData.size();
+    zkassert(data_size % 40 == 0);
+    for (int i = 0; i < data_size; i += 40)
+    {
+        uint64_t version_;
+        std::memcpy(&version_, data + i, 8);
+        if(version_==version){
+            value = mpz_class(data + i + 8, 32);
+            return true;
+        }
+    }*/
+    return false;
+}
+
+zkresult Database64::writeRemoteKV(const uint64_t version, const Goldilocks::Element (&key)[4], const mpz_class &value)
+{
+    zkresult result = ZKR_SUCCESS;
+    
+    
+    if (config.dbMultiWrite)
+    {
+        multiWrite.Lock();
+
+        /*
+            PENDING: rick
+        */
+
+        multiWrite.Unlock();
+    }
+    else
+    {
+        const string &tableName = config.dbKeyValueTableName;
+
+        string keyStr_ = fea2string(fr, key[0], key[1], key[2], key[3]); 
+        string keyStr = NormalizeToNFormat(keyStr_, 64);
+        
+        string insertStr = to_string(version) + value.get_str(16);
+        assert(insertStr.size() == 80);
+
+        if (config.logRemoteDbReads)
+        {
+            zklog.info("Database64::writeRemoteKV() table=" + tableName + " key=" + keyStr);
+        }
+
+        // Get a free read db connection
+        DatabaseConnection * pDatabaseConnection = getConnection();
+
+        try
+        {
+            // Prepare the query
+            string query = "SELECT * FROM " + tableName + " WHERE key = E\'\\\\x" + keyStr + "\';";
+
+            pqxx::result rows;
+
+            // Start a transaction.
+            pqxx::nontransaction n(*(pDatabaseConnection->pConnection));
+
+            // Execute the query
+            rows = n.exec(query);
+
+            // Commit your transaction
+            n.commit();
+
+            // Process the result
+            if (rows.size() == 0)
+            {
+                disposeConnection(pDatabaseConnection);
+                return ZKR_DB_KEY_NOT_FOUND;
+            }
+            else if (rows.size() > 1)
+            {
+                zklog.error("Database64::writeRemoteKV() table=" + tableName + " got more than one row for the same key: " + to_string(rows.size()) + "for key=" + keyStr);
+                exitProcess();
+            }
+
+            const pqxx::row&  row = rows[0];
+            if (row.size() != 2)
+            {
+                zklog.error("DatabaseKV::writeRemoteKV() table=" + tableName + " got an invalid number of colums for the row: " + to_string(row.size()) + "for key=" + keyStr);
+                exitProcess();
+            }
+            const pqxx::field& fieldData = row[1];
+            //processar insert;
+            if(!fieldData.is_null()){
+                string data = removeBSXIfExists64(fieldData.c_str());
+                int data_size = data.size();
+                if(data_size == maxVersions*80){
+                    data = data.substr(0, data_size - 80);
+                }
+                insertStr = insertStr + data;
+            }
+        }
+        catch (const std::exception &e)
+        {
+            zklog.error("Database64::writeRemoteKV() table=" + tableName + " exception: " + string(e.what()) + " connection=" + to_string((uint64_t)pDatabaseConnection));
+            queryFailed();
+            disposeConnection(pDatabaseConnection);
+            return ZKR_DB_ERROR;
+        }
+
+            
+        try
+        {        
+            string query = "INSERT INTO " + tableName + " ( key, data ) VALUES ( E\'\\\\x" + keyStr + "\', E\'\\\\x" + insertStr + "\' ) ON CONFLICT (key) DO UPDATE SET data = E'\\\\x" + insertStr + "\';";
+#ifdef DATABASE_COMMIT
+            if (autoCommit)
+#endif
+            {
+                pqxx::work w(*(pDatabaseConnection->pConnection));
+                pqxx::result res = w.exec(query);
+                w.commit();
+            }
+#ifdef DATABASE_COMMIT
+            else
+            {
+                if (transaction == NULL)
+                    transaction = new pqxx::work{*pConnectionWrite};
+                pqxx::result res = transaction->exec(query);
+            }
+#endif
+        }
+        catch (const std::exception &e)
+        {
+            zklog.error("Database::writeRemoteVK() table=" + tableName + " exception: " + string(e.what()) + " connection=" + to_string((uint64_t)pDatabaseConnection));
+            result = ZKR_DB_ERROR;
+            queryFailed();
+        }
+
+        disposeConnection(pDatabaseConnection);
+
+    }
+    return result;
+}
+
+zkresult Database64::readRemoteVersion(const Goldilocks::Element (&root)[4], uint64_t version){
+    
+    const string &tableName = config.dbVersionTableName;
+
+    string rootStr_ = fea2string(fr, root[0], root[1], root[2], root[3]); 
+    string rootStr = NormalizeToNFormat(rootStr_, 64);
+
+    if (config.logRemoteDbReads)
+    {
+        zklog.info("Database64::readRemoteVersion() table=" + tableName + " key=" + rootStr);
+    }
+
+    // Get a free read db connection
+    DatabaseConnection * pDatabaseConnection = getConnection();
+
+    try
+    {
+        // Prepare the query
+        string query = "SELECT * FROM " + tableName + " WHERE hash = E\'\\\\x" + rootStr + "\';";
+
+        pqxx::result rows;
+
+        // Start a transaction.
+        pqxx::nontransaction n(*(pDatabaseConnection->pConnection));
+
+        // Execute the query
+        rows = n.exec(query);
+
+        // Commit your transaction
+        n.commit();
+
+        // Process the result
+        if (rows.size() == 0)
+        {
+            disposeConnection(pDatabaseConnection);
+            return ZKR_DB_KEY_NOT_FOUND;
+        }
+        else if (rows.size() > 1)
+        {
+            zklog.error("Database64::readRemoteVersion() table=" + tableName + " got more than one row for the same key: " + to_string(rows.size()) + "for key: " + rootStr);
+            exitProcess();
+        }
+
+        pqxx::row const row = rows[0];
+        if (row.size() != 2)
+        {
+            zklog.error("DatabaseKV::readRemoteVersion() table=" + tableName + " got an invalid number of colums for the row: " + to_string(row.size()) + "for key: " + rootStr);
+            exitProcess();
+        }
+        pqxx::field const fieldData = row[1];
+        if (!fieldData.is_null()) {
+            fieldData.as<uint64_t>(version);
+        } else {
+            zklog.error("DatabaseKV::readRemoteVersion() table=" + tableName + " got a null version for root: " + rootStr);
+        }
+    }
+    catch (const std::exception &e)
+    {
+        zklog.error("Database64::readRemoteVersion() table=" + tableName + " exception: " + string(e.what()) + " connection=" + to_string((uint64_t)pDatabaseConnection));
+        queryFailed();
+        disposeConnection(pDatabaseConnection);
+        return ZKR_DB_ERROR;
+    }
+    
+    // Dispose the read db conneciton
+    disposeConnection(pDatabaseConnection);
+
+    return ZKR_SUCCESS;
+}
+
+zkresult Database64::writeRemoteVersion(const Goldilocks::Element (&root)[4], const uint64_t version){
+    
+    zkresult result = ZKR_SUCCESS;
+    string rootStr_ = fea2string(fr, root[0], root[1], root[2], root[3]); 
+    string rootStr = NormalizeToNFormat(rootStr_, 64);
+    
+    
+    if (config.dbMultiWrite)
+    {
+        multiWrite.Lock();
+
+        /*
+            rick: pending
+        */
+
+        multiWrite.Unlock();
+    }
+    else
+    {
+        const string &tableName =  config.dbVersionTableName;
+        string query = "INSERT INTO " + tableName + " ( hash, version ) VALUES ( E\'\\\\x" + rootStr + "\', E\'\\\\x" + to_string(version) + "\' ) ON CONFLICT (hash) DO NOTHING;";
+        DatabaseConnection * pDatabaseConnection = getConnection();
+
+        try
+        {        
+
+#ifdef DATABASE_COMMIT
+            if (autoCommit)
+#endif
+            {
+                pqxx::work w(*(pDatabaseConnection->pConnection));
+                pqxx::result res = w.exec(query);
+                w.commit();
+            }
+#ifdef DATABASE_COMMIT
+            else
+            {
+                if (transaction == NULL)
+                    transaction = new pqxx::work{*pConnectionWrite};
+                pqxx::result res = transaction->exec(query);
+            }
+#endif
+        }
+        catch (const std::exception &e)
+        {
+            zklog.error("Database::writeRemoteVersion() table=" + tableName + " exception: " + string(e.what()) + " connection=" + to_string((uint64_t)pDatabaseConnection));
+            result = ZKR_DB_ERROR;
+            queryFailed();
+        }
+
+        disposeConnection(pDatabaseConnection);
+    }
+
+    return result;
+}
+
 zkresult Database64::createStateRoot(void)
 {
     // Copy the state root in the first 4 elements of dbValue
diff --git a/src/hashdb64/database_64.hpp b/src/hashdb64/database_64.hpp
index 5239d95fa..eb65d348c 100644
--- a/src/hashdb64/database_64.hpp
+++ b/src/hashdb64/database_64.hpp
@@ -14,6 +14,9 @@
 #include "database_connection.hpp"
 #include "zkassert.hpp"
 #include "multi_write_64.hpp"
+#include "database_versions_associtive_cache.hpp"
+#include "database_kv_associative_cache.hpp"
+#include "key_value.hpp"
 
 using namespace std;
 
@@ -70,6 +73,7 @@ class Database64
 private:
     pthread_t senderPthread; // Database sender thread
     pthread_t cacheSynchPthread; // Cache synchronization thread
+    int maxVersions; // Maximum number of versions to store in the database KV
 
 private:
     // Remote database based on Postgres (PostgreSQL)
@@ -79,11 +83,20 @@ class Database64
     zkresult writeRemote(bool bProgram, const string &key, const string &value);
     zkresult writeGetTreeFunction(void);
 
+    zkresult readRemoteKV(const uint64_t version, const Goldilocks::Element (&key)[4],  mpz_class value); 
+    zkresult writeRemoteKV(const uint64_t version, const Goldilocks::Element (&key)[4], const mpz_class &value);
+    zkresult readRemoteVersion(const Goldilocks::Element (&root)[4], uint64_t version);
+    zkresult writeRemoteVersion(const Goldilocks::Element (&root)[4], const uint64_t version); 
+
+    bool extractVersion(const pqxx::field& fieldData, const uint64_t version, mpz_class &value);
+
 public:
 #ifdef DATABASE_USE_CACHE
     // Cache static instances
     static DatabaseMTCache64 dbMTCache;
     static DatabaseProgramCache64 dbProgramCache;
+    static DatabaseKVAssociativeCache dbKVACache;
+    static DatabaseVersionsAssociativeCache dbVersionACache;
 
     // This is a fixed key to store the latest state root hash, used to load it to the cache
     // This key is "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
@@ -103,8 +116,15 @@ class Database64
     zkresult read(vector<DB64Query> &dbQueries);
     zkresult write(const string &_key, const Goldilocks::Element* vkey, const string &value, const bool persistent);
     zkresult write(vector<DB64Query> &dbQueries, const bool persistent);
-     zkresult getProgram(const string &_key, vector<uint8_t> &value, DatabaseMap *dbReadLog);
+    zkresult getProgram(const string &_key, vector<uint8_t> &value, DatabaseMap *dbReadLog);
     zkresult setProgram(const string &_key, const vector<uint8_t> &value, const bool persistent);
+    zkresult readKV(const Goldilocks::Element (&root)[4], const Goldilocks::Element (&key)[4], mpz_class &value, DatabaseMap *dbReadLog); 
+    zkresult readKV(const Goldilocks::Element (&root)[4], vector<KeyValue> &KVs, DatabaseMap *dbReadLog);
+    zkresult writeKV(const Goldilocks::Element (&root)[4], const Goldilocks::Element (&key)[4], const mpz_class &value, bool persistent);
+    zkresult writeKV(const Goldilocks::Element (&root)[4], const vector<KeyValue> &KVs, bool persistent);
+    zkresult readVersion(const Goldilocks::Element (&root)[4], uint64_t& version, DatabaseMap *dbReadLog);
+    zkresult writeVersion(const Goldilocks::Element (&root)[4], const uint64_t version, bool persistent);
+
 
 private:
     zkresult createStateRoot(void);
diff --git a/src/hashdb64/database_kv_associative_cache.hpp b/src/hashdb64/database_kv_associative_cache.hpp
index e710ce080..b4f128467 100644
--- a/src/hashdb64/database_kv_associative_cache.hpp
+++ b/src/hashdb64/database_kv_associative_cache.hpp
@@ -22,7 +22,7 @@ class DatabaseKVAssociativeCache
         uint32_t *indexes;
         Goldilocks::Element *keys;
         uint64_t *versions;
-        mpz_class *values; //rick: this I do not like
+        mpz_class *values;
         uint32_t currentCacheIndex; 
 
         uint64_t attempts;
diff --git a/src/hashdb64/database_versions_associtive_cache.cpp b/src/hashdb64/database_versions_associtive_cache.cpp
index 15945a925..997844291 100644
--- a/src/hashdb64/database_versions_associtive_cache.cpp
+++ b/src/hashdb64/database_versions_associtive_cache.cpp
@@ -84,7 +84,7 @@ void DatabaseVersionsAssociativeCache::postConstruct(int log2IndexesSize_, int l
     indexesMask = indexesSize - 1;
 };
 
-void DatabaseVersionsAssociativeCache::addKeyVersion(Goldilocks::Element (&key)[4], const uint64_t version)
+void DatabaseVersionsAssociativeCache::addKeyVersion(const Goldilocks::Element (&key)[4], const uint64_t version)
 {
     
     lock_guard<recursive_mutex> guard(mlock);
diff --git a/src/hashdb64/database_versions_associtive_cache.hpp b/src/hashdb64/database_versions_associtive_cache.hpp
index ffec03e9d..b4fb03b07 100644
--- a/src/hashdb64/database_versions_associtive_cache.hpp
+++ b/src/hashdb64/database_versions_associtive_cache.hpp
@@ -38,7 +38,7 @@ class DatabaseVersionsAssociativeCache
         ~DatabaseVersionsAssociativeCache();
         void postConstruct(int log2IndexesSize_, int log2CacheSize_, string name_);
 
-        void addKeyVersion(Goldilocks::Element (&key)[4], const uint64_t version); //rick: no update!
+        void addKeyVersion(const Goldilocks::Element (&key)[4], const uint64_t version); //rick: no update!
         bool findKey(const Goldilocks::Element (&key)[4], uint64_t &version);
         
         inline bool enabled() const { return (log2IndexesSize > 0); };

From a61f967b3bb731934a6906ecc8b6d716e8d37638 Mon Sep 17 00:00:00 2001
From: rickb80 <rickborrell@gmail.com>
Date: Sat, 2 Sep 2023 19:19:19 +0200
Subject: [PATCH 13/17] Add multiwrites

---
 src/hashdb64/database_64.cpp         | 209 ++++++++++++++++-----------
 src/hashdb64/database_64.hpp         |   2 +-
 src/hashdb64/multi_write_64.cpp      | 125 ++++++++++++++++
 src/hashdb64/multi_write_64.hpp      |   2 +
 src/hashdb64/multi_write_data_64.hpp |  37 ++++-
 5 files changed, 286 insertions(+), 89 deletions(-)

diff --git a/src/hashdb64/database_64.cpp b/src/hashdb64/database_64.cpp
index 1cb46f691..2b1fd68bb 100644
--- a/src/hashdb64/database_64.cpp
+++ b/src/hashdb64/database_64.cpp
@@ -14,6 +14,7 @@
 #include "exit_process.hpp"
 #include "zkmax.hpp"
 #include "hashdb_remote.hpp"
+#include "key_value.hpp"
 
 #ifdef DATABASE_USE_CACHE
 
@@ -380,6 +381,7 @@ zkresult Database64::readKV(const Goldilocks::Element (&root)[4], const Goldiloc
         string keyStr_ = fea2string(fr, key[0], key[1], key[2], key[3]); 
         keyStr = NormalizeToNFormat(keyStr_, 64);
     }
+    KeyValue mwEntry;
     
     uint64_t version;
     rv = readVersion(root, version, dbReadLog);
@@ -393,23 +395,17 @@ zkresult Database64::readKV(const Goldilocks::Element (&root)[4], const Goldiloc
             rkv = ZKR_SUCCESS;
 
         } 
-        #if 0 //MULTIWRITE PART
         // If the key is pending to be stored in database, but already deleted from cache
-        else if (config.dbMultiWrite && multiWrite.findNode(key, value))
+        else if (config.dbMultiWrite && multiWrite.findKeyValue(version, mwEntry))
         {
+            value = mwEntry.value;
             // Add to the read log
-            if (dbReadLog != NULL) dbReadLog->add(key, value, true, TimeDiff(t));
+            if (dbReadLog != NULL) dbReadLog->add(keyStr, value, true, TimeDiff(t));
 
             // Store it locally to avoid any future remote access for this key
-            if(usingAssociativeCache()){
-                dbMTACache.addKeyValue(vkey, value, false);
-            }
-            else if(dbMTCache.enabled()){                
-                dbMTCache.add(key, value, false);
-            }
-            r = ZKR_SUCCESS;
+            dbKVACache.addKeyValueVersion(version, key, value, false);                
+            rkv = ZKR_SUCCESS;
         }
-        #endif
         else if(useRemoteDB)
         {
         
@@ -441,20 +437,20 @@ zkresult Database64::readKV(const Goldilocks::Element (&root)[4], const Goldiloc
                 if (dbReadLog != NULL) dbReadLog->add(keyStr, value, false, TimeDiff(t));
             }
         }
-        // If we could not find the value, report the error
-        if (rv != ZKR_SUCCESS || rkv != ZKR_SUCCESS)
-        {
-            zklog.warning("Database64::readKV() requested a key that does not exist (ZKR_DB_KEY_NOT_FOUND): " + keyStr + ", now trying to read from the hasDB");
-            string valueStr;
-            rkv = read(keyStr, key, valueStr, dbReadLog);
-            if(rkv == ZKR_SUCCESS){
-                value = mpz_class(valueStr, 16);
-                writeKV(root, key, value, true);
-                if (dbReadLog != NULL) dbReadLog->add(keyStr, value, false, TimeDiff(t));
-            }else{
-                zklog.error("Database64::readKV() requested a key in that does not exist even in hasdb (ZKR_DB_KEY_NOT_FOUND): " + keyStr);
-                rkv = ZKR_DB_KEY_NOT_FOUND;
-            }
+    }
+    // If we could not find the value, report the error
+    if (rv != ZKR_SUCCESS || rkv != ZKR_SUCCESS)
+    {
+        zklog.warning("Database64::readKV() requested a key that does not exist (ZKR_DB_KEY_NOT_FOUND): " + keyStr + ", now trying to read from the hasDB");
+        string valueStr;
+        rkv = read(keyStr, key, valueStr, dbReadLog);
+        if(rkv == ZKR_SUCCESS){
+            value = mpz_class(valueStr, 16);
+            writeKV(root, key, value, true);
+            if (dbReadLog != NULL) dbReadLog->add(keyStr, value, false, TimeDiff(t));
+        }else{
+            zklog.error("Database64::readKV() requested a key in that does not exist even in hasdb (ZKR_DB_KEY_NOT_FOUND): " + keyStr);
+            rkv = ZKR_DB_KEY_NOT_FOUND;
         }
     }
 #ifdef LOG_DB_READ
@@ -585,57 +581,49 @@ zkresult Database64::readVersion(const Goldilocks::Element (&root)[4], uint64_t&
         if (dbReadLog != NULL) dbReadLog->add(rootStr, version, true, TimeDiff(t));
         r = ZKR_SUCCESS;
 
-    } 
-#if 0 //MULTIWRITE PART
-    // If the key is pending to be stored in database, but already deleted from cache
-    else if (config.dbMultiWrite && multiWrite.findNode(key, value))
-    {
-        // Add to the read log
-        if (dbReadLog != NULL) dbReadLog->add(key, value, true, TimeDiff(t));
-
-        // Store it locally to avoid any future remote access for this key
-        if(usingAssociativeCache()){
-            dbMTACache.addKeyValue(vkey, value, false);
-        }
-        else if(dbMTCache.enabled()){                
-            dbMTCache.add(key, value, false);
-        }
-        r = ZKR_SUCCESS;
-    }
-#endif
-    else if(useRemoteDB)
-    {
-    
-        r = readRemoteVersion(root, version);
-        if ( (r != ZKR_SUCCESS) && (config.dbReadRetryDelay > 0) )
+    }else{
+        if(!dbReadLog->getSaveKeys()){
+            string rootStr_ = fea2string(fr, root[0], root[1], root[2], root[3]); 
+            rootStr = NormalizeToNFormat(rootStr_, 64);
+        } 
+        // If the key is pending to be stored in database, but already deleted from cache
+        if (config.dbMultiWrite && multiWrite.findVersion(rootStr, version))
         {
-            for (uint64_t i=0; i<config.dbReadRetryCounter; i++)
-            {
-                if(!dbReadLog->getSaveKeys()){
-                    string rootStr_ = fea2string(fr, root[0], root[1], root[2], root[3]); 
-                    rootStr = NormalizeToNFormat(rootStr_, 64);
-                }
-                zklog.warning("Database64::readVersion() failed calling readRemote() with error=" + zkresult2string(r) + "; will retry after " + to_string(config.dbReadRetryDelay) + "us key=" + rootStr + " i=" + to_string(i));
+            // Add to the read log
+            if (dbReadLog != NULL) dbReadLog->add(rootStr, version, true, TimeDiff(t));
 
-                // Retry after dbReadRetryDelay us
-                usleep(config.dbReadRetryDelay);
-                r = readRemoteVersion(root, version);
-                if (r == ZKR_SUCCESS)
+            // Store it locally to avoid any future remote access for this key
+            
+            r = ZKR_SUCCESS;
+
+        }else if(useRemoteDB){
+        
+            r = readRemoteVersion(root, version);
+            if ( (r != ZKR_SUCCESS) && (config.dbReadRetryDelay > 0) )
+            {
+                for (uint64_t i=0; i<config.dbReadRetryCounter; i++)
                 {
-                    break;
+                    zklog.warning("Database64::readVersion() failed calling readRemote() with error=" + zkresult2string(r) + "; will retry after " + to_string(config.dbReadRetryDelay) + "us key=" + rootStr + " i=" + to_string(i));
+
+                    // Retry after dbReadRetryDelay us
+                    usleep(config.dbReadRetryDelay);
+                    r = readRemoteVersion(root, version);
+                    if (r == ZKR_SUCCESS)
+                    {
+                        break;
+                    }
+                    zklog.warning("Database64::readVersion() retried readRemote() after dbReadRetryDelay=" + to_string(config.dbReadRetryDelay) + "us and failed with error=" + zkresult2string(r) + " i=" + to_string(i));
                 }
-                zklog.warning("Database64::readVersion() retried readRemote() after dbReadRetryDelay=" + to_string(config.dbReadRetryDelay) + "us and failed with error=" + zkresult2string(r) + " i=" + to_string(i));
+            }        
+            if (r == ZKR_SUCCESS)
+            {
+                dbVersionACache.addKeyVersion(root, version);
+                
+                // Add to the read log
+                if (dbReadLog != NULL) dbReadLog->add(rootStr, version, false, TimeDiff(t));
             }
-        }        
-        if (r == ZKR_SUCCESS)
-        {
-            dbVersionACache.addKeyVersion(root, version);
-            
-            // Add to the read log
-            if (dbReadLog != NULL) dbReadLog->add(rootStr, version, false, TimeDiff(t));
         }
     }
-
     // If we could not find the value, report the error
     if (r == ZKR_UNSPECIFIED)
     {
@@ -1230,28 +1218,29 @@ bool Database64::extractVersion(const pqxx::field& fieldData, const uint64_t ver
     return false;
 }
 
-zkresult Database64::writeRemoteKV(const uint64_t version, const Goldilocks::Element (&key)[4], const mpz_class &value)
+zkresult Database64::writeRemoteKV(const uint64_t version, const Goldilocks::Element (&key)[4], const mpz_class &value, bool noMultiWrite)
 {
-    zkresult result = ZKR_SUCCESS;
-    
-    
-    if (config.dbMultiWrite)
+    zkresult result = ZKR_SUCCESS; 
+    if (config.dbMultiWrite && !noMultiWrite)
     {
         multiWrite.Lock();
-
-        /*
-            PENDING: rick
-        */
-
+        KeyValue auxKV;
+        auxKV.key.fe[0] = key[0];
+        auxKV.key.fe[1] = key[1];
+        auxKV.key.fe[2] = key[2];
+        auxKV.key.fe[3] = key[3];
+        auxKV.value = value;
+        multiWrite.data[multiWrite.pendingToFlushDataIndex].keyValueIntray[version] = auxKV;
+#ifdef LOG_DB_WRITE_REMOTE
+            zklog.info("Database64::writeRemote() version=" + to_string(version) + " multiWrite=[" + multiWrite.print() + "]");
+#endif
         multiWrite.Unlock();
     }
     else
     {
         const string &tableName = config.dbKeyValueTableName;
-
         string keyStr_ = fea2string(fr, key[0], key[1], key[2], key[3]); 
         string keyStr = NormalizeToNFormat(keyStr_, 64);
-        
         string insertStr = to_string(version) + value.get_str(16);
         assert(insertStr.size() == 80);
 
@@ -1426,15 +1415,13 @@ zkresult Database64::writeRemoteVersion(const Goldilocks::Element (&root)[4], co
     string rootStr_ = fea2string(fr, root[0], root[1], root[2], root[3]); 
     string rootStr = NormalizeToNFormat(rootStr_, 64);
     
-    
     if (config.dbMultiWrite)
     {
         multiWrite.Lock();
-
-        /*
-            rick: pending
-        */
-
+        multiWrite.data[multiWrite.pendingToFlushDataIndex].versionIntray[rootStr] = version;
+#ifdef LOG_DB_WRITE_REMOTE
+            zklog.info("Database64::writeRemote() root=" + rootStr + " version " + to_string(version) + " multiWrite=[" + multiWrite.print() + "]");
+#endif
         multiWrite.Unlock();
     }
     else
@@ -2101,6 +2088,54 @@ zkresult Database64::sendData (void)
                 }
             }
 
+            // If there are keyValues add to db
+            if (data.keyValue.size() > 0)
+            {
+                unordered_map<uint64_t, KeyValue>::const_iterator it=data.keyValue.begin();
+                while (it != data.keyValue.end())
+                {
+                   writeRemoteKV(it->first,it->second.key.fe,it->second.value, true);
+                }
+            }
+            // If there are versions add to db
+            if (data.version.size() > 0)
+            {
+                unordered_map<string, uint64_t>::const_iterator it=data.version.begin();
+                while (it != data.version.end())
+                {
+                    // If queries is empty or last query is full, add a new query
+                    if ( (data.multiQuery.queries.size() == 0) || (data.multiQuery.queries[currentQuery].full))
+                    {
+                        SingleQuery query;
+                        data.multiQuery.queries.emplace_back(query);
+                        currentQuery = data.multiQuery.queries.size() - 1;
+                    }
+
+                    data.multiQuery.queries[currentQuery].query += "INSERT INTO " + config.dbVersionTableName + " ( hash, version ) VALUES ";
+                    firstValue = true;
+                    for (; it != data.version.end(); it++)
+                    {
+                        if (!firstValue)
+                        {
+                            data.multiQuery.queries[currentQuery].query += ", ";
+                        }
+                        firstValue = false;
+                        data.multiQuery.queries[currentQuery].query += "( E\'\\\\x" + it->first + "\'," + to_string(it->second) +"  ) ";
+#ifdef LOG_DB_SEND_DATA
+                        zklog.info("Database64::sendData() inserting version key=" + it->first + " version=" + to_string(it->second));
+#endif
+                        if (data.multiQuery.queries[currentQuery].query.size() >= config.dbMultiWriteSingleQuerySize)
+                        {
+                            // Mark query as full
+                            data.multiQuery.queries[currentQuery].full = true;
+                            break;
+                        }
+                    }
+                    data.multiQuery.queries[currentQuery].query += " ON CONFLICT (hash) DO NOTHING;";
+                   
+                }
+            }
+
             // If there are program add the corresponding query
             if (data.program.size() > 0)
             {
diff --git a/src/hashdb64/database_64.hpp b/src/hashdb64/database_64.hpp
index eb65d348c..0065d6406 100644
--- a/src/hashdb64/database_64.hpp
+++ b/src/hashdb64/database_64.hpp
@@ -84,7 +84,7 @@ class Database64
     zkresult writeGetTreeFunction(void);
 
     zkresult readRemoteKV(const uint64_t version, const Goldilocks::Element (&key)[4],  mpz_class value); 
-    zkresult writeRemoteKV(const uint64_t version, const Goldilocks::Element (&key)[4], const mpz_class &value);
+    zkresult writeRemoteKV(const uint64_t version, const Goldilocks::Element (&key)[4], const mpz_class &value, bool noMultiWrite = false);
     zkresult readRemoteVersion(const Goldilocks::Element (&root)[4], uint64_t version);
     zkresult writeRemoteVersion(const Goldilocks::Element (&root)[4], const uint64_t version); 
 
diff --git a/src/hashdb64/multi_write_64.cpp b/src/hashdb64/multi_write_64.cpp
index 084cf505a..aa96a2f0e 100644
--- a/src/hashdb64/multi_write_64.cpp
+++ b/src/hashdb64/multi_write_64.cpp
@@ -165,4 +165,129 @@ bool MultiWrite64::findProgram(const string &key, vector<uint8_t> &value)
     Unlock();
 
     return bResult;
+}
+
+bool MultiWrite64::findKeyValue(const uint64_t version, KeyValue &kv){
+
+    bool bResult = false;
+    Lock();
+
+    unordered_map<uint64_t, KeyValue>::const_iterator it;
+
+    // Search in data[pendingToFlushDataIndex].keyValue
+    if (bResult == false)
+    {
+        it = data[pendingToFlushDataIndex].keyValue.find(version);
+        if (it != data[pendingToFlushDataIndex].keyValue.end())
+        {
+            kv = it->second;
+            bResult = true;
+
+#ifdef LOG_DB_MULTI_WRITE_FIND_NODES
+            zklog.info("MultiWrite64::findKeyValue() data[pendingToFlushDataIndex].keyValue found version=" + to_string(version) + " key=" + fea2string(fr, it->second.key.fe) + " value=" + it->second.value.get_str());
+#endif
+        }
+    }
+
+    // Search in data[pendingToFlushDataIndex].nodesIntray
+    if (bResult == false)
+    {
+        it = data[pendingToFlushDataIndex].keyValueIntray.find(version);
+        if (it != data[pendingToFlushDataIndex].keyValueIntray.end())
+        {
+            kv = it->second;
+            bResult = true;
+
+#ifdef LOG_DB_MULTI_WRITE_FIND_NODES
+            zklog.info("MultiWrite64::findKeyValue() data[pendingToFlushDataIndex].keyValueIntray found version=" + to_string(version) + " key=" + fea2string(fr, it->second.key.fe) + " value=" + it->second.value.get_str());
+#endif
+        }
+    }
+
+    // If there is still some data pending to be stored on database
+    if (storingFlushId != storedFlushId)
+    {
+        // Search in data[storingDataIndex].keyValue
+        if (bResult == false)
+        {
+            it = data[storingDataIndex].keyValue.find(version);
+            if (it != data[storingDataIndex].keyValue.end())
+            {
+                kv = it->second;
+                bResult = true;
+#ifdef LOG_DB_MULTI_WRITE_FIND_NODES
+                zklog.info("MultiWrite64::findKeyValue() data[storingDataIndex].keyValueIntray found version=" + to_string(version) + " key=" + fea2string(fr, it->second.key.fe) + " value=" + it->second.value.get_str());
+#endif
+            }
+        }
+
+        // data[storingDataIndex].nodesIntray must be empty
+        zkassert(data[storingDataIndex].keyValueIntray.size() == 0);
+    }
+
+    Unlock();
+
+    return bResult;
+
+}
+
+bool MultiWrite64::findVersion(const string &key, uint64_t &version){
+    
+        bool bResult = false;
+        Lock();
+    
+        unordered_map<string, uint64_t>::const_iterator it;
+    
+        // Search in data[pendingToFlushDataIndex].version
+        if (bResult == false)
+        {
+            it = data[pendingToFlushDataIndex].version.find(key);
+            if (it != data[pendingToFlushDataIndex].version.end())
+            {
+                version = it->second;
+                bResult = true;
+                #ifdef LOG_DB_MULTI_WRITE_FIND_NODES
+                    zklog.info("MultiWrite64::findVersion() data[pendingToFlushDataIndex].version found key=" + key + " version=" + to_string(version));
+                #endif
+            }
+        }
+
+        // Search in data[pendingToFlushDataIndex].versionIntray
+        if (bResult == false)
+        {
+            it = data[pendingToFlushDataIndex].versionIntray.find(key);
+            if (it != data[pendingToFlushDataIndex].versionIntray.end())
+            {
+                version = it->second;
+                bResult = true;
+                #ifdef LOG_DB_MULTI_WRITE_FIND_NODES
+                    zklog.info("MultiWrite64::findVersion() data[pendingToFlushDataIndex].versionIntray found key=" + key + " version=" + to_string(version));
+                #endif
+            }
+        }
+
+        // If there is still some data pending to be stored on database
+        if (storingFlushId != storedFlushId)
+        {
+            // Search in data[storingDataIndex].version
+            if (bResult == false)
+            {
+                it = data[storingDataIndex].version.find(key);
+                if (it != data[storingDataIndex].version.end())
+                {
+                    version = it->second;
+                    bResult = true;
+                    #ifdef LOG_DB_MULTI_WRITE_FIND_NODES
+                        zklog.info("MultiWrite64::findVersion() data[storingDataIndex].version found key=" + key + " version=" + to_string(version));
+                    #endif
+                }
+            }
+
+            // data[storingDataIndex].versionIntray must be empty
+            zkassert(data[storingDataIndex].versionIntray.size() == 0);
+        }
+
+        Unlock();
+        return bResult;
+
 }
\ No newline at end of file
diff --git a/src/hashdb64/multi_write_64.hpp b/src/hashdb64/multi_write_64.hpp
index fd918dc12..cae40ba37 100644
--- a/src/hashdb64/multi_write_64.hpp
+++ b/src/hashdb64/multi_write_64.hpp
@@ -34,6 +34,8 @@ class MultiWrite64
 
     bool findNode(const string &key, string &value);
     bool findProgram(const string &key, vector<uint8_t> &value);
+    bool findKeyValue(const uint64_t version, KeyValue &kv);
+    bool findVersion(const string &key, uint64_t &version);
 };
 
 #endif
\ No newline at end of file
diff --git a/src/hashdb64/multi_write_data_64.hpp b/src/hashdb64/multi_write_data_64.hpp
index 9da4c0caa..ef292b06d 100644
--- a/src/hashdb64/multi_write_data_64.hpp
+++ b/src/hashdb64/multi_write_data_64.hpp
@@ -6,6 +6,7 @@
 #include "definitions.hpp"
 #include "zklog.hpp"
 #include "multi_query.hpp"
+#include "key_value.hpp"
 
 using namespace std;
 
@@ -17,6 +18,10 @@ class MultiWriteData64
     unordered_map<string, string> programIntray;
     unordered_map<string, string> nodes;
     unordered_map<string, string> nodesIntray;
+    unordered_map<uint64_t, KeyValue> keyValue;
+    unordered_map<uint64_t, KeyValue> keyValueIntray;
+    unordered_map<string, uint64_t> version;
+    unordered_map<string, uint64_t> versionIntray;
     string nodesStateRoot;
 
     // SQL queries, including all data to store in database
@@ -32,6 +37,10 @@ class MultiWriteData64
         programIntray.clear();
         nodes.clear();
         nodesIntray.clear();
+        keyValue.clear();
+        keyValueIntray.clear();
+        version.clear();
+        versionIntray.clear();
         nodesStateRoot.clear();
         multiQuery.reset();
         stored = false;
@@ -43,7 +52,11 @@ class MultiWriteData64
                (nodesIntray.size() == 0) &&
                (program.size() == 0) &&
                (programIntray.size() == 0) &&
-               (nodesStateRoot.size() == 0);
+               (nodesStateRoot.size() == 0) &&
+               (keyValue.size() == 0) &&
+               (keyValueIntray.size() == 0) &&
+               (version.size() == 0) &&
+               (versionIntray.size() == 0);
     }
 
     void acceptIntray (bool bSenderCalling = false)
@@ -70,6 +83,28 @@ class MultiWriteData64
             nodes.merge(nodesIntray);
             nodesIntray.clear();
         }
+        if (keyValueIntray.size() > 0)
+        {
+#ifdef LOG_DB_ACCEPT_INTRAY
+            if (bSenderCalling)
+            {
+                zklog.info("MultiWriteData64::acceptIntray() rescuing " + to_string(keyValueIntray.size()) + " keyValue pairs");
+            }
+#endif
+            keyValue.merge(keyValueIntray);
+            keyValueIntray.clear();
+        }
+        if (versionIntray.size() > 0)
+        {
+#ifdef LOG_DB_ACCEPT_INTRAY
+            if (bSenderCalling)
+            {
+                zklog.info("MultiWriteData64::acceptIntray() rescuing " + to_string(versionIntray.size()) + " versions");
+            }   
+#endif
+            version.merge(versionIntray);
+            versionIntray.clear();
+        }
     }
 };
 

From d2843ee1aad8b24946e334b6807fe490bed3cc65 Mon Sep 17 00:00:00 2001
From: rickb80 <rickborrell@gmail.com>
Date: Mon, 4 Sep 2023 14:18:29 +0200
Subject: [PATCH 14/17] writeKV based in version

---
 src/config/zkresult.cpp      |  5 ++-
 src/hashdb/database.cpp      |  2 +-
 src/hashdb64/database_64.cpp | 63 ++++++++++++++++++++++++++++++++++++
 src/hashdb64/database_64.hpp |  2 ++
 4 files changed, 70 insertions(+), 2 deletions(-)

diff --git a/src/config/zkresult.cpp b/src/config/zkresult.cpp
index 5dff0bfe0..49037ee16 100644
--- a/src/config/zkresult.cpp
+++ b/src/config/zkresult.cpp
@@ -86,7 +86,10 @@ struct
     { ZKR_SM_MAIN_MEMALIGN_WRITE8_MISMATCH, "ZKR_SM_MAIN_MEMALIGN_WRITE8_MISMATCH" },
     { ZKR_SM_MAIN_MEMALIGN_READ_MISMATCH, "ZKR_SM_MAIN_MEMALIGN_READ_MISMATCH" },
     { ZKR_SM_MAIN_HASHK_READ_OUT_OF_RANGE, "ZKR_SM_MAIN_HASHK_READ_OUT_OF_RANGE" },
-    { ZKR_SM_MAIN_HASHP_READ_OUT_OF_RANGE, "ZKR_SM_MAIN_HASHP_READ_OUT_OF_RANGE" }
+    { ZKR_SM_MAIN_HASHP_READ_OUT_OF_RANGE, "ZKR_SM_MAIN_HASHP_READ_OUT_OF_RANGE" },
+    { ZKR_DB_VERSION_NOT_FOUND, "ZKR_DB_VERSION_NOT_FOUND" }
+
+    
 };
 
 string zkresult2string (int code)
diff --git a/src/hashdb/database.cpp b/src/hashdb/database.cpp
index 47ad01212..d9458f367 100644
--- a/src/hashdb/database.cpp
+++ b/src/hashdb/database.cpp
@@ -134,7 +134,7 @@ zkresult Database::read(const string &_key, Goldilocks::Element (&vkey)[4], vect
         if (dbReadLog != NULL) dbReadLog->add(key, value, true, TimeDiff(t));
         r = ZKR_SUCCESS;
 
-    } else if( dbMTCache.enabled() && dbMTCache.find(key, value)){
+    } else if( dbMTCache.enabled() || dbMTCache.find(key, value)){
         
         if (dbReadLog != NULL) dbReadLog->add(key, value, true, TimeDiff(t));
         r = ZKR_SUCCESS;
diff --git a/src/hashdb64/database_64.cpp b/src/hashdb64/database_64.cpp
index 2b1fd68bb..eb9c1a308 100644
--- a/src/hashdb64/database_64.cpp
+++ b/src/hashdb64/database_64.cpp
@@ -541,6 +541,55 @@ zkresult Database64::writeKV(const Goldilocks::Element (&root)[4], const Goldilo
     return rkv;
 }
 
+zkresult Database64::writeKV(const uint64_t& version, const Goldilocks::Element (&key)[4], const mpz_class &value, bool persistent)
+{
+
+    // Check that it has  been initialized before
+    if (!bInitialized)
+    {
+        zklog.error("Database64::writeKV() called uninitialized");
+        exitProcess();
+    }
+    if (config.dbMultiWrite && !dbKVACache.enabled() && !persistent)
+    {
+        zklog.error("Database64::writeKV() called with multi-write active, cache disabled and no persistance in database, so there is no place to store the date");
+        return ZKR_DB_ERROR;
+    }
+    zkresult rkv = ZKR_UNSPECIFIED;
+    if ( useRemoteDB && persistent)
+    {
+        
+        rkv = writeRemoteKV(version, key, value);
+    }
+    else
+    {
+        rkv = ZKR_SUCCESS;
+    }
+
+    if (rkv == ZKR_SUCCESS)
+    {
+        dbKVACache.addKeyValueVersion(version, key, value, false);
+        
+    }
+
+#ifdef LOG_DB_WRITE
+    {
+        string keyStr_ = fea2string(fr, key[0], key[1], key[2], key[3]); 
+        string keyStr = NormalizeToNFormat(keyStr_, 64);
+        string s = "Database64::writeKV()";
+        if (rkv != ZKR_SUCCESS)
+            s += " ERROR=" + zkresult2string(rkv);
+        s += " key=" + keyStr;
+        s += " value=";
+        s += value.get_str(16);
+        s += " persistent=" + to_string(persistent);
+        zklog.info(s);
+    }
+#endif
+
+    return rkv;
+}
+
 zkresult Database64::writeKV(const Goldilocks::Element (&root)[4], const vector<KeyValue> &KVs, bool persistent){
     zkresult zkr;
     for (uint64_t i=0; i<KVs.size(); i++)
@@ -555,6 +604,20 @@ zkresult Database64::writeKV(const Goldilocks::Element (&root)[4], const vector<
     return ZKR_SUCCESS;
 }
 
+zkresult Database64::writeKV(const uint64_t& version, const vector<KeyValue> &KVs, bool persistent){
+    zkresult zkr;
+    for (uint64_t i=0; i<KVs.size(); i++)
+    {
+        zkr = writeKV(version, KVs[i].key.fe, KVs[i].value, persistent);
+        if (zkr != ZKR_SUCCESS)
+        {
+            zklog.error("Database64::writeKV(KBs) failed calling write() result=" + zkresult2string(zkr) + " key=" + fea2string(fr, KVs[i].key.fe[0], KVs[i].key.fe[1], KVs[i].key.fe[2], KVs[i].key.fe[3]) );
+            return zkr;
+        }
+    }
+    return ZKR_SUCCESS;
+}
+
 zkresult Database64::readVersion(const Goldilocks::Element (&root)[4], uint64_t& version, DatabaseMap *dbReadLog){
 
     // Check that it has been initialized before
diff --git a/src/hashdb64/database_64.hpp b/src/hashdb64/database_64.hpp
index 0065d6406..7628af2ac 100644
--- a/src/hashdb64/database_64.hpp
+++ b/src/hashdb64/database_64.hpp
@@ -121,7 +121,9 @@ class Database64
     zkresult readKV(const Goldilocks::Element (&root)[4], const Goldilocks::Element (&key)[4], mpz_class &value, DatabaseMap *dbReadLog); 
     zkresult readKV(const Goldilocks::Element (&root)[4], vector<KeyValue> &KVs, DatabaseMap *dbReadLog);
     zkresult writeKV(const Goldilocks::Element (&root)[4], const Goldilocks::Element (&key)[4], const mpz_class &value, bool persistent);
+    zkresult writeKV(const uint64_t& version, const Goldilocks::Element (&key)[4], const mpz_class &value, bool persistent);
     zkresult writeKV(const Goldilocks::Element (&root)[4], const vector<KeyValue> &KVs, bool persistent);
+    zkresult writeKV(const uint64_t& version, const vector<KeyValue> &KVs, bool persistent);
     zkresult readVersion(const Goldilocks::Element (&root)[4], uint64_t& version, DatabaseMap *dbReadLog);
     zkresult writeVersion(const Goldilocks::Element (&root)[4], const uint64_t version, bool persistent);
 

From b5d50ac8399213fb517633e6bbd6534a6cd1109a Mon Sep 17 00:00:00 2001
From: rickb80 <rickborrell@gmail.com>
Date: Mon, 4 Sep 2023 15:44:32 +0200
Subject: [PATCH 15/17] Added latest version

---
 src/config/config.cpp        |   2 +
 src/config/config.hpp        |   1 +
 src/hashdb64/database_64.cpp | 159 +++++++++++++++++++++++++++++++++++
 src/hashdb64/database_64.hpp |   6 +-
 tools/statedb/create_db.sh   |   2 +
 5 files changed, 169 insertions(+), 1 deletion(-)

diff --git a/src/config/config.cpp b/src/config/config.cpp
index 78a8b76ae..16ae151b6 100644
--- a/src/config/config.cpp
+++ b/src/config/config.cpp
@@ -262,6 +262,7 @@ void Config::load(json &config)
     ParseString(config, "dbProgramTableName", "DB_PROGRAM_TABLE_NAME", dbProgramTableName, "state.program");
     ParseString(config, "dbKeyValueTableName", "DB_KEYVALUE_TABLE_NAME", dbKeyValueTableName, "state.keyvalue");
     ParseString(config, "dbKeyVersionTableName", "DB_VERSION_TABLE_NAME", dbVersionTableName, "state.version");
+    ParseString(config, "dbLatestVersionTableName", "DB_LATEST_VERSION_TABLE_NAME", dbLatestVersionTableName, "state.latestVersion");
     ParseBool(config, "dbMultiWrite", "DB_MULTIWRITE", dbMultiWrite, true);
     ParseU64(config, "dbMultiWriteSingleQuerySize", "DB_MULTIWRITE_SINGLE_QUERY_SIZE", dbMultiWriteSingleQuerySize, 20*1024*1024);
     ParseBool(config, "dbConnectionsPool", "DB_CONNECTIONS_POOL", dbConnectionsPool, true);
@@ -456,6 +457,7 @@ void Config::print(void)
     zklog.info("    dbProgramTableName=" + dbProgramTableName);
     zklog.info("    dbKeyValueTableName=" + dbKeyValueTableName);
     zklog.info("    dbVersionTableName=" + dbVersionTableName);
+    zklog.info("    dbLatestVersionTableName=" + dbLatestVersionTableName);
     zklog.info("    dbMultiWrite=" + to_string(dbMultiWrite));
     zklog.info("    dbMultiWriteSingleQuerySize=" + to_string(dbMultiWriteSingleQuerySize));
     zklog.info("    dbConnectionsPool=" + to_string(dbConnectionsPool));
diff --git a/src/config/config.hpp b/src/config/config.hpp
index dec3d6739..2fa8662c0 100644
--- a/src/config/config.hpp
+++ b/src/config/config.hpp
@@ -147,6 +147,7 @@ class Config
     string dbProgramTableName;
     string dbKeyValueTableName;
     string dbVersionTableName;
+    string dbLatestVersionTableName;
     bool dbMultiWrite;
     uint64_t dbMultiWriteSingleQuerySize;
     bool dbConnectionsPool;
diff --git a/src/hashdb64/database_64.cpp b/src/hashdb64/database_64.cpp
index eb9c1a308..b38e7a918 100644
--- a/src/hashdb64/database_64.cpp
+++ b/src/hashdb64/database_64.cpp
@@ -25,6 +25,8 @@ DatabaseMTCache64 Database64::dbMTCache;
 DatabaseProgramCache64 Database64::dbProgramCache;
 DatabaseKVAssociativeCache Database64::dbKVACache;
 DatabaseVersionsAssociativeCache Database64::dbVersionACache;
+uint64_t Database64::latestVersionCache=1;
+
 
 string Database64::dbStateRootKey("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); // 64 f's
 Goldilocks::Element Database64::dbStateRootvKey[4] = {0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF};
@@ -761,6 +763,46 @@ zkresult Database64::writeVersion(const Goldilocks::Element (&root)[4], const ui
     return r;
 }
 
+zkresult Database64::readLatestVersion(uint64_t &version){
+
+    if (!bInitialized)
+    {
+        zklog.error("Database64::readLatestVersion() called uninitialized");
+        exitProcess();
+    }
+    version = latestVersionCache;
+    return ZKR_SUCCESS;
+}
+
+zkresult Database64::createLatestVersion(uint64_t &version, bool persistent)
+{
+
+    if (!bInitialized)
+    {
+        zklog.error("Database64::createLatestVersion() called uninitialized");
+        exitProcess();
+    }
+    
+    zkresult r;
+
+    if ( useRemoteDB && persistent)
+    {
+        
+        r = createRemoteLatestVersion(version);
+    }
+    else
+    {
+        r = ZKR_SUCCESS;
+    }
+
+    if (r == ZKR_SUCCESS)
+    {
+        latestVersionCache=version;
+        
+    }
+    return r;
+}
+
 void Database64::initRemote(void)
 {
     TimerStart(DB_INIT_REMOTE);
@@ -1526,6 +1568,123 @@ zkresult Database64::writeRemoteVersion(const Goldilocks::Element (&root)[4], co
     return result;
 }
 
+zkresult Database64::readRemoteLatestVersion(uint64_t &version){
+    
+    const string &tableName = config.dbLatestVersionTableName;
+
+
+    if (config.logRemoteDbReads)
+    {
+        zklog.info("Database64::readRemoteLatestVersion() table=" + tableName );
+    }
+
+    // Get a free read db connection
+    DatabaseConnection * pDatabaseConnection = getConnection();
+
+    try
+    {
+        // Prepare the query
+        string query = "SELECT * FROM " + tableName + ";";
+
+        pqxx::result rows;
+
+        // Start a transaction.
+        pqxx::nontransaction n(*(pDatabaseConnection->pConnection));
+
+        // Execute the query
+        rows = n.exec(query);
+
+        // Commit your transaction
+        n.commit();
+
+        // Process the result
+        if (rows.size() == 0)
+        {
+            disposeConnection(pDatabaseConnection);
+            return ZKR_DB_KEY_NOT_FOUND;
+        }
+        else if (rows.size() > 1)
+        {
+            zklog.error("Database64::readRemoteLatestVersion() table=" + tableName + " got more than one row;" );
+            exitProcess();
+        }
+
+        pqxx::row const row = rows[0];
+        if (row.size() != 1)
+        {
+            zklog.error("DatabaseKV::readRemoteLatestVersion() table=" + tableName + " got an invalid number of colums;");
+            exitProcess();
+        }
+        pqxx::field const fieldData = row[0];
+        if (!fieldData.is_null()) {
+            fieldData.as<uint64_t>(version);
+        } else {
+            zklog.error("DatabaseKV::readRemoteLatestVersion() table=" + tableName + " got a null version;");
+        }
+    }
+    catch (const std::exception &e)
+    {
+        zklog.error("Database64::readRemoteLatestVersion() table=" + tableName + " exception: " + string(e.what()) + " connection=" + to_string((uint64_t)pDatabaseConnection));
+        queryFailed();
+        disposeConnection(pDatabaseConnection);
+        return ZKR_DB_ERROR;
+    }
+    
+    // Dispose the read db conneciton
+    disposeConnection(pDatabaseConnection);
+
+    return ZKR_SUCCESS;
+}
+zkresult Database64::createRemoteLatestVersion(uint64_t &version){
+    
+    
+    zkresult rr = readLatestVersion(version);
+    if (rr != ZKR_SUCCESS)
+    {
+        zklog.error("Database64::createRemoteLatestVersion() readRemoteLatestVersion() failed with error=" + zkresult2string(rr));
+        return rr;
+    }
+    uint64_t newVersion = version+1;
+
+    zkresult rw = ZKR_SUCCESS;    
+
+    const string &tableName =  config.dbLatestVersionTableName;
+    string query = "UPDATE " + tableName + " ( version ) VALUES ( " + to_string(newVersion) +");";
+    DatabaseConnection * pDatabaseConnection = getConnection();
+
+    try
+    {        
+
+#ifdef DATABASE_COMMIT
+        if (autoCommit)
+#endif
+        {
+            pqxx::work w(*(pDatabaseConnection->pConnection));
+            pqxx::result res = w.exec(query);
+            w.commit();
+        }
+#ifdef DATABASE_COMMIT
+        else
+        {
+            if (transaction == NULL)
+                transaction = new pqxx::work{*pConnectionWrite};
+            pqxx::result res = transaction->exec(query);
+        }
+#endif
+    }
+    catch (const std::exception &e)
+    {
+        zklog.error("Database::writeRemoteVersion() table=" + tableName + " exception: " + string(e.what()) + " connection=" + to_string((uint64_t)pDatabaseConnection));
+        rw = ZKR_DB_ERROR;
+        queryFailed();
+    }
+
+    disposeConnection(pDatabaseConnection);
+    
+
+    return rw;
+}
+
 zkresult Database64::createStateRoot(void)
 {
     // Copy the state root in the first 4 elements of dbValue
diff --git a/src/hashdb64/database_64.hpp b/src/hashdb64/database_64.hpp
index 7628af2ac..aa9bdcf1c 100644
--- a/src/hashdb64/database_64.hpp
+++ b/src/hashdb64/database_64.hpp
@@ -87,6 +87,8 @@ class Database64
     zkresult writeRemoteKV(const uint64_t version, const Goldilocks::Element (&key)[4], const mpz_class &value, bool noMultiWrite = false);
     zkresult readRemoteVersion(const Goldilocks::Element (&root)[4], uint64_t version);
     zkresult writeRemoteVersion(const Goldilocks::Element (&root)[4], const uint64_t version); 
+    zkresult readRemoteLatestVersion(uint64_t &version);
+    zkresult createRemoteLatestVersion(uint64_t &version);
 
     bool extractVersion(const pqxx::field& fieldData, const uint64_t version, mpz_class &value);
 
@@ -97,6 +99,7 @@ class Database64
     static DatabaseProgramCache64 dbProgramCache;
     static DatabaseKVAssociativeCache dbKVACache;
     static DatabaseVersionsAssociativeCache dbVersionACache;
+    static uint64_t latestVersionCache;
 
     // This is a fixed key to store the latest state root hash, used to load it to the cache
     // This key is "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
@@ -126,7 +129,8 @@ class Database64
     zkresult writeKV(const uint64_t& version, const vector<KeyValue> &KVs, bool persistent);
     zkresult readVersion(const Goldilocks::Element (&root)[4], uint64_t& version, DatabaseMap *dbReadLog);
     zkresult writeVersion(const Goldilocks::Element (&root)[4], const uint64_t version, bool persistent);
-
+    zkresult readLatestVersion(uint64_t &version);
+    zkresult createLatestVersion(uint64_t &version, bool persistent);
 
 private:
     zkresult createStateRoot(void);
diff --git a/tools/statedb/create_db.sh b/tools/statedb/create_db.sh
index 5f9659be7..be9a1925d 100755
--- a/tools/statedb/create_db.sh
+++ b/tools/statedb/create_db.sh
@@ -18,6 +18,8 @@ PGPASSWORD=$3 psql -U $2 -h 127.0.0.1 -p 5432 -d $1 -c 'create table state.nodes
 PGPASSWORD=$3 psql -U $2 -h 127.0.0.1 -p 5432 -d $1 -c 'create table state.program (hash bytea primary key, data bytea not null);'
 PGPASSWORD=$3 psql -U $2 -h 127.0.0.1 -p 5432 -d $1 -c 'create table state.keyvalue (key bytea primary key, data bytea);'
 PGPASSWORD=$3 psql -U $2 -h 127.0.0.1 -p 5432 -d $1 -c 'create table state.version (hash bytea primary key, version bigint);'
+PGPASSWORD=$3 psql -U $2 -h 127.0.0.1 -p 5432 -d $1 -c 'create table state.latestVersion (version bigint);'
+
 
 if [ $# == 4 ]
     then

From 487f60f38078ff4d809b3de01592e0c63a38aaa6 Mon Sep 17 00:00:00 2001
From: rickb80 <rickborrell@gmail.com>
Date: Mon, 4 Sep 2023 15:49:41 +0200
Subject: [PATCH 16/17] Bug reparied in cache enabled checking

---
 src/hashdb/database.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/hashdb/database.cpp b/src/hashdb/database.cpp
index d9458f367..a0bed0241 100644
--- a/src/hashdb/database.cpp
+++ b/src/hashdb/database.cpp
@@ -134,7 +134,7 @@ zkresult Database::read(const string &_key, Goldilocks::Element (&vkey)[4], vect
         if (dbReadLog != NULL) dbReadLog->add(key, value, true, TimeDiff(t));
         r = ZKR_SUCCESS;
 
-    } else if( dbMTCache.enabled() || dbMTCache.find(key, value)){
+    } else if( dbMTCache.enabled() && dbMTCache.find(key, value)){
         
         if (dbReadLog != NULL) dbReadLog->add(key, value, true, TimeDiff(t));
         r = ZKR_SUCCESS;
@@ -294,7 +294,7 @@ zkresult Database::write(const string &_key, const Goldilocks::Element* vkey, co
         exitProcess();
     }
 
-    if (config.dbMultiWrite && !(dbMTCache.enabled() && dbMTACache.enabled()) && !persistent)
+    if (config.dbMultiWrite && !(dbMTCache.enabled() || dbMTACache.enabled()) && !persistent)
     {
         zklog.error("Database::write() called with multi-write active, cache disabled and no persistance in database, so there is no place to store the date");
         return ZKR_DB_ERROR;

From 0c1b7c4cd0464f53b8a5bcce939212425a872319 Mon Sep 17 00:00:00 2001
From: rickb80 <rickborrell@gmail.com>
Date: Mon, 4 Sep 2023 17:04:39 +0200
Subject: [PATCH 17/17] Change latestversion table name

---
 src/config/config.cpp        | 2 +-
 src/hashdb64/database_64.cpp | 2 +-
 tools/statedb/create_db.sh   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/config/config.cpp b/src/config/config.cpp
index 16ae151b6..34d11a302 100644
--- a/src/config/config.cpp
+++ b/src/config/config.cpp
@@ -262,7 +262,7 @@ void Config::load(json &config)
     ParseString(config, "dbProgramTableName", "DB_PROGRAM_TABLE_NAME", dbProgramTableName, "state.program");
     ParseString(config, "dbKeyValueTableName", "DB_KEYVALUE_TABLE_NAME", dbKeyValueTableName, "state.keyvalue");
     ParseString(config, "dbKeyVersionTableName", "DB_VERSION_TABLE_NAME", dbVersionTableName, "state.version");
-    ParseString(config, "dbLatestVersionTableName", "DB_LATEST_VERSION_TABLE_NAME", dbLatestVersionTableName, "state.latestVersion");
+    ParseString(config, "dbLatestVersionTableName", "DB_LATEST_VERSION_TABLE_NAME", dbLatestVersionTableName, "state.latestversion");
     ParseBool(config, "dbMultiWrite", "DB_MULTIWRITE", dbMultiWrite, true);
     ParseU64(config, "dbMultiWriteSingleQuerySize", "DB_MULTIWRITE_SINGLE_QUERY_SIZE", dbMultiWriteSingleQuerySize, 20*1024*1024);
     ParseBool(config, "dbConnectionsPool", "DB_CONNECTIONS_POOL", dbConnectionsPool, true);
diff --git a/src/hashdb64/database_64.cpp b/src/hashdb64/database_64.cpp
index b38e7a918..9ad480973 100644
--- a/src/hashdb64/database_64.cpp
+++ b/src/hashdb64/database_64.cpp
@@ -25,7 +25,7 @@ DatabaseMTCache64 Database64::dbMTCache;
 DatabaseProgramCache64 Database64::dbProgramCache;
 DatabaseKVAssociativeCache Database64::dbKVACache;
 DatabaseVersionsAssociativeCache Database64::dbVersionACache;
-uint64_t Database64::latestVersionCache=1;
+uint64_t Database64::latestVersionCache=0;
 
 
 string Database64::dbStateRootKey("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); // 64 f's
diff --git a/tools/statedb/create_db.sh b/tools/statedb/create_db.sh
index be9a1925d..4e7e5adef 100755
--- a/tools/statedb/create_db.sh
+++ b/tools/statedb/create_db.sh
@@ -18,7 +18,7 @@ PGPASSWORD=$3 psql -U $2 -h 127.0.0.1 -p 5432 -d $1 -c 'create table state.nodes
 PGPASSWORD=$3 psql -U $2 -h 127.0.0.1 -p 5432 -d $1 -c 'create table state.program (hash bytea primary key, data bytea not null);'
 PGPASSWORD=$3 psql -U $2 -h 127.0.0.1 -p 5432 -d $1 -c 'create table state.keyvalue (key bytea primary key, data bytea);'
 PGPASSWORD=$3 psql -U $2 -h 127.0.0.1 -p 5432 -d $1 -c 'create table state.version (hash bytea primary key, version bigint);'
-PGPASSWORD=$3 psql -U $2 -h 127.0.0.1 -p 5432 -d $1 -c 'create table state.latestVersion (version bigint);'
+PGPASSWORD=$3 psql -U $2 -h 127.0.0.1 -p 5432 -d $1 -c 'create table state.latestversion (version bigint);'
 
 
 if [ $# == 4 ]