From b4dd44e5573d429b1d912155b14a7dba99a9b9b9 Mon Sep 17 00:00:00 2001 From: rickb80 Date: Fri, 8 Sep 2023 09:44:44 +0200 Subject: [PATCH 01/12] Version test remote --- test/hashdb/database_kv_remote_test.cpp | 89 +++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 test/hashdb/database_kv_remote_test.cpp diff --git a/test/hashdb/database_kv_remote_test.cpp b/test/hashdb/database_kv_remote_test.cpp new file mode 100644 index 000000000..85387b57a --- /dev/null +++ b/test/hashdb/database_kv_remote_test.cpp @@ -0,0 +1,89 @@ +#include "database_kv_remote_test.hpp" +#include "goldilocks_base_field.hpp" +#include "hashdb.hpp" +#include "hashdb_singleton.hpp" + +#define DBKV_REMOTE_TEST_NUMBER_OF_WRITES 1000 + + +uint64_t DatabaseKVRemoteTest (const Config &config){ + + TimerStart(DATABASE_KV_REMOTE_TEST); + + if(config.dbMultiWrite == true){ + zklog.error("DatabaseKVRemoteTest() this test must be run with config.dbMultiWrite=false"); + exitProcess(); + } + HashDB * pHashDB = (HashDB *)hashDBSingleton.get(); + if (pHashDB == NULL) + { + zklog.error("DatabaseKVRemoteTest() failed calling HashDBSingleton::get()"); + exitProcess(); + } + Database64 * pDatabase64 = &pHashDB->db64; + + uint64_t numberOfFailedTests = 0; + Goldilocks fr; + + // + // Lattest version + // + uint64_t version = 33; + pDatabase64->writeLatestVersion(version, true); + pDatabase64->readLatestVersion(version); + if(version != 33) + { + zklog.error("DatabaseKVRemoteTest() failed calling Database64.readLatestVersion()"); + numberOfFailedTests += 1; + } + pDatabase64->writeLatestVersion(0, false); + pDatabase64->readLatestVersion(version); + if(version != 33) + { + zklog.error("DatabaseKVRemoteTest() failed calling Database64.readLatestVersion()"); + numberOfFailedTests += 1; + } + // + // Version + // + Goldilocks::Element root[4]; + + mpz_class rootScalar; + string rootString; + version = 1; + + for (uint64_t i=0; iwriteVersion(root, version, true); + version+=1; + } + /*for (uint64_t i=0; i Date: Fri, 8 Sep 2023 10:16:12 +0200 Subject: [PATCH 02/12] Avoid hashKey duplicity in associative caches... --- src/hashdb64/database_kv_associative_cache.cpp | 2 +- src/hashdb64/database_kv_associative_cache.hpp | 10 ---------- src/hashdb64/database_versions_associtive_cache.cpp | 2 +- src/hashdb64/database_versions_associtive_cache.hpp | 10 ---------- 4 files changed, 2 insertions(+), 22 deletions(-) diff --git a/src/hashdb64/database_kv_associative_cache.cpp b/src/hashdb64/database_kv_associative_cache.cpp index b066bf939..584a6af49 100644 --- a/src/hashdb64/database_kv_associative_cache.cpp +++ b/src/hashdb64/database_kv_associative_cache.cpp @@ -191,7 +191,7 @@ void DatabaseKVAssociativeCache::forcedInsertion(uint32_t (&usedRawCacheIndexes) // find a slot into my indexes // Goldilocks::Element key_hashed[4]; - hashKey_p(key_hashed, &keys[(inputRawCacheIndex & cacheMask) * 4]); + hashKey(key_hashed, (Goldilocks::Element(&)[4])keys[(inputRawCacheIndex & cacheMask) * 4]); Goldilocks::Element *inputKey = &key_hashed[0]; uint32_t minRawCacheIndex = UINT32_MAX; int pos = -1; diff --git a/src/hashdb64/database_kv_associative_cache.hpp b/src/hashdb64/database_kv_associative_cache.hpp index 36422d8e0..0bbcf744f 100644 --- a/src/hashdb64/database_kv_associative_cache.hpp +++ b/src/hashdb64/database_kv_associative_cache.hpp @@ -66,16 +66,6 @@ class DatabaseKVAssociativeCache PoseidonGoldilocks pg; pg.hash_seq(keyOut, key_hash_imput); }; - inline void hashKey_p(Goldilocks::Element (&keyOut)[4], const Goldilocks::Element * keyIn) const{ //rick: convertir - Goldilocks::Element key_hash_imput[12]; - for(int i=0; i<4; i++){ - key_hash_imput[i] = keyIn[i]; - key_hash_imput[i+4] = keyIn[i]; - key_hash_imput[i+8] = keyIn[i]; - } - PoseidonGoldilocks pg; - pg.hash_seq(keyOut, key_hash_imput); - }; }; #endif \ No newline at end of file diff --git a/src/hashdb64/database_versions_associtive_cache.cpp b/src/hashdb64/database_versions_associtive_cache.cpp index 9e00bb9ed..a36cb20f7 100644 --- a/src/hashdb64/database_versions_associtive_cache.cpp +++ b/src/hashdb64/database_versions_associtive_cache.cpp @@ -173,7 +173,7 @@ void DatabaseVersionsAssociativeCache::forcedInsertion(uint32_t (&usedRawCacheIn // find a slot into my indexes // Goldilocks::Element key_hashed[4]; - hashKey_p(key_hashed, &keys[(inputRawCacheIndex & cacheMask) * 4]); + hashKey(key_hashed, (Goldilocks::Element(&)[4])keys[(inputRawCacheIndex & cacheMask) * 4]); Goldilocks::Element *inputKey = &key_hashed[0]; uint32_t minRawCacheIndex = UINT32_MAX; int pos = -1; diff --git a/src/hashdb64/database_versions_associtive_cache.hpp b/src/hashdb64/database_versions_associtive_cache.hpp index 6f0f0da01..2d368bceb 100644 --- a/src/hashdb64/database_versions_associtive_cache.hpp +++ b/src/hashdb64/database_versions_associtive_cache.hpp @@ -63,16 +63,6 @@ class DatabaseVersionsAssociativeCache PoseidonGoldilocks pg; pg.hash_seq(keyOut, key_hash_imput); }; - inline void hashKey_p(Goldilocks::Element (&keyOut)[4], const Goldilocks::Element * keyIn) const{ //rick: convertir - Goldilocks::Element key_hash_imput[12]; - for(int i=0; i<4; i++){ - key_hash_imput[i] = keyIn[i]; - key_hash_imput[i+4] = keyIn[i]; - key_hash_imput[i+8] = keyIn[i]; - } - PoseidonGoldilocks pg; - pg.hash_seq(keyOut, key_hash_imput); - }; }; #endif From 9de39819d448ffa644ff8b551873c80f23bec72d Mon Sep 17 00:00:00 2001 From: rickb80 Date: Fri, 8 Sep 2023 14:53:55 +0200 Subject: [PATCH 03/12] Add getTree as last option for KV and fixes --- src/config/zkresult.cpp | 2 +- src/config/zkresult.hpp | 2 +- src/hashdb64/database_64.cpp | 117 ++++++++++++++++------------------- 3 files changed, 57 insertions(+), 64 deletions(-) diff --git a/src/config/zkresult.cpp b/src/config/zkresult.cpp index eb9532852..6145f9b8a 100644 --- a/src/config/zkresult.cpp +++ b/src/config/zkresult.cpp @@ -87,7 +87,7 @@ struct { ZKR_SM_MAIN_MEMALIGN_READ_MISMATCH, "ZKR_SM_MAIN_MEMALIGN_READ_MISMATCH" }, { ZKR_SM_MAIN_HASHK_READ_OUT_OF_RANGE, "ZKR_SM_MAIN_HASHK_READ_OUT_OF_RANGE" }, { ZKR_SM_MAIN_HASHP_READ_OUT_OF_RANGE, "ZKR_SM_MAIN_HASHP_READ_OUT_OF_RANGE" }, - { ZKR_DB_VERSION_NOT_FOUND, "ZKR_DB_VERSION_NOT_FOUND" }, + { ZKR_DB_VERSION_NOT_FOUND_KVDB, "ZKR_DB_VERSION_NOT_FOUND_KBDB" }, { ZKR_DB_VERSION_NOT_FOUND_GLOBAL, "ZKR_DB_VERSION_NOT_FOUND_GLOBAL"} diff --git a/src/config/zkresult.hpp b/src/config/zkresult.hpp index 01ece0449..4294fc59f 100644 --- a/src/config/zkresult.hpp +++ b/src/config/zkresult.hpp @@ -88,7 +88,7 @@ typedef enum : int ZKR_SM_MAIN_MEMALIGN_READ_MISMATCH = 78, // Main state memory align read ROM operation check failed ZKR_SM_MAIN_HASHK_READ_OUT_OF_RANGE = 79, // Main state Keccak hash check found read out of range ZKR_SM_MAIN_HASHP_READ_OUT_OF_RANGE = 80, // Main state Poseidon hash check found read out of range - ZKR_DB_VERSION_NOT_FOUND = 81, // Version not found in KeyValue database + ZKR_DB_VERSION_NOT_FOUND_KVDB = 81, // Version not found in KeyValue database ZKR_DB_VERSION_NOT_FOUND_GLOBAL = 82, // Version not found in KeyValue database and not present in hashDB neither diff --git a/src/hashdb64/database_64.cpp b/src/hashdb64/database_64.cpp index 25311f72c..6e8d4b7f7 100644 --- a/src/hashdb64/database_64.cpp +++ b/src/hashdb64/database_64.cpp @@ -15,6 +15,7 @@ #include "zkmax.hpp" #include "hashdb_remote.hpp" #include "key_value.hpp" +#include "tree_64.hpp" #ifdef DATABASE_USE_CACHE @@ -377,6 +378,7 @@ zkresult Database64::readKV(const Goldilocks::Element (&root)[4], const Goldiloc zkresult rkv = ZKR_UNSPECIFIED; zkresult rv = ZKR_UNSPECIFIED; + zkresult rout = ZKR_UNSPECIFIED; string keyStr = ""; if(dbReadLog->getSaveKeys()){ @@ -411,56 +413,47 @@ zkresult Database64::readKV(const Goldilocks::Element (&root)[4], const Goldiloc else if(useRemoteDB) { - rkv = readRemoteKV(version, key, value); - if ( (rkv != ZKR_SUCCESS) && (config.dbReadRetryDelay > 0) ) - { - if(!dbReadLog->getSaveKeys()){ - string keyStr_ = fea2string(fr, key[0], key[1], key[2], key[3]); - keyStr = NormalizeToNFormat(keyStr_, 64); - } - for (uint64_t i=0; i 1) { @@ -1275,6 +1268,9 @@ zkresult Database64::readRemoteKV(const uint64_t version, const Goldilocks::Elem zklog.error("DatabaseKV::readRemoteKV() table=" + tableName + " got an invalid number of colums for the row: " + to_string(row.size()) + "for key=" + keyStr); exitProcess(); } + + // Dispose the read db conneciton + disposeConnection(pDatabaseConnection); return extractVersion(row[1], version, value); @@ -1290,7 +1286,7 @@ zkresult Database64::readRemoteKV(const uint64_t version, const Goldilocks::Elem // Dispose the read db conneciton disposeConnection(pDatabaseConnection); - return ZKR_SUCCESS; + return ZKR_DB_ERROR;//should never return here } zkresult Database64::extractVersion(const pqxx::field& fieldData, const uint64_t version, mpz_class &value){ @@ -1330,7 +1326,7 @@ zkresult Database64::extractVersion(const pqxx::field& fieldData, const uint64_t zklog.error("Database64::extractVersion() got an invalid data size: " + to_string(data_size)); exitProcess(); } - return ZKR_DB_VERSION_NOT_FOUND; + return ZKR_DB_VERSION_NOT_FOUND_KVDB; } } @@ -1387,33 +1383,30 @@ zkresult Database64::writeRemoteKV(const uint64_t version, const Goldilocks::Ele n.commit(); // Process the result - if (rows.size() == 0) - { - disposeConnection(pDatabaseConnection); - return ZKR_DB_KEY_NOT_FOUND; - } - else if (rows.size() > 1) - { - zklog.error("Database64::writeRemoteKV() table=" + tableName + " got more than one row for the same key: " + to_string(rows.size()) + "for key=" + keyStr); - exitProcess(); - } + if(rows.size() > 0){ + if (rows.size() > 1) + { + zklog.error("Database64::writeRemoteKV() table=" + tableName + " got more than one row for the same key: " + to_string(rows.size()) + "for key=" + keyStr); + exitProcess(); + } - const pqxx::row& row = rows[0]; - if (row.size() != 2) - { - zklog.error("DatabaseKV::writeRemoteKV() table=" + tableName + " got an invalid number of colums for the row: " + to_string(row.size()) + "for key=" + keyStr); - exitProcess(); - } - const pqxx::field& fieldData = row[1]; - //processar insert; - if(!fieldData.is_null()){ - string data = removeBSXIfExists64(fieldData.c_str()); - int data_size = data.size(); - if(data_size == maxVersions*80){ - data = data.substr(0, data_size - 80); + const pqxx::row& row = rows[0]; + if (row.size() != 2) + { + zklog.error("DatabaseKV::writeRemoteKV() table=" + tableName + " got an invalid number of colums for the row: " + to_string(row.size()) + "for key=" + keyStr); + exitProcess(); } - insertStr = insertStr + data; - } + const pqxx::field& fieldData = row[1]; + //processar insert; + if(!fieldData.is_null()){ + string data = removeBSXIfExists64(fieldData.c_str()); + int data_size = data.size(); + if(data_size == maxVersions*80){ + data = data.substr(0, data_size - 80); + } + insertStr = insertStr + data; + } + } } catch (const std::exception &e) { From 437022a2a53904dede629a8ecef520de3a7d900f Mon Sep 17 00:00:00 2001 From: rickb80 Date: Fri, 8 Sep 2023 15:28:22 +0200 Subject: [PATCH 04/12] Don not fill kv_associative_cache with repeated values --- src/hashdb64/database_kv_associative_cache.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/hashdb64/database_kv_associative_cache.cpp b/src/hashdb64/database_kv_associative_cache.cpp index 584a6af49..e649d3d9c 100644 --- a/src/hashdb64/database_kv_associative_cache.cpp +++ b/src/hashdb64/database_kv_associative_cache.cpp @@ -114,6 +114,7 @@ void DatabaseKVAssociativeCache::addKeyValueVersion(const uint64_t version, cons cacheIndex = cacheIndexRaw & cacheMask; uint32_t cacheIndexKey = cacheIndex * 4; uint32_t cacheIndexVersions = cacheIndex * 2; + uint32_t cacheIndexValue = cacheIndex; if (!emptyCacheSlot(cacheIndexRaw)){ if( keys[cacheIndexKey + 0].fe == key[0].fe && @@ -124,7 +125,15 @@ void DatabaseKVAssociativeCache::addKeyValueVersion(const uint64_t version, cons if(versions[cacheIndexVersions] == version){ presentSameVersion = true; if(update == false) return; + } else if (versions[cacheIndexVersions] > version){ + zklog.error("DatabaseKVAssociativeCache::addKeyValueVersion() adding lower version than the current one"); + exitProcess(); + } else { + if(value==values[cacheIndexValue]){ + return; + } } + tableIndexUse = tableIndex; cacheIndexPrev = cacheIndex; break; From 591d22119637742f6a609eceed07b6398124fbdd Mon Sep 17 00:00:00 2001 From: rickb80 Date: Fri, 8 Sep 2023 16:13:54 +0200 Subject: [PATCH 05/12] Add zeros into KVCache --- src/hashdb64/database_64.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/hashdb64/database_64.cpp b/src/hashdb64/database_64.cpp index 6e8d4b7f7..93941847b 100644 --- a/src/hashdb64/database_64.cpp +++ b/src/hashdb64/database_64.cpp @@ -423,6 +423,8 @@ zkresult Database64::readKV(const Goldilocks::Element (&root)[4], const Goldiloc if( rkv == ZKR_DB_VERSION_NOT_FOUND_GLOBAL){ value = 0; rout = rkv; + mpz_class zero(0); + dbKVACache.addKeyValueVersion(version, key, zero, false); }else{ zkresult rtree = ZKR_UNSPECIFIED; vector keyValues(1); @@ -1406,7 +1408,13 @@ zkresult Database64::writeRemoteKV(const uint64_t version, const Goldilocks::Ele } insertStr = insertStr + data; } - } + } else{ + if(version > 1){ + //this must be reproduced into multiwrite + mpz_class zero(0); + dbKVACache.addKeyValueVersion(version-1, key, zero, false); + } + } } catch (const std::exception &e) { From 7bda7dbf8326d7f8ce3672f9e160a3e1b2952656 Mon Sep 17 00:00:00 2001 From: rickb80 Date: Fri, 8 Sep 2023 19:24:20 +0200 Subject: [PATCH 06/12] Fixed multiwrite usage for keyValue --- src/hashdb64/database_64.cpp | 30 ++++---- src/hashdb64/database_64.hpp | 2 +- src/hashdb64/multi_write_64.cpp | 107 +++++++++++++++++++-------- src/hashdb64/multi_write_64.hpp | 2 +- src/hashdb64/multi_write_data_64.hpp | 50 ++++++++++--- 5 files changed, 134 insertions(+), 57 deletions(-) diff --git a/src/hashdb64/database_64.cpp b/src/hashdb64/database_64.cpp index 93941847b..c882083f0 100644 --- a/src/hashdb64/database_64.cpp +++ b/src/hashdb64/database_64.cpp @@ -385,7 +385,6 @@ zkresult Database64::readKV(const Goldilocks::Element (&root)[4], const Goldiloc string keyStr_ = fea2string(fr, key[0], key[1], key[2], key[3]); keyStr = NormalizeToNFormat(keyStr_, 64); } - KeyValue mwEntry; uint64_t version; rv = readVersion(root, version, dbReadLog); @@ -400,14 +399,13 @@ zkresult Database64::readKV(const Goldilocks::Element (&root)[4], const Goldiloc } // If the key is pending to be stored in database, but already deleted from cache - else if (config.dbMultiWrite && multiWrite.findKeyValue(version, mwEntry)) + else if (config.dbMultiWrite && multiWrite.findKeyValue(version, key, value)) { - value = mwEntry.value; // Add to the read log if (dbReadLog != NULL) dbReadLog->add(keyStr, value, true, TimeDiff(t)); // Store it locally to avoid any future remote access for this key - dbKVACache.addKeyValueVersion(version, key, value, false); + dbKVACache.addKeyValueVersion(version, key, value, false); //rick problematic rkv = ZKR_SUCCESS; } else if(useRemoteDB) @@ -416,7 +414,7 @@ zkresult Database64::readKV(const Goldilocks::Element (&root)[4], const Goldiloc rkv = readRemoteKV(version, key, value); if (rkv == ZKR_SUCCESS) { - dbKVACache.addKeyValueVersion(version, key, value, false); + dbKVACache.addKeyValueVersion(version, key, value, false); //rick problematic if (dbReadLog != NULL) dbReadLog->add(keyStr, value, false, TimeDiff(t)); } else { @@ -424,7 +422,7 @@ zkresult Database64::readKV(const Goldilocks::Element (&root)[4], const Goldiloc value = 0; rout = rkv; mpz_class zero(0); - dbKVACache.addKeyValueVersion(version, key, zero, false); + dbKVACache.addKeyValueVersion(version, key, zero, false); //rick problematic }else{ zkresult rtree = ZKR_UNSPECIFIED; vector keyValues(1); @@ -1332,10 +1330,10 @@ zkresult Database64::extractVersion(const pqxx::field& fieldData, const uint64_t } } -zkresult Database64::writeRemoteKV(const uint64_t version, const Goldilocks::Element (&key)[4], const mpz_class &value, bool noMultiWrite) +zkresult Database64::writeRemoteKV(const uint64_t version, const Goldilocks::Element (&key)[4], const mpz_class &value, bool useMultiWrite) { zkresult result = ZKR_SUCCESS; - if (config.dbMultiWrite && !noMultiWrite) + if (config.dbMultiWrite && useMultiWrite) { multiWrite.Lock(); KeyValue auxKV; @@ -1344,7 +1342,10 @@ zkresult Database64::writeRemoteKV(const uint64_t version, const Goldilocks::Ele auxKV.key[2] = key[2]; auxKV.key[3] = key[3]; auxKV.value = value; - multiWrite.data[multiWrite.pendingToFlushDataIndex].keyValueIntray[version] = auxKV; + multiWrite.data[multiWrite.pendingToFlushDataIndex].keyValueAIntray[version].push_back(auxKV); + string keyStr_ = fea2string(fr, key[0], key[1], key[2], key[3]); + string keyStr = NormalizeToNFormat(keyStr_, 64); + multiWrite.data[multiWrite.pendingToFlushDataIndex].keyVersionsIntray[keyStr].push_back(version); #ifdef LOG_DB_WRITE_REMOTE zklog.info("Database64::writeRemote() version=" + to_string(version) + " multiWrite=[" + multiWrite.print() + "]"); #endif @@ -2325,12 +2326,15 @@ zkresult Database64::sendData (void) } // If there are keyValues add to db - if (data.keyValue.size() > 0) + if (data.keyValueA.size() > 0) { - unordered_map::const_iterator it=data.keyValue.begin(); - while (it != data.keyValue.end()) + map>::const_iterator it=data.keyValueA.begin(); + while (it != data.keyValueA.end()) { - writeRemoteKV(it->first,it->second.key,it->second.value, true); + for(auto it2=it->second.begin(); it2!=it->second.end(); it2++) + { + writeRemoteKV(it->first,it2->key,it2->value, false); + } } } // If there are versions add to db diff --git a/src/hashdb64/database_64.hpp b/src/hashdb64/database_64.hpp index 92ba15f52..e4bee0352 100644 --- a/src/hashdb64/database_64.hpp +++ b/src/hashdb64/database_64.hpp @@ -84,7 +84,7 @@ class Database64 zkresult writeGetTreeFunction(void); zkresult readRemoteKV(const uint64_t version, const Goldilocks::Element (&key)[4], mpz_class value); - zkresult writeRemoteKV(const uint64_t version, const Goldilocks::Element (&key)[4], const mpz_class &value, bool noMultiWrite = false); + zkresult writeRemoteKV(const uint64_t version, const Goldilocks::Element (&key)[4], const mpz_class &value, bool useMultiWrite = true); zkresult readRemoteVersion(const Goldilocks::Element (&root)[4], uint64_t version); zkresult writeRemoteVersion(const Goldilocks::Element (&root)[4], const uint64_t version); zkresult readRemoteLatestVersion(uint64_t &version); diff --git a/src/hashdb64/multi_write_64.cpp b/src/hashdb64/multi_write_64.cpp index aa96a2f0e..da49eef9c 100644 --- a/src/hashdb64/multi_write_64.cpp +++ b/src/hashdb64/multi_write_64.cpp @@ -4,6 +4,19 @@ using namespace std; +uint64_t previousAvailableVersion(const uint64_t versionIn, const vector &versions) +{ + uint64_t versionOut = 0; + for (auto it = versions.begin(); it != versions.end(); ++it) + { + if (*it < versionIn && *it > versionOut) + { + versionOut = *it; + } + } + return versionOut; +} + MultiWrite64::MultiWrite64(Goldilocks & fr) : fr(fr), lastFlushId(0), @@ -167,62 +180,94 @@ bool MultiWrite64::findProgram(const string &key, vector &value) return bResult; } -bool MultiWrite64::findKeyValue(const uint64_t version, KeyValue &kv){ +bool MultiWrite64::findKeyValue(const uint64_t version,const Goldilocks::Element (&key)[4], mpz_class &value){ + bool bResult = false; Lock(); + string keyStr_ = fea2string(fr, key[0], key[1], key[2], key[3]); + string keyStr = NormalizeToNFormat(keyStr_, 64); - unordered_map::const_iterator it; - // Search in data[pendingToFlushDataIndex].keyValue + map>::const_iterator it; + + // Search in data[pendingToFlushDataIndex].keyValueA if (bResult == false) { - it = data[pendingToFlushDataIndex].keyValue.find(version); - if (it != data[pendingToFlushDataIndex].keyValue.end()) - { - kv = it->second; - bResult = true; + uint64_t versionPrevious = previousAvailableVersion(version, data[pendingToFlushDataIndex].keyVersions[keyStr]); + if(versionPrevious != 0){ + it = data[pendingToFlushDataIndex].keyValueA.find(versionPrevious); + if (it != data[pendingToFlushDataIndex].keyValueA.end()) + { + for(auto it2 = it->second.begin(); it2 != it->second.end(); ++it2){ + if(it2->key[0]==key[0] && it2->key[1]==key[1] && it2->key[2]==key[2] && it2->key[3]==key[3]){ + value = it2->value; + bResult = true; + break; + } + } -#ifdef LOG_DB_MULTI_WRITE_FIND_NODES - zklog.info("MultiWrite64::findKeyValue() data[pendingToFlushDataIndex].keyValue found version=" + to_string(version) + " key=" + fea2string(fr, it->second.key.fe) + " value=" + it->second.value.get_str()); -#endif + #ifdef LOG_DB_MULTI_WRITE_FIND_NODES + zklog.info("MultiWrite64::findkeyValueA() data[pendingToFlushDataIndex].keyValueA found version=" + to_string(version) + " key=" + keyStr + " value=" + value.get_str()); + #endif + } } } + unordered_map>::const_iterator it_; - // Search in data[pendingToFlushDataIndex].nodesIntray + // Search in data[pendingToFlushDataIndex].keyValueAIntray if (bResult == false) { - it = data[pendingToFlushDataIndex].keyValueIntray.find(version); - if (it != data[pendingToFlushDataIndex].keyValueIntray.end()) - { - kv = it->second; - bResult = true; + uint64_t versionPrevious = previousAvailableVersion(version, data[pendingToFlushDataIndex].keyVersionsIntray[keyStr]); + if(versionPrevious != 0){ + it_ = data[pendingToFlushDataIndex].keyValueAIntray.find(versionPrevious); + if (it_ != data[pendingToFlushDataIndex].keyValueAIntray.end()) + { + for(auto it2 = it_->second.begin(); it2 != it_->second.end(); ++it2){ + if(it2->key[0]==key[0] && it2->key[1]==key[1] && it2->key[2]==key[2] && it2->key[3]==key[3]){ + value = it2->value; + bResult = true; + break; + } + } -#ifdef LOG_DB_MULTI_WRITE_FIND_NODES - zklog.info("MultiWrite64::findKeyValue() data[pendingToFlushDataIndex].keyValueIntray found version=" + to_string(version) + " key=" + fea2string(fr, it->second.key.fe) + " value=" + it->second.value.get_str()); -#endif + #ifdef LOG_DB_MULTI_WRITE_FIND_NODES + zklog.info("MultiWrite64::findkeyValueAIntray() data[pendingToFlushDataIndex].keyValueAIntray found version=" + to_string(version) + " key=" + keyStr + " value=" + value.get_str()); + #endif + } } } // If there is still some data pending to be stored on database if (storingFlushId != storedFlushId) { - // Search in data[storingDataIndex].keyValue + + // Search in data[storingDataIndex].keyValueA if (bResult == false) { - it = data[storingDataIndex].keyValue.find(version); - if (it != data[storingDataIndex].keyValue.end()) - { - kv = it->second; - bResult = true; -#ifdef LOG_DB_MULTI_WRITE_FIND_NODES - zklog.info("MultiWrite64::findKeyValue() data[storingDataIndex].keyValueIntray found version=" + to_string(version) + " key=" + fea2string(fr, it->second.key.fe) + " value=" + it->second.value.get_str()); -#endif + uint64_t versionPrevious = previousAvailableVersion(version, data[storingDataIndex].keyVersions[keyStr]); + if(versionPrevious != 0){ + it = data[storingDataIndex].keyValueA.find(versionPrevious); + if (it != data[storingDataIndex].keyValueA.end()) + { + for(auto it2 = it->second.begin(); it2 != it->second.end(); ++it2){ + if(it2->key[0]==key[0] && it2->key[1]==key[1] && it2->key[2]==key[2] && it2->key[3]==key[3]){ + value = it2->value; + bResult = true; + break; + } + } + + #ifdef LOG_DB_MULTI_WRITE_FIND_NODES + zklog.info("MultiWrite64::findkeyValueA() data[storingDataIndex].keyValueA found version=" + to_string(version) + " key=" + keyStr + " value=" + value.get_str()); + #endif + } } } + - // data[storingDataIndex].nodesIntray must be empty - zkassert(data[storingDataIndex].keyValueIntray.size() == 0); + // data[storingDataIndex].keyValueIntray must be empty + zkassert(data[storingDataIndex].keyValueAIntray.size() == 0); } Unlock(); diff --git a/src/hashdb64/multi_write_64.hpp b/src/hashdb64/multi_write_64.hpp index cae40ba37..37c696293 100644 --- a/src/hashdb64/multi_write_64.hpp +++ b/src/hashdb64/multi_write_64.hpp @@ -34,7 +34,7 @@ class MultiWrite64 bool findNode(const string &key, string &value); bool findProgram(const string &key, vector &value); - bool findKeyValue(const uint64_t version, KeyValue &kv); + bool findKeyValue(const uint64_t version,const Goldilocks::Element (&key)[4], mpz_class &value); bool findVersion(const string &key, uint64_t &version); }; diff --git a/src/hashdb64/multi_write_data_64.hpp b/src/hashdb64/multi_write_data_64.hpp index 4936e161a..bd86d313d 100644 --- a/src/hashdb64/multi_write_data_64.hpp +++ b/src/hashdb64/multi_write_data_64.hpp @@ -3,6 +3,7 @@ #include #include +#include #include "definitions.hpp" #include "zklog.hpp" #include "multi_query.hpp" @@ -18,8 +19,10 @@ class MultiWriteData64 unordered_map programIntray; unordered_map nodes; unordered_map nodesIntray; - unordered_map keyValue; - unordered_map keyValueIntray; + map> keyValueA; //keyValue must be inserted to the db in order with respect to version! + unordered_map> keyValueAIntray; + unordered_map> keyVersions; + unordered_map> keyVersionsIntray; unordered_map version; unordered_map versionIntray; uint64_t latestVersion; @@ -38,8 +41,10 @@ class MultiWriteData64 programIntray.clear(); nodes.clear(); nodesIntray.clear(); - keyValue.clear(); - keyValueIntray.clear(); + keyValueA.clear(); + keyValueAIntray.clear(); + keyVersions.clear(); + keyVersionsIntray.clear(); version.clear(); versionIntray.clear(); nodesStateRoot.clear(); @@ -54,8 +59,8 @@ class MultiWriteData64 (program.size() == 0) && (programIntray.size() == 0) && (nodesStateRoot.size() == 0) && - (keyValue.size() == 0) && - (keyValueIntray.size() == 0) && + (keyValueA.size() == 0) && + (keyValueAIntray.size() == 0) && (version.size() == 0) && (versionIntray.size() == 0); } @@ -84,16 +89,22 @@ class MultiWriteData64 nodes.merge(nodesIntray); nodesIntray.clear(); } - if (keyValueIntray.size() > 0) + if (keyValueAIntray.size() > 0) { #ifdef LOG_DB_ACCEPT_INTRAY if (bSenderCalling) { - zklog.info("MultiWriteData64::acceptIntray() rescuing " + to_string(keyValueIntray.size()) + " keyValue pairs"); + zklog.info("MultiWriteData64::acceptIntray() rescuing " + to_string(keyValueAIntray.size()) + " keyValueA pairs"); } -#endif - keyValue.merge(keyValueIntray); - keyValueIntray.clear(); +#endif + for(auto it = keyValueAIntray.begin(); it != keyValueAIntray.end(); ++it){ + if(keyValueA.find(it->first) == keyValueA.end()){ + keyValueA[it->first] = it->second; + }else{ + keyValueA[it->first].insert(keyValueA[it->first].end(), it->second.begin(), it->second.end()); + } + } + keyValueAIntray.clear(); } if (versionIntray.size() > 0) { @@ -106,6 +117,23 @@ class MultiWriteData64 version.merge(versionIntray); versionIntray.clear(); } + if (keyVersionsIntray.size() > 0) + { +#ifdef LOG_DB_ACCEPT_INTRAY + if (bSenderCalling) + { + zklog.info("MultiWriteData64::acceptIntray() rescuing " + to_string(keyVersionsIntray.size()) + " keyVersions"); + } +#endif + for(auto it = keyVersionsIntray.begin(); it != keyVersionsIntray.end(); ++it){ + if(keyVersions.find(it->first) == keyVersions.end()){ + keyVersions[it->first] = it->second; + }else{ + keyVersions[it->first].insert(keyVersions[it->first].end(), it->second.begin(), it->second.end()); + } + } + keyVersionsIntray.clear(); + } } }; From b91eccaf12fa4e0af7466d06cef8170b935c53b2 Mon Sep 17 00:00:00 2001 From: rickb80 Date: Tue, 12 Sep 2023 17:49:44 +0200 Subject: [PATCH 07/12] Manage links among versions in kv cache, eliminate update --- src/hashdb64/database_kv_associative_cache.cpp | 18 ++++++++++++------ src/hashdb64/database_kv_associative_cache.hpp | 3 ++- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/hashdb64/database_kv_associative_cache.cpp b/src/hashdb64/database_kv_associative_cache.cpp index e649d3d9c..17bc7bf72 100644 --- a/src/hashdb64/database_kv_associative_cache.cpp +++ b/src/hashdb64/database_kv_associative_cache.cpp @@ -81,6 +81,11 @@ void DatabaseKVAssociativeCache::postConstruct(int log2IndexesSize_, int log2Cac if(versions != NULL) delete[] versions; versions = new uint64_t[2 * cacheSize]; + #pragma omp parallel for schedule(static) num_threads(4) + for (size_t i = 0; i < cacheSize; i++) + { + indexes[i*2+1] = UINT64_MAX; + } currentCacheIndex = 0; attempts = 0; @@ -92,7 +97,7 @@ void DatabaseKVAssociativeCache::postConstruct(int log2IndexesSize_, int log2Cac indexesMask = indexesSize - 1; }; -void DatabaseKVAssociativeCache::addKeyValueVersion(const uint64_t version, const Goldilocks::Element (&key)[4], const mpz_class &value, bool update){ +void DatabaseKVAssociativeCache::addKeyValueVersion(const uint64_t version, const Goldilocks::Element (&key)[4], const mpz_class &value, bool link){ lock_guard guard(mlock); bool emptySlot = false; @@ -124,7 +129,7 @@ void DatabaseKVAssociativeCache::addKeyValueVersion(const uint64_t version, cons present = true; if(versions[cacheIndexVersions] == version){ presentSameVersion = true; - if(update == false) return; + return; } else if (versions[cacheIndexVersions] > version){ zklog.error("DatabaseKVAssociativeCache::addKeyValueVersion() adding lower version than the current one"); exitProcess(); @@ -168,10 +173,10 @@ void DatabaseKVAssociativeCache::addKeyValueVersion(const uint64_t version, cons keys[cacheIndexKey + 3].fe = key[3].fe; values[cacheIndexValue] = value; versions[cacheIndexVersions] = version; - if(present & !presentSameVersion){ + if(present && !presentSameVersion && link){ versions[cacheIndexVersions+1] = cacheIndexPrev; }else{ - versions[cacheIndexVersions+1] = 0; + versions[cacheIndexVersions+1] = UINT64_MAX; } // // Forced index insertion @@ -274,16 +279,17 @@ bool DatabaseKVAssociativeCache::findKey( const uint64_t version, const Goldiloc keys[cacheIndexKey + 2].fe == key[2].fe && keys[cacheIndexKey + 3].fe == key[3].fe){ - if( versions[cacheIndexVersions] <= version){ //rick: I assume they are ordered + if( versions[cacheIndexVersions] <= version){ //We assume they are ordered uint32_t cacheIndexValue = cacheIndex; ++hits; value = values[cacheIndexValue]; return true; } + if(versions[cacheIndexVersions+1]==UINT64_MAX) return false; //No more versions cacheIndex = versions[cacheIndexVersions+1] & cacheMask; cacheIndexKey = cacheIndex * 4; cacheIndexVersions = cacheIndex * 2; - + }else{ if(j>0){ return false; diff --git a/src/hashdb64/database_kv_associative_cache.hpp b/src/hashdb64/database_kv_associative_cache.hpp index 0bbcf744f..b931c7fc8 100644 --- a/src/hashdb64/database_kv_associative_cache.hpp +++ b/src/hashdb64/database_kv_associative_cache.hpp @@ -7,6 +7,7 @@ #include "zklog.hpp" #include "zkmax.hpp" #include "poseidon_goldilocks.hpp" +#include "version_value.hpp" using namespace std; @@ -42,7 +43,7 @@ class DatabaseKVAssociativeCache ~DatabaseKVAssociativeCache(); void postConstruct(int log2IndexesSize_, int log2CacheSize_, string name_); - void addKeyValueVersion(const uint64_t version, const Goldilocks::Element (&key)[4], const mpz_class &value, bool update); + void addKeyValueVersion(const uint64_t version, const Goldilocks::Element (&key)[4], const mpz_class &value, bool link = true); bool findKey( const uint64_t version, const Goldilocks::Element (&key)[4], mpz_class &value); inline bool enabled() const { return (log2IndexesSize > 0); }; From 498733c931fdfba3f49ecc2b3f92d87c66837d9a Mon Sep 17 00:00:00 2001 From: rickb80 Date: Tue, 12 Sep 2023 18:44:43 +0200 Subject: [PATCH 08/12] readRemoteKV extracts the upstream versions --- src/hashdb64/database_64.cpp | 71 ++++++++++++++---------------------- src/hashdb64/database_64.hpp | 5 ++- 2 files changed, 30 insertions(+), 46 deletions(-) diff --git a/src/hashdb64/database_64.cpp b/src/hashdb64/database_64.cpp index c882083f0..af8db0bb5 100644 --- a/src/hashdb64/database_64.cpp +++ b/src/hashdb64/database_64.cpp @@ -43,7 +43,8 @@ Database64::Database64 (Goldilocks &fr, const Config &config) : config(config), connectionsPool(NULL), multiWrite(fr), - maxVersions(config.kvDBMaxVersions) + maxVersions(config.kvDBMaxVersions), + maxVersionsUpload(10) //add into config if needed { // Init mutex pthread_mutex_init(&connMutex, NULL); @@ -403,15 +404,13 @@ zkresult Database64::readKV(const Goldilocks::Element (&root)[4], const Goldiloc { // Add to the read log if (dbReadLog != NULL) dbReadLog->add(keyStr, value, true, TimeDiff(t)); - - // Store it locally to avoid any future remote access for this key - dbKVACache.addKeyValueVersion(version, key, value, false); //rick problematic + // We do not store into cache as we do not want to manage the chain of versions rkv = ZKR_SUCCESS; } else if(useRemoteDB) { - - rkv = readRemoteKV(version, key, value); + vector upstremVersionValues; + rkv = readRemoteKV(version, key, value, upstremVersionValues); if (rkv == ZKR_SUCCESS) { dbKVACache.addKeyValueVersion(version, key, value, false); //rick problematic @@ -422,7 +421,8 @@ zkresult Database64::readKV(const Goldilocks::Element (&root)[4], const Goldiloc value = 0; rout = rkv; mpz_class zero(0); - dbKVACache.addKeyValueVersion(version, key, zero, false); //rick problematic + // Add a zero into the cache to avoid future remote access for this key (not problematic management of versions as there is only one version) + dbKVACache.addKeyValueVersion(version, key, zero, false); }else{ zkresult rtree = ZKR_UNSPECIFIED; vector keyValues(1); @@ -474,54 +474,28 @@ zkresult Database64::readKV(const Goldilocks::Element (&root)[4], vector &upstreamVersionValues) { const string &tableName = config.dbKeyValueTableName; @@ -1271,7 +1245,7 @@ zkresult Database64::readRemoteKV(const uint64_t version, const Goldilocks::Elem // Dispose the read db conneciton disposeConnection(pDatabaseConnection); - return extractVersion(row[1], version, value); + return extractVersion(row[1], version, value, upstreamVersionValues); } @@ -1289,22 +1263,32 @@ zkresult Database64::readRemoteKV(const uint64_t version, const Goldilocks::Elem return ZKR_DB_ERROR;//should never return here } -zkresult Database64::extractVersion(const pqxx::field& fieldData, const uint64_t version, mpz_class &value){ +zkresult Database64::extractVersion(const pqxx::field& fieldData, const uint64_t version, mpz_class &value, vector &upstreamVersionValues){ + upstreamVersionValues.clear(); int data_size = 0; if(!fieldData.is_null()){ string data = removeBSXIfExists64(fieldData.c_str()); data_size = data.size(); zkassert(data_size % 80 == 0); + int nUpstreams = 0; for (int i = 0; i < data_size; i += 80) { + string versionStr = data.substr(i, 16); mpz_class aux(versionStr, 16); uint64_t version_ = aux.get_ui(); + if(nUpstreams < maxVersions){ + VersionValue vv; + vv.version = version_; + vv.value = mpz_class(data.substr(i + 16, 64), 16); + upstreamVersionValues.push_back(vv); + } if(version_==version){ value = mpz_class(data.substr(i + 16, 64), 16); return ZKR_SUCCESS; } + ++nUpstreams; } } /*const char * data = fieldData.c_str(); @@ -1411,7 +1395,6 @@ zkresult Database64::writeRemoteKV(const uint64_t version, const Goldilocks::Ele } } else{ if(version > 1){ - //this must be reproduced into multiwrite mpz_class zero(0); dbKVACache.addKeyValueVersion(version-1, key, zero, false); } diff --git a/src/hashdb64/database_64.hpp b/src/hashdb64/database_64.hpp index e4bee0352..50faa0f2c 100644 --- a/src/hashdb64/database_64.hpp +++ b/src/hashdb64/database_64.hpp @@ -74,6 +74,7 @@ class Database64 pthread_t senderPthread; // Database sender thread pthread_t cacheSynchPthread; // Cache synchronization thread int maxVersions; // Maximum number of versions to store in the database KV + int maxVersionsUpload; // Maximum number of versions to upload from the database KV to the cache when there is a cache miss private: // Remote database based on Postgres (PostgreSQL) @@ -83,14 +84,14 @@ class Database64 zkresult writeRemote(bool bProgram, const string &key, const string &value); zkresult writeGetTreeFunction(void); - zkresult readRemoteKV(const uint64_t version, const Goldilocks::Element (&key)[4], mpz_class value); + zkresult readRemoteKV(const uint64_t version, const Goldilocks::Element (&key)[4], mpz_class value, vector &upstreamVersionValues); zkresult writeRemoteKV(const uint64_t version, const Goldilocks::Element (&key)[4], const mpz_class &value, bool useMultiWrite = true); zkresult readRemoteVersion(const Goldilocks::Element (&root)[4], uint64_t version); zkresult writeRemoteVersion(const Goldilocks::Element (&root)[4], const uint64_t version); zkresult readRemoteLatestVersion(uint64_t &version); zkresult writeRemoteLatestVersion(const uint64_t version); - zkresult extractVersion(const pqxx::field& fieldData, const uint64_t version, mpz_class &value); + zkresult extractVersion(const pqxx::field& fieldData, const uint64_t version, mpz_class &value, vector &upstreamVersionValues); public: #ifdef DATABASE_USE_CACHE From 7e13644703fe3d86583786d5541fe64af8106679 Mon Sep 17 00:00:00 2001 From: rickb80 Date: Wed, 13 Sep 2023 11:33:10 +0200 Subject: [PATCH 09/12] downstreamAddKeyZero and uploadKeyValueVersion --- src/hashdb64/database_64.cpp | 2 +- .../database_kv_associative_cache.cpp | 127 +++++++++++++++++- .../database_kv_associative_cache.hpp | 3 + src/hashdb64/version_value.hpp | 13 ++ 4 files changed, 142 insertions(+), 3 deletions(-) create mode 100644 src/hashdb64/version_value.hpp diff --git a/src/hashdb64/database_64.cpp b/src/hashdb64/database_64.cpp index ad68d8872..58d5dd0ed 100644 --- a/src/hashdb64/database_64.cpp +++ b/src/hashdb64/database_64.cpp @@ -1224,7 +1224,7 @@ zkresult Database64::writeRemoteKV(const uint64_t version, const Goldilocks::Ele } else{ if(version > 1){ mpz_class zero(0); - dbKVACache.addKeyValueVersion(version-1, key, zero, false); + dbKVACache.downstreamAddKeyZeroVersion(version, key); } } } diff --git a/src/hashdb64/database_kv_associative_cache.cpp b/src/hashdb64/database_kv_associative_cache.cpp index 17bc7bf72..38d747af0 100644 --- a/src/hashdb64/database_kv_associative_cache.cpp +++ b/src/hashdb64/database_kv_associative_cache.cpp @@ -84,7 +84,7 @@ void DatabaseKVAssociativeCache::postConstruct(int log2IndexesSize_, int log2Cac #pragma omp parallel for schedule(static) num_threads(4) for (size_t i = 0; i < cacheSize; i++) { - indexes[i*2+1] = UINT64_MAX; + versions[i*2+1] = UINT64_MAX; } currentCacheIndex = 0; @@ -105,7 +105,7 @@ void DatabaseKVAssociativeCache::addKeyValueVersion(const uint64_t version, cons bool presentSameVersion = false; uint32_t cacheIndex; uint32_t tableIndexUse=0; - uint32_t cacheIndexPrev; + uint64_t cacheIndexPrev=UINT64_MAX; // // Check if present in one of the four slots @@ -189,6 +189,129 @@ void DatabaseKVAssociativeCache::addKeyValueVersion(const uint64_t version, cons } } +void DatabaseKVAssociativeCache::downstreamAddKeyZeroVersion(const uint64_t version, const Goldilocks::Element (&key)[4]){ + + if(version<2){ + + zklog.error("DatabaseKVAssociativeCache::downstreamAddKeyZeroVersion() version must be greater than 1"); + return; + } + lock_guard guard(mlock); + bool presentSameVersion = false; + uint32_t cacheIndexZero; + // + // Check if present in one of the four slots + // + Goldilocks::Element key_hashed[4]; + hashKey(key_hashed, key); + for (int i = 0; i < 4; ++i) + { + uint32_t tableIndex = (uint32_t)(key_hashed[i].fe & indexesMask); + uint32_t cacheIndexRaw = indexes[tableIndex]; + uint32_t cacheIndex = cacheIndexRaw & cacheMask; + uint32_t cacheIndexKey = cacheIndex * 4; + uint32_t cacheIndexVersions = cacheIndex * 2; + + if (!emptyCacheSlot(cacheIndexRaw)){ + if( keys[cacheIndexKey + 0].fe == key[0].fe && + keys[cacheIndexKey + 1].fe == key[1].fe && + keys[cacheIndexKey + 2].fe == key[2].fe && + keys[cacheIndexKey + 3].fe == key[3].fe && + versions[cacheIndexVersions] == version && + versions[cacheIndexVersions+1] == UINT64_MAX){ //if linked zero can not be added + presentSameVersion = true; + cacheIndexZero = currentCacheIndex & cacheMask; + currentCacheIndex = (currentCacheIndex == UINT32_MAX) ? 0 : (currentCacheIndex + 1); + versions[cacheIndexVersions+1] = cacheIndexZero; + break; + } + } + } + + // + // Evaluate cacheIndexKey and + // + if(presentSameVersion){ + + uint64_t cacheIndexKey = cacheIndexZero * 4; + uint64_t cacheIndexValue = cacheIndexZero; + uint64_t cacheIndexVersions = cacheIndexZero * 2; + + // + // Add value + // + keys[cacheIndexKey + 0].fe = key[0].fe; + keys[cacheIndexKey + 1].fe = key[1].fe; + keys[cacheIndexKey + 2].fe = key[2].fe; + keys[cacheIndexKey + 3].fe = key[3].fe; + values[cacheIndexValue] = 0; + versions[cacheIndexVersions] = version-1; + versions[cacheIndexVersions+1] = UINT64_MAX; + } + return; + +} + +void DatabaseKVAssociativeCache::uploadKeyValueVersions(const Goldilocks::Element (&key)[4], vector &versionsValues){ + + + //Get last version for the key + uint64_t lastVersion = getLastCachedVersion(key); + if(lastVersion == UINT64_MAX){ + for (std::vector::reverse_iterator it = versionsValues.rbegin(); it != versionsValues.rend(); ++it) { + addKeyValueVersion(it->version, key, it->value, true); + } + }else{ + //rick: but this situation is not tolerable no?? + + //Find the verison in my vector and add from there + bool found = false; + for(std::vector::reverse_iterator it = versionsValues.rbegin(); it != versionsValues.rend(); ++it) { + if(found){ + addKeyValueVersion(it->version, key, it->value, true); + } + if(it->version == lastVersion){ + found = true; + } + } + if(!found){ + bool linked = false; + for (std::vector::reverse_iterator it = versionsValues.rbegin(); it != versionsValues.rend(); ++it) { + addKeyValueVersion(it->version, key, it->value, linked); + linked = true; + } + } + } +} + + +uint64_t DatabaseKVAssociativeCache::getLastCachedVersion(const Goldilocks::Element (&key)[4]){ + lock_guard guard(mlock); + // + // Check if present in one of the four slots + // + Goldilocks::Element key_hashed[4]; + hashKey(key_hashed, key); + for (int i = 0; i < 4; ++i) + { + uint32_t tableIndex = (uint32_t)(key_hashed[i].fe & indexesMask); + uint32_t cacheIndexRaw = indexes[tableIndex]; + uint32_t cacheIndex = cacheIndexRaw & cacheMask; + uint32_t cacheIndexKey = cacheIndex * 4; + uint32_t cacheIndexVersions = cacheIndex * 2; + + if (!emptyCacheSlot(cacheIndexRaw)){ + if( keys[cacheIndexKey + 0].fe == key[0].fe && + keys[cacheIndexKey + 1].fe == key[1].fe && + keys[cacheIndexKey + 2].fe == key[2].fe && + keys[cacheIndexKey + 3].fe == key[3].fe){ + return versions[cacheIndexVersions]; + } + } + } + return UINT64_MAX; +} + void DatabaseKVAssociativeCache::forcedInsertion(uint32_t (&usedRawCacheIndexes)[10], int &iters) { uint32_t inputRawCacheIndex = usedRawCacheIndexes[iters]; diff --git a/src/hashdb64/database_kv_associative_cache.hpp b/src/hashdb64/database_kv_associative_cache.hpp index b931c7fc8..c46f1c869 100644 --- a/src/hashdb64/database_kv_associative_cache.hpp +++ b/src/hashdb64/database_kv_associative_cache.hpp @@ -44,7 +44,10 @@ class DatabaseKVAssociativeCache void postConstruct(int log2IndexesSize_, int log2CacheSize_, string name_); void addKeyValueVersion(const uint64_t version, const Goldilocks::Element (&key)[4], const mpz_class &value, bool link = true); + void downstreamAddKeyZeroVersion(const uint64_t version, const Goldilocks::Element (&key)[4]); + void uploadKeyValueVersions(const Goldilocks::Element (&key)[4], vector &versionsValues); bool findKey( const uint64_t version, const Goldilocks::Element (&key)[4], mpz_class &value); + uint64_t getLastCachedVersion(const Goldilocks::Element (&key)[4]); inline bool enabled() const { return (log2IndexesSize > 0); }; inline uint32_t getCacheSize() const { return cacheSize; }; diff --git a/src/hashdb64/version_value.hpp b/src/hashdb64/version_value.hpp new file mode 100644 index 000000000..3eb18b37a --- /dev/null +++ b/src/hashdb64/version_value.hpp @@ -0,0 +1,13 @@ +#ifndef VERSION_VALUE_HPP +#define VERSION_VALUE_HPP + +#include + +class VersionValue +{ +public: + uint64_t version; + mpz_class value; +}; + +#endif \ No newline at end of file From c35607ebe87e775cf12b5d0e7d447d0ee5fcc111 Mon Sep 17 00:00:00 2001 From: rickb80 Date: Wed, 13 Sep 2023 16:47:43 +0200 Subject: [PATCH 10/12] fix issues in the kv cache design --- src/hashdb64/database_64.cpp | 26 ++++--- .../database_kv_associative_cache.cpp | 73 ++++++++++--------- .../database_kv_associative_cache.hpp | 5 +- .../database_associative_cache_test.cpp | 2 +- 4 files changed, 60 insertions(+), 46 deletions(-) diff --git a/src/hashdb64/database_64.cpp b/src/hashdb64/database_64.cpp index 58d5dd0ed..25a72480a 100644 --- a/src/hashdb64/database_64.cpp +++ b/src/hashdb64/database_64.cpp @@ -358,16 +358,21 @@ zkresult Database64::readKV(const Goldilocks::Element (&root)[4], const Goldiloc rkv = readRemoteKV(version, key, value, upstremVersionValues); if (rkv == ZKR_SUCCESS) { - dbKVACache.addKeyValueVersion(version, key, value, false); //rick problematic + dbKVACache.uploadKeyValueVersions(key, upstremVersionValues); if (dbReadLog != NULL) dbReadLog->add(keyStr, value, false, TimeDiff(t)); } else { - if( rkv == ZKR_DB_VERSION_NOT_FOUND_GLOBAL){ + if( rkv == ZKR_DB_KEY_NOT_FOUND){ + rout = rkv; + // Add a zero into the cache to avoid future remote access for this key (not problematic management of versions as there is only one version) + mpz_class zero(0); + dbKVACache.addKeyValueVersion(0, key, zero); + }else if( rkv == ZKR_DB_VERSION_NOT_FOUND_GLOBAL){ value = 0; rout = rkv; mpz_class zero(0); // Add a zero into the cache to avoid future remote access for this key (not problematic management of versions as there is only one version) - dbKVACache.addKeyValueVersion(version, key, zero, false); + dbKVACache.uploadKeyValueVersions(key, upstremVersionValues); }else{ zkresult rtree = ZKR_UNSPECIFIED; vector keyValues(1); @@ -476,7 +481,7 @@ zkresult Database64::writeKV(const uint64_t& version, const Goldilocks::Element if (rkv == ZKR_SUCCESS) { - dbKVACache.addKeyValueVersion(version, key, value, false); + dbKVACache.addKeyValueVersion(version, key, value); } @@ -1056,7 +1061,7 @@ zkresult Database64::readRemoteKV(const uint64_t version, const Goldilocks::Elem if (rows.size() == 0) { disposeConnection(pDatabaseConnection); - return ZKR_DB_VERSION_NOT_FOUND_GLOBAL; + return ZKR_DB_KEY_NOT_FOUND; } else if (rows.size() > 1) { @@ -1222,10 +1227,13 @@ zkresult Database64::writeRemoteKV(const uint64_t version, const Goldilocks::Ele insertStr = insertStr + data; } } else{ - if(version > 1){ - mpz_class zero(0); - dbKVACache.downstreamAddKeyZeroVersion(version, key); - } + string valueZero = NormalizeToNFormat("0",64); + string versionZero = NormalizeToNFormat(U64toString(0,16),16); + string insertZero = versionStr + valueStr; + insertStr = insertStr + versionZero + valueZero; + mpz_class zero(0); + dbKVACache.downstreamAddKeyZeroVersion(version, key); + } } catch (const std::exception &e) diff --git a/src/hashdb64/database_kv_associative_cache.cpp b/src/hashdb64/database_kv_associative_cache.cpp index 38d747af0..d18b31e38 100644 --- a/src/hashdb64/database_kv_associative_cache.cpp +++ b/src/hashdb64/database_kv_associative_cache.cpp @@ -97,7 +97,7 @@ void DatabaseKVAssociativeCache::postConstruct(int log2IndexesSize_, int log2Cac indexesMask = indexesSize - 1; }; -void DatabaseKVAssociativeCache::addKeyValueVersion(const uint64_t version, const Goldilocks::Element (&key)[4], const mpz_class &value, bool link){ +void DatabaseKVAssociativeCache::addKeyValueVersion(const uint64_t version, const Goldilocks::Element (&key)[4], const mpz_class &value){ lock_guard guard(mlock); bool emptySlot = false; @@ -173,7 +173,7 @@ void DatabaseKVAssociativeCache::addKeyValueVersion(const uint64_t version, cons keys[cacheIndexKey + 3].fe = key[3].fe; values[cacheIndexValue] = value; versions[cacheIndexVersions] = version; - if(present && !presentSameVersion && link){ + if(present && !presentSameVersion){ versions[cacheIndexVersions+1] = cacheIndexPrev; }else{ versions[cacheIndexVersions+1] = UINT64_MAX; @@ -254,62 +254,67 @@ void DatabaseKVAssociativeCache::downstreamAddKeyZeroVersion(const uint64_t vers void DatabaseKVAssociativeCache::uploadKeyValueVersions(const Goldilocks::Element (&key)[4], vector &versionsValues){ - + lock_guard guard(mlock); //Get last version for the key - uint64_t lastVersion = getLastCachedVersion(key); - if(lastVersion == UINT64_MAX){ + vector versions; + getLastCachedVersions(key, versions, versionsValues.size()); + if(versions.size()==0){ for (std::vector::reverse_iterator it = versionsValues.rbegin(); it != versionsValues.rend(); ++it) { - addKeyValueVersion(it->version, key, it->value, true); + addKeyValueVersion(it->version, key, it->value); } }else{ - //rick: but this situation is not tolerable no?? - - //Find the verison in my vector and add from there - bool found = false; - for(std::vector::reverse_iterator it = versionsValues.rbegin(); it != versionsValues.rend(); ++it) { - if(found){ - addKeyValueVersion(it->version, key, it->value, true); - } - if(it->version == lastVersion){ - found = true; - } - } - if(!found){ - bool linked = false; - for (std::vector::reverse_iterator it = versionsValues.rbegin(); it != versionsValues.rend(); ++it) { - addKeyValueVersion(it->version, key, it->value, linked); - linked = true; + vector::const_iterator it = versionsValues.begin(); + for(vector::const_iterator it2 = versions.begin(); it2 != versions.end(); ++it2){ + if(*it2 != it->version){ + zklog.error("DatabaseKVAssociativeCache::uploadKeyValueVersions() versions mismatch between cache and db"); + exitProcess(); } } + //If vector.zize() < versionsValues.size() we could add some backward versions in the cache but not supported yet } } - -uint64_t DatabaseKVAssociativeCache::getLastCachedVersion(const Goldilocks::Element (&key)[4]){ +void DatabaseKVAssociativeCache::getLastCachedVersions(const Goldilocks::Element (&key)[4], vector &versions, const int maxVersions){ + + versions.clear(); lock_guard guard(mlock); // - // Check if present in one of the four slots + // Find the value // Goldilocks::Element key_hashed[4]; hashKey(key_hashed, key); - for (int i = 0; i < 4; ++i) + for (int i = 0; i < 4; i++) { - uint32_t tableIndex = (uint32_t)(key_hashed[i].fe & indexesMask); - uint32_t cacheIndexRaw = indexes[tableIndex]; - uint32_t cacheIndex = cacheIndexRaw & cacheMask; + uint32_t cacheIndexRaw = indexes[key_hashed[i].fe & indexesMask]; + if (emptyCacheSlot(cacheIndexRaw)) continue; + + uint32_t cacheIndex = cacheIndexRaw & cacheMask; uint32_t cacheIndexKey = cacheIndex * 4; uint32_t cacheIndexVersions = cacheIndex * 2; - if (!emptyCacheSlot(cacheIndexRaw)){ - if( keys[cacheIndexKey + 0].fe == key[0].fe && + for(int j=0; j0){ + return; + }else{ + break; + } } } } - return UINT64_MAX; + return; + } void DatabaseKVAssociativeCache::forcedInsertion(uint32_t (&usedRawCacheIndexes)[10], int &iters) diff --git a/src/hashdb64/database_kv_associative_cache.hpp b/src/hashdb64/database_kv_associative_cache.hpp index c46f1c869..1fbecd580 100644 --- a/src/hashdb64/database_kv_associative_cache.hpp +++ b/src/hashdb64/database_kv_associative_cache.hpp @@ -43,11 +43,12 @@ class DatabaseKVAssociativeCache ~DatabaseKVAssociativeCache(); void postConstruct(int log2IndexesSize_, int log2CacheSize_, string name_); - void addKeyValueVersion(const uint64_t version, const Goldilocks::Element (&key)[4], const mpz_class &value, bool link = true); + void addKeyValueVersion(const uint64_t version, const Goldilocks::Element (&key)[4], const mpz_class &value); void downstreamAddKeyZeroVersion(const uint64_t version, const Goldilocks::Element (&key)[4]); void uploadKeyValueVersions(const Goldilocks::Element (&key)[4], vector &versionsValues); bool findKey( const uint64_t version, const Goldilocks::Element (&key)[4], mpz_class &value); - uint64_t getLastCachedVersion(const Goldilocks::Element (&key)[4]); + void getLastCachedVersions(const Goldilocks::Element (&key)[4], vector &versions, const int maxVersions); + inline bool enabled() const { return (log2IndexesSize > 0); }; inline uint32_t getCacheSize() const { return cacheSize; }; diff --git a/test/hashdb/database_associative_cache_test.cpp b/test/hashdb/database_associative_cache_test.cpp index 863ebaacb..0ba33b3e3 100644 --- a/test/hashdb/database_associative_cache_test.cpp +++ b/test/hashdb/database_associative_cache_test.cpp @@ -95,7 +95,7 @@ uint64_t DatabaseAssociativeCacheTest (void) keyScalar = i; scalar2fea(fr, keyScalar, key); valueScalar = (version/2)*NUMBER_OF_DB_CACHE_ADDS + i; - dbKVACache.addKeyValueVersion(version, key, valueScalar,update); + dbKVACache.addKeyValueVersion(version, key, valueScalar); } } for(version=0; version<=10; version++){ From faebe03debd2342c12bceff9a649e3fe82ceaa7e Mon Sep 17 00:00:00 2001 From: rickb80 Date: Wed, 13 Sep 2023 23:00:09 +0200 Subject: [PATCH 11/12] Fixes in KV cacha and database --- src/hashdb64/database_64.cpp | 5 +- .../database_kv_associative_cache.cpp | 82 +++++++++++-------- src/hashdb64/multi_write_64.cpp | 44 +++++----- src/hashdb64/multi_write_data_64.hpp | 2 + 4 files changed, 73 insertions(+), 60 deletions(-) diff --git a/src/hashdb64/database_64.cpp b/src/hashdb64/database_64.cpp index 25a72480a..a3e310a37 100644 --- a/src/hashdb64/database_64.cpp +++ b/src/hashdb64/database_64.cpp @@ -368,9 +368,7 @@ zkresult Database64::readKV(const Goldilocks::Element (&root)[4], const Goldiloc mpz_class zero(0); dbKVACache.addKeyValueVersion(0, key, zero); }else if( rkv == ZKR_DB_VERSION_NOT_FOUND_GLOBAL){ - value = 0; rout = rkv; - mpz_class zero(0); // Add a zero into the cache to avoid future remote access for this key (not problematic management of versions as there is only one version) dbKVACache.uploadKeyValueVersions(key, upstremVersionValues); }else{ @@ -1117,7 +1115,7 @@ zkresult Database64::extractVersion(const pqxx::field& fieldData, const uint64_t vv.value = mpz_class(data.substr(i + 16, 64), 16); upstreamVersionValues.push_back(vv); } - if(version_==version){ + if(version >= version_){ value = mpz_class(data.substr(i + 16, 64), 16); return ZKR_SUCCESS; } @@ -1976,6 +1974,7 @@ zkresult Database64::sendData (void) { writeRemoteKV(it->first,it2->key,it2->value, false); } + ++it; } } // If there are versions add to db diff --git a/src/hashdb64/database_kv_associative_cache.cpp b/src/hashdb64/database_kv_associative_cache.cpp index d18b31e38..701367dd8 100644 --- a/src/hashdb64/database_kv_associative_cache.cpp +++ b/src/hashdb64/database_kv_associative_cache.cpp @@ -14,7 +14,7 @@ DatabaseKVAssociativeCache::DatabaseKVAssociativeCache() indexesSize = 0; log2CacheSize = 0; cacheSize = 0; - maxVersions = 100; //rick: as parameter + maxVersions = 0; indexes = NULL; keys = NULL; values = NULL; @@ -61,8 +61,7 @@ void DatabaseKVAssociativeCache::postConstruct(int log2IndexesSize_, int log2Cac exitProcess(); } cacheSize = 1 << log2CacheSize_; - maxVersions = 100; //rick: as parameter - + maxVersions = cacheSize; if(indexes != NULL) delete[] indexes; indexes = new uint32_t[indexesSize]; @@ -102,7 +101,6 @@ void DatabaseKVAssociativeCache::addKeyValueVersion(const uint64_t version, cons lock_guard guard(mlock); bool emptySlot = false; bool present = false; - bool presentSameVersion = false; uint32_t cacheIndex; uint32_t tableIndexUse=0; uint64_t cacheIndexPrev=UINT64_MAX; @@ -128,9 +126,8 @@ void DatabaseKVAssociativeCache::addKeyValueVersion(const uint64_t version, cons keys[cacheIndexKey + 3].fe == key[3].fe){ present = true; if(versions[cacheIndexVersions] == version){ - presentSameVersion = true; - return; - } else if (versions[cacheIndexVersions] > version){ + return; //no update + } else if (version < versions[cacheIndexVersions]){ zklog.error("DatabaseKVAssociativeCache::addKeyValueVersion() adding lower version than the current one"); exitProcess(); } else { @@ -152,13 +149,12 @@ void DatabaseKVAssociativeCache::addKeyValueVersion(const uint64_t version, cons // // Evaluate cacheIndexKey and // - if(!presentSameVersion){ - if(emptySlot == true || present){ - indexes[tableIndexUse] = currentCacheIndex; - } - cacheIndex = (uint32_t)(currentCacheIndex & cacheMask); - currentCacheIndex = (currentCacheIndex == UINT32_MAX) ? 0 : (currentCacheIndex + 1); + if(emptySlot == true || present){ + indexes[tableIndexUse] = currentCacheIndex; } + cacheIndex = (uint32_t)(currentCacheIndex & cacheMask); + currentCacheIndex = (currentCacheIndex == UINT32_MAX) ? 0 : (currentCacheIndex + 1); + uint64_t cacheIndexKey, cacheIndexValue, cacheIndexVersions; cacheIndexKey = cacheIndex * 4; cacheIndexValue = cacheIndex; @@ -173,7 +169,7 @@ void DatabaseKVAssociativeCache::addKeyValueVersion(const uint64_t version, cons keys[cacheIndexKey + 3].fe = key[3].fe; values[cacheIndexValue] = value; versions[cacheIndexVersions] = version; - if(present && !presentSameVersion){ + if(present){ versions[cacheIndexVersions+1] = cacheIndexPrev; }else{ versions[cacheIndexVersions+1] = UINT64_MAX; @@ -191,19 +187,15 @@ void DatabaseKVAssociativeCache::addKeyValueVersion(const uint64_t version, cons void DatabaseKVAssociativeCache::downstreamAddKeyZeroVersion(const uint64_t version, const Goldilocks::Element (&key)[4]){ - if(version<2){ - - zklog.error("DatabaseKVAssociativeCache::downstreamAddKeyZeroVersion() version must be greater than 1"); - return; - } lock_guard guard(mlock); bool presentSameVersion = false; - uint32_t cacheIndexZero; + uint32_t cacheIndexZero = 0; // // Check if present in one of the four slots // Goldilocks::Element key_hashed[4]; hashKey(key_hashed, key); + bool breakOuterLoop = false; for (int i = 0; i < 4; ++i) { uint32_t tableIndex = (uint32_t)(key_hashed[i].fe & indexesMask); @@ -213,19 +205,37 @@ void DatabaseKVAssociativeCache::downstreamAddKeyZeroVersion(const uint64_t vers uint32_t cacheIndexVersions = cacheIndex * 2; if (!emptyCacheSlot(cacheIndexRaw)){ - if( keys[cacheIndexKey + 0].fe == key[0].fe && - keys[cacheIndexKey + 1].fe == key[1].fe && - keys[cacheIndexKey + 2].fe == key[2].fe && - keys[cacheIndexKey + 3].fe == key[3].fe && - versions[cacheIndexVersions] == version && - versions[cacheIndexVersions+1] == UINT64_MAX){ //if linked zero can not be added - presentSameVersion = true; - cacheIndexZero = currentCacheIndex & cacheMask; - currentCacheIndex = (currentCacheIndex == UINT32_MAX) ? 0 : (currentCacheIndex + 1); - versions[cacheIndexVersions+1] = cacheIndexZero; - break; + for(int j=0; j0){ + return; + }else{ + break; + } + } } } + if(breakOuterLoop) break; } // @@ -245,7 +255,7 @@ void DatabaseKVAssociativeCache::downstreamAddKeyZeroVersion(const uint64_t vers keys[cacheIndexKey + 2].fe = key[2].fe; keys[cacheIndexKey + 3].fe = key[3].fe; values[cacheIndexValue] = 0; - versions[cacheIndexVersions] = version-1; + versions[cacheIndexVersions] = 0; versions[cacheIndexVersions+1] = UINT64_MAX; } return; @@ -264,7 +274,7 @@ void DatabaseKVAssociativeCache::uploadKeyValueVersions(const Goldilocks::Elemen } }else{ vector::const_iterator it = versionsValues.begin(); - for(vector::const_iterator it2 = versions.begin(); it2 != versions.end(); ++it2){ + for(vector::const_iterator it2 = versions.begin(); it2 != versions.end(); ++it2, ++it){ if(*it2 != it->version){ zklog.error("DatabaseKVAssociativeCache::uploadKeyValueVersions() versions mismatch between cache and db"); exitProcess(); @@ -274,7 +284,7 @@ void DatabaseKVAssociativeCache::uploadKeyValueVersions(const Goldilocks::Elemen } } -void DatabaseKVAssociativeCache::getLastCachedVersions(const Goldilocks::Element (&key)[4], vector &versions, const int maxVersions){ +void DatabaseKVAssociativeCache::getLastCachedVersions(const Goldilocks::Element (&key)[4], vector &versions, const int maxVersionsOut){ versions.clear(); lock_guard guard(mlock); @@ -292,7 +302,7 @@ void DatabaseKVAssociativeCache::getLastCachedVersions(const Goldilocks::Element uint32_t cacheIndexKey = cacheIndex * 4; uint32_t cacheIndexVersions = cacheIndex * 2; - for(int j=0; j= versions[cacheIndexVersions] ){ uint32_t cacheIndexValue = cacheIndex; ++hits; value = values[cacheIndexValue]; diff --git a/src/hashdb64/multi_write_64.cpp b/src/hashdb64/multi_write_64.cpp index da49eef9c..28b8ab8b6 100644 --- a/src/hashdb64/multi_write_64.cpp +++ b/src/hashdb64/multi_write_64.cpp @@ -6,10 +6,10 @@ using namespace std; uint64_t previousAvailableVersion(const uint64_t versionIn, const vector &versions) { - uint64_t versionOut = 0; + uint64_t versionOut = UINT64_MAX; for (auto it = versions.begin(); it != versions.end(); ++it) { - if (*it < versionIn && *it > versionOut) + if (*it <= versionIn) { versionOut = *it; } @@ -187,19 +187,19 @@ bool MultiWrite64::findKeyValue(const uint64_t version,const Goldilocks::Element Lock(); string keyStr_ = fea2string(fr, key[0], key[1], key[2], key[3]); string keyStr = NormalizeToNFormat(keyStr_, 64); + + unordered_map>::const_iterator it_; - - map>::const_iterator it; - - // Search in data[pendingToFlushDataIndex].keyValueA + // Search in data[pendingToFlushDataIndex].keyValueAIntray + // Very important to start locking for intray first since in has newever versions if (bResult == false) { - uint64_t versionPrevious = previousAvailableVersion(version, data[pendingToFlushDataIndex].keyVersions[keyStr]); - if(versionPrevious != 0){ - it = data[pendingToFlushDataIndex].keyValueA.find(versionPrevious); - if (it != data[pendingToFlushDataIndex].keyValueA.end()) + uint64_t versionPrevious = previousAvailableVersion(version, data[pendingToFlushDataIndex].keyVersionsIntray[keyStr]); + if(versionPrevious != UINT64_MAX){ + it_ = data[pendingToFlushDataIndex].keyValueAIntray.find(versionPrevious); + if (it_ != data[pendingToFlushDataIndex].keyValueAIntray.end()) { - for(auto it2 = it->second.begin(); it2 != it->second.end(); ++it2){ + for(auto it2 = it_->second.begin(); it2 != it_->second.end(); ++it2){ if(it2->key[0]==key[0] && it2->key[1]==key[1] && it2->key[2]==key[2] && it2->key[3]==key[3]){ value = it2->value; bResult = true; @@ -208,22 +208,23 @@ bool MultiWrite64::findKeyValue(const uint64_t version,const Goldilocks::Element } #ifdef LOG_DB_MULTI_WRITE_FIND_NODES - zklog.info("MultiWrite64::findkeyValueA() data[pendingToFlushDataIndex].keyValueA found version=" + to_string(version) + " key=" + keyStr + " value=" + value.get_str()); + zklog.info("MultiWrite64::findkeyValueAIntray() data[pendingToFlushDataIndex].keyValueAIntray found version=" + to_string(version) + " key=" + keyStr + " value=" + value.get_str()); #endif } } } - unordered_map>::const_iterator it_; - // Search in data[pendingToFlushDataIndex].keyValueAIntray + map>::const_iterator it; + + // Search in data[pendingToFlushDataIndex].keyValueA if (bResult == false) { - uint64_t versionPrevious = previousAvailableVersion(version, data[pendingToFlushDataIndex].keyVersionsIntray[keyStr]); - if(versionPrevious != 0){ - it_ = data[pendingToFlushDataIndex].keyValueAIntray.find(versionPrevious); - if (it_ != data[pendingToFlushDataIndex].keyValueAIntray.end()) + uint64_t versionPrevious = previousAvailableVersion(version, data[pendingToFlushDataIndex].keyVersions[keyStr]); + if(versionPrevious != UINT64_MAX){ + it = data[pendingToFlushDataIndex].keyValueA.find(versionPrevious); + if (it != data[pendingToFlushDataIndex].keyValueA.end()) { - for(auto it2 = it_->second.begin(); it2 != it_->second.end(); ++it2){ + for(auto it2 = it->second.begin(); it2 != it->second.end(); ++it2){ if(it2->key[0]==key[0] && it2->key[1]==key[1] && it2->key[2]==key[2] && it2->key[3]==key[3]){ value = it2->value; bResult = true; @@ -232,11 +233,12 @@ bool MultiWrite64::findKeyValue(const uint64_t version,const Goldilocks::Element } #ifdef LOG_DB_MULTI_WRITE_FIND_NODES - zklog.info("MultiWrite64::findkeyValueAIntray() data[pendingToFlushDataIndex].keyValueAIntray found version=" + to_string(version) + " key=" + keyStr + " value=" + value.get_str()); + zklog.info("MultiWrite64::findkeyValueA() data[pendingToFlushDataIndex].keyValueA found version=" + to_string(version) + " key=" + keyStr + " value=" + value.get_str()); #endif } } } + // If there is still some data pending to be stored on database if (storingFlushId != storedFlushId) @@ -246,7 +248,7 @@ bool MultiWrite64::findKeyValue(const uint64_t version,const Goldilocks::Element if (bResult == false) { uint64_t versionPrevious = previousAvailableVersion(version, data[storingDataIndex].keyVersions[keyStr]); - if(versionPrevious != 0){ + if(versionPrevious != UINT64_MAX){ it = data[storingDataIndex].keyValueA.find(versionPrevious); if (it != data[storingDataIndex].keyValueA.end()) { diff --git a/src/hashdb64/multi_write_data_64.hpp b/src/hashdb64/multi_write_data_64.hpp index bd86d313d..613e50ef4 100644 --- a/src/hashdb64/multi_write_data_64.hpp +++ b/src/hashdb64/multi_write_data_64.hpp @@ -61,6 +61,8 @@ class MultiWriteData64 (nodesStateRoot.size() == 0) && (keyValueA.size() == 0) && (keyValueAIntray.size() == 0) && + (keyVersions.size() == 0) && + (keyVersionsIntray.size() == 0) && (version.size() == 0) && (versionIntray.size() == 0); } From d299cccc2506d07f5b5c014c6709edf716e72aa3 Mon Sep 17 00:00:00 2001 From: rickb80 Date: Thu, 14 Sep 2023 11:11:49 +0200 Subject: [PATCH 12/12] Fix missing argument in ReadTree --- src/hashdb64/database_64.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hashdb64/database_64.cpp b/src/hashdb64/database_64.cpp index a3e310a37..b56b3dc12 100644 --- a/src/hashdb64/database_64.cpp +++ b/src/hashdb64/database_64.cpp @@ -378,7 +378,7 @@ zkresult Database64::readKV(const Goldilocks::Element (&root)[4], const Goldiloc keyValues[0].key[1] = key[1]; keyValues[0].key[2] = key[2]; keyValues[0].key[3] = key[3]; - rtree = tree64.ReadTree(*this, root, keyValues); + rtree = tree64.ReadTree(*this, root, keyValues, NULL); value = keyValues[0].value; rout = rtree; }