From 181a645efc7dac97c096dc05a9deb946c2d2ce44 Mon Sep 17 00:00:00 2001 From: blue Date: Thu, 6 Apr 2023 02:01:24 +0300 Subject: [PATCH] some more transaction methods, method for valueReference in cache, some errors fix --- src/cache.h | 5 + src/cache.hpp | 255 ++++++++++++++++++++++++++++---------- src/storage.hpp | 3 + test/cachetransaction.cpp | 68 +++++----- 4 files changed, 231 insertions(+), 100 deletions(-) diff --git a/src/cache.h b/src/cache.h index 39e7902..2810f85 100644 --- a/src/cache.h +++ b/src/cache.h @@ -84,14 +84,19 @@ public: virtual void removeRecord(const K& key, TransactionID txn) override; virtual bool checkRecord(const K& key) const override; virtual bool checkRecord(const K& key, TransactionID txn) const override; + virtual void getRecord(const K& key, V& out) const override; + virtual void getRecord(const K& key, V& out, TransactionID txn) const override; virtual V getRecord(const K& key) const override; virtual V getRecord(const K& key, TransactionID txn) const override; virtual SizeType count() const override; virtual SizeType count(TransactionID txn) const override; virtual std::map readAll() const override; + virtual std::map readAll(TransactionID txn) const override; virtual void replaceAll(const std::map& data) override; + virtual void replaceAll(const std::map& data, TransactionID txn) override; virtual SizeType addRecords(const std::map& data, bool overwrite = false) override; + virtual SizeType addRecords(const std::map& data, TransactionID txn, bool overwrite = false) override; protected: Mode* mode; diff --git a/src/cache.hpp b/src/cache.hpp index 5c0d9ca..a28f363 100644 --- a/src/cache.hpp +++ b/src/cache.hpp @@ -112,7 +112,8 @@ void LMDBAL::Cache::handleForceRecord(const K& key, const V& value, bool a if (added) abscent->erase(key); - std::pair::iterator, bool> result = cache->insert(std::make_pair(key, value)); + std::pair::iterator, bool> result = + cache->insert(std::make_pair(key, value)); if (!result.second) result.first->second = value; else if (!added) //this way database had value but cache didn't, so, need to decrease sizeDifference @@ -137,7 +138,8 @@ void LMDBAL::Cache::changeRecord(const K& key, const V& value) { try { Storage::changeRecord(key, value); - typename std::pair::iterator, bool> res = cache->insert(std::make_pair(key, value)); + typename std::pair::iterator, bool> res = + cache->insert(std::make_pair(key, value)); if (!res.second) res.first->second = value; else @@ -183,7 +185,8 @@ void LMDBAL::Cache::handleChangeRecord(const K& key, const V& value) { if (*mode == Mode::full) { cache->at(key) = value; } else { - typename std::pair::iterator, bool> res = cache->insert(std::make_pair(key, value)); + typename std::pair::iterator, bool> res = + cache->insert(std::make_pair(key, value)); if (!res.second) res.first->second = value; else @@ -195,18 +198,29 @@ template V LMDBAL::Cache::getRecord(const K& key) const { iStorage::ensureOpened(iStorage::getRecordMethodName); + V value; + Cache::getRecord(key, value); + return value; +} + +template +void LMDBAL::Cache::getRecord(const K& key, V& out) const { + iStorage::ensureOpened(iStorage::getRecordMethodName); + typename std::map::const_iterator itr = cache->find(key); - if (itr != cache->end()) - return itr->second; + if (itr != cache->end()) { + out = itr->second; + return; + } if (*mode == Mode::full || abscent->count(key) != 0) iStorage::throwNotFound(iStorage::toString(key)); try { - V value = Storage::getRecord(key); - cache->insert(std::make_pair(key, value)); + Storage::getRecord(key, out); + cache->insert(std::make_pair(key, out)); handleMode(); - return value; + return; } catch (const NotFound& error) { if (*mode != Mode::full) abscent->insert(key); @@ -215,68 +229,79 @@ V LMDBAL::Cache::getRecord(const K& key) const { } } + template V LMDBAL::Cache::getRecord(const K& key, TransactionID txn) const { iStorage::ensureOpened(iStorage::getRecordMethodName); + V value; + Cache::getRecord(key, value, txn); + return value; +} + +template +void LMDBAL::Cache::getRecord(const K& key, V& out, TransactionID txn) const { + iStorage::ensureOpened(iStorage::getRecordMethodName); + //if there are any changes made within this transaction //I will be able to see them among pending changes //so, I'm going to go through them in reverse order //and check every key. If it has anything to do this requested key //there is a way to tell... - std::optional candidate = std::nullopt; + bool currentTransaction = false; typename TransactionCache::const_iterator tc = transactionCache->find(txn); if (tc != transactionCache->end()) { + currentTransaction = true; const Queue& queue = tc->second; for (typename Queue::const_reverse_iterator i = queue.rbegin(), end = queue.rend(); i != end; ++i) { const Entry& entry = *i; switch (entry.first) { case Operation::add: - if (static_cast*>(entry.second)->first == key) - return static_cast*>(entry.second)->second; - break; - case Operation::remove: - if (*static_cast(entry.second) == key) { - if (candidate.has_value()) - return candidate.value(); - else - iStorage::throwNotFound(iStorage::toString(key)); + if (static_cast*>(entry.second)->first == key) { + out = static_cast*>(entry.second)->second; + return; } break; + case Operation::remove: + iStorage::throwNotFound(iStorage::toString(key)); + break; case Operation::change: - if (static_cast*>(entry.second)->first == key) - return static_cast*>(entry.second)->second; + if (static_cast*>(entry.second)->first == key) { + out = static_cast*>(entry.second)->second; + return; + } + break; case Operation::force: - if (std::get<1>(*static_cast*>(entry.second)) == key) - return std::get<2>(*static_cast*>(entry.second)); + if (std::get<1>(*static_cast*>(entry.second)) == key) { + out = std::get<2>(*static_cast*>(entry.second)); + return; + } break; case Operation::drop: - if (candidate.has_value()) - return candidate.value(); - else - iStorage::throwNotFound(iStorage::toString(key)); + iStorage::throwNotFound(iStorage::toString(key)); break; case Operation::replace: { std::map* newMap = static_cast*>(entry.second); typename std::map::const_iterator vitr = newMap->find(key); - if (vitr != newMap->end()) - return vitr->second; + if (vitr != newMap->end()) { + out = vitr->second; + return; + } else { + iStorage::throwNotFound(iStorage::toString(key)); + } } break; case Operation::addMany: { - const std::tuple>& tuple = *static_cast>*>(entry.second); + const std::tuple>& tuple = + *static_cast>*>(entry.second); const std::map& newElements = std::get<2>(tuple); typename std::map::const_iterator vitr = newElements.find(key); if (vitr != newElements.end()) { - if (std::get<0>(tuple)) //if the command was to overwrite - - return vitr->second; //it's clear, current value is the actual - //but if it wasn't, I'm going to remember - if (!candidate.has_value()) //only the last (which is the first, keeping in mind reverse order) - candidate = vitr->second; //occurance and return it in case I meet any NotFound condition + out = vitr->second; + return; } - } break; } @@ -286,35 +311,24 @@ V LMDBAL::Cache::getRecord(const K& key, TransactionID txn) const { //which caused the changes i just need to check it among local cache typename std::map::const_iterator itr = cache->find(key); - if (itr != cache->end()) - return itr->second; - - if (*mode == Mode::full || abscent->count(key) != 0) { - if (candidate.has_value()) - return candidate.value(); - else - iStorage::throwNotFound(iStorage::toString(key)); + if (itr != cache->end()) { + out = itr->second; + return; } - try { - V value = Storage::getRecord(key); - cache->insert(std::make_pair(key, value)); - handleMode(); - return value; - } catch (const NotFound& error) { - if (*mode != Mode::full) - abscent->insert(key); + if (*mode == Mode::full || abscent->count(key) != 0) + iStorage::throwNotFound(iStorage::toString(key)); - if (candidate.has_value()) { - throw Unknown(iStorage::dbName(), -"Something completely wrong have happened: \ -cache has a pending addition transaction \ -(probably as a result of calling addRecords \ -method with overwrite parameter == false, \ -(default is false)), but the database reports \ -that there is no such element in the database under current transaction", - iStorage::name); + try { + Storage::getRecord(key, out, txn); + if (!currentTransaction) { + cache->insert(std::make_pair(key, out)); + handleMode(); } + return; + } catch (const NotFound& error) { + if (! currentTransaction && *mode != Mode::full) + abscent->insert(key); throw error; } @@ -353,8 +367,10 @@ bool LMDBAL::Cache::checkRecord(const K& key, TransactionID txn) const { //so, I'm going to go through them in reverse order //and check every key. If it has anything to do this requested key //there is a way to tell... + bool currentTransaction = false; typename TransactionCache::const_iterator tc = transactionCache->find(txn); if (tc != transactionCache->end()) { + currentTransaction = true; const Queue& queue = tc->second; for (typename Queue::const_reverse_iterator i = queue.rbegin(), end = queue.rend(); i != end; ++i) { const Entry& entry = *i; @@ -382,9 +398,13 @@ bool LMDBAL::Cache::checkRecord(const K& key, TransactionID txn) const { case Operation::replace: if (static_cast*>(entry.second)->count(key) > 0) return true; + else + return false; break; case Operation::addMany: - if (std::get<2>(*static_cast>*>(entry.second)).count(key) > 0) + if (std::get<2>( + *static_cast>*>(entry.second) + ).count(key) > 0) return true; break; } @@ -402,11 +422,13 @@ bool LMDBAL::Cache::checkRecord(const K& key, TransactionID txn) const { try { V value = Storage::getRecord(key, txn); - cache->insert(std::make_pair(key, value)); - handleMode(); + if (!currentTransaction) { + cache->insert(std::make_pair(key, value)); + handleMode(); + } return true; } catch (const NotFound& error) { - if (*mode != Mode::full) + if (!currentTransaction && *mode != Mode::full) abscent->insert(key); return false; @@ -427,6 +449,81 @@ std::map LMDBAL::Cache::readAll() const { return *cache; } +template +std::map LMDBAL::Cache::readAll(TransactionID txn) const { + iStorage::ensureOpened(iStorage::readAllMethodName); + + typename TransactionCache::iterator tc = transactionCache->find(txn); + if (tc != transactionCache->end()) { + Queue& queue = tc->second; + if (*mode != Mode::full) { + std::map result = *cache; + + for (typename Queue::const_iterator i = queue.begin(), end = queue.end(); i != end; ++i) { + const Entry& entry = *i; + switch (entry.first) { + case Operation::add: + result.insert(*static_cast*>(entry.second)); + break; + case Operation::remove: + result.erase(*static_cast(entry.second)); + break; + case Operation::change: { + std::pair* pair = static_cast*>(entry.second); + result.at(pair->first) = pair->second; + } + break; + case Operation::force:{ + const std::tuple& tuple = + *static_cast*>(entry.second); + result[std::get<1>(tuple)] = std::get<2>(tuple); + } + break; + case Operation::drop: + result.clear(); + break; + case Operation::replace: + result = *static_cast*>(entry.second); + break; + case Operation::addMany: { + const std::tuple>& t = + *static_cast>*>(entry.second); + const std::map& added = std::get<2>(t); + bool overwrite = std::get<0>(t); + for (const std::pair& pair : added) { + if (overwrite) + result[pair.first] = pair.second; + else + result.insert(pair); + } + } + break; + } + } + + return result; + } else { + std::map* result = new std::map(); + Storage::readAll(*result, txn); + + //queue.clear(); //since I'm getting a complete state of the database + queue.emplace_back(Operation::replace, result); //I can as well erase all previous cache entries + + return *result; + } + + } else { + if (*mode != Mode::full) { //there is a room for optimization + *mode = Mode::full; //I can read and deserialize only those values + *cache = Storage::readAll(txn); //that are missing in the cache + abscent->clear(); + *sizeDifference = 0; + } + + return *cache; + } +} + template void LMDBAL::Cache::replaceAll(const std::map& data) { Storage::replaceAll(data); @@ -439,6 +536,18 @@ void LMDBAL::Cache::replaceAll(const std::map& data) { } } +template +void LMDBAL::Cache::replaceAll(const std::map& data, TransactionID txn) { + Storage::replaceAll(data, txn); + + typename TransactionCache::iterator tc = transactionCache->find(txn); + if (tc != transactionCache->end()) { + //queue.clear(); + std::map* map = new std::map(data); //since I'm getting a complete state of the database + tc->second.emplace_back(Operation::replace, map); //I can as well erase all previous cache entries + } +} + template void LMDBAL::Cache::handleReplaceAll(std::map* data) { delete cache; @@ -459,6 +568,20 @@ LMDBAL::SizeType LMDBAL::Cache::addRecords(const std::map& data, boo return newSize; } +template +LMDBAL::SizeType LMDBAL::Cache::addRecords(const std::map& data, TransactionID txn, bool overwrite) { + SizeType newSize = Storage::addRecords(data, txn, overwrite); + + typename TransactionCache::iterator tc = transactionCache->find(txn); + if (tc != transactionCache->end()) { + std::tuple>* tuple = + new std::tuple>(overwrite, newSize, data); + tc->second.emplace_back(Operation::addMany, tuple); + } + + return newSize; +} + template void LMDBAL::Cache::handleAddRecords(const std::map& data, bool overwrite, SizeType newSize) { Mode& m = *mode; @@ -579,7 +702,7 @@ uint32_t LMDBAL::Cache::count(TransactionID txn) const { break; case Operation::force: if (std::get<0>(*static_cast*>(entry.second))) - return diff; + ++diff; break; case Operation::drop: return false; @@ -588,7 +711,7 @@ uint32_t LMDBAL::Cache::count(TransactionID txn) const { return static_cast*>(entry.second)->size() + diff; break; case Operation::addMany: - return Storage::count(txn); //it's just close to impossible to tell + return std::get<1>(*static_cast>*>(entry.second)) + diff; break; } } diff --git a/src/storage.hpp b/src/storage.hpp index 68ad50e..8508268 100644 --- a/src/storage.hpp +++ b/src/storage.hpp @@ -334,6 +334,9 @@ uint32_t LMDBAL::Storage::addRecords(const std::map& data, Transacti lmdbData = valueSerializer->setData(pair.second); rc = mdb_put(txn, dbi, &lmdbKey, &lmdbData, overwrite ? 0 : MDB_NOOVERWRITE); + if (rc == MDB_KEYEXIST) + throwDuplicate(toString(pair.first)); + if (rc != MDB_SUCCESS) throwUnknown(rc); } diff --git a/test/cachetransaction.cpp b/test/cachetransaction.cpp index 5ae7ea3..f158f0e 100644 --- a/test/cachetransaction.cpp +++ b/test/cachetransaction.cpp @@ -132,41 +132,41 @@ TEST_F(CacheTransactionsTest, Reading) { } TEST_F(CacheTransactionsTest, ConcurentReading) { - // EXPECT_EQ(db->ready(), true); - // - // LMDBAL::SizeType size = c1->count(); - // LMDBAL::TransactionID txn = db->beginTransaction(); - // EXPECT_EQ(c1->getRecord(5, txn), 13); - // EXPECT_EQ(c1->getRecord(5), 13); - // - // c1->removeRecord(5, txn); - // - // EXPECT_FALSE(c1->checkRecord(5, txn)); - // EXPECT_EQ(c1->getRecord(5), 13); - // - // c1->addRecord(5, 571, txn); - // EXPECT_EQ(c1->getRecord(5, txn), 571); - // EXPECT_EQ(c1->getRecord(5), 13); - // - // c1->forceRecord(5, -472, txn); - // EXPECT_EQ(c1->getRecord(5, txn), -472); - // EXPECT_EQ(c1->getRecord(5), 13); - // - // c1->replaceAll({ - // {1, 75} - // }, txn); - // EXPECT_FALSE(c1->checkRecord(5, txn)); - // EXPECT_EQ(c1->getRecord(5), 13); - // EXPECT_EQ(c1->count(txn), 1); - // EXPECT_EQ(c1->count(), size); - // - // db->commitTransaction(txn); - // - // EXPECT_FALSE(c1->checkRecord(5)); - // EXPECT_EQ(c1->count(), 1); + EXPECT_EQ(db->ready(), true); + + LMDBAL::SizeType size = c1->count(); + LMDBAL::TransactionID txn = db->beginTransaction(); + EXPECT_EQ(c1->getRecord(5, txn), 13); + EXPECT_EQ(c1->getRecord(5), 13); + + c1->removeRecord(5, txn); + + EXPECT_FALSE(c1->checkRecord(5, txn)); + EXPECT_EQ(c1->getRecord(5), 13); + + c1->addRecord(5, 571, txn); + EXPECT_EQ(c1->getRecord(5, txn), 571); + EXPECT_EQ(c1->getRecord(5), 13); + + c1->forceRecord(5, -472, txn); + EXPECT_EQ(c1->getRecord(5, txn), -472); + EXPECT_EQ(c1->getRecord(5), 13); + + c1->replaceAll({ + {1, 75} + }, txn); + EXPECT_FALSE(c1->checkRecord(5, txn)); + EXPECT_EQ(c1->getRecord(5), 13); + EXPECT_EQ(c1->count(txn), 1); + EXPECT_EQ(c1->count(), size); + + db->commitTransaction(txn); + + EXPECT_FALSE(c1->checkRecord(5)); + EXPECT_EQ(c1->count(), 1); } -/* + TEST_F(CacheTransactionsTest, ConcurentModification) { EXPECT_EQ(db->ready(), true); @@ -228,4 +228,4 @@ TEST_F(CacheTransactionsTest, ConcurentModification) { std::cout << "checking final result" << std::endl; EXPECT_EQ(c1->getRecord(5), -46); } -*/ +