bmckwm/lmdbal
1
0
Fork 0
forked from blue/lmdbal
Code Pull Requests Activity

some more transaction thought on cache

Browse Source
This commit is contained in:
Blue 2023-04-03 21:48:13 +03:00
parent f0779ae2aa
commit 8be63d7551
Signed by untrusted user: blue
GPG Key ID: 9B203B252A63EE38
6 changed files with 241 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/base.cpp
View File

@ -120,13 +120,15 @@ void LMDBAL::Base::drop() {
if (!opened)
throw Closed("drop", name);
TransactionID txn = beginPrivateTransaction(emptyName);
TransactionID txn = beginTransaction();
for (const std::pair<const std::string, iStorage*>& pair : storages) {
int rc = pair.second->drop(txn);
if (rc)
if (rc != MDB_SUCCESS) {
abortTransaction(txn);
throw Unknown(name, mdb_strerror(rc), pair.first);
}
commitPrivateTransaction(txn, emptyName);
}
commitTransaction(txn);
}
LMDBAL::TransactionID LMDBAL::Base::beginReadOnlyTransaction() const {
@ -138,7 +140,7 @@ LMDBAL::TransactionID LMDBAL::Base::beginTransaction() const {
void LMDBAL::Base::abortTransaction(LMDBAL::TransactionID id) const {
return abortTransaction(id, emptyName);}
void LMDBAL::Base::commitTransaction(LMDBAL::TransactionID id) const {
void LMDBAL::Base::commitTransaction(LMDBAL::TransactionID id) {
return commitTransaction(id, emptyName);}
LMDBAL::TransactionID LMDBAL::Base::beginReadOnlyTransaction(const std::string& storageName) const {
@ -180,7 +182,7 @@ void LMDBAL::Base::abortTransaction(LMDBAL::TransactionID id, const std::string&
transactions->erase(itr);
}
void LMDBAL::Base::commitTransaction(LMDBAL::TransactionID id, const std::string& storageName) const {
void LMDBAL::Base::commitTransaction(LMDBAL::TransactionID id, const std::string& storageName) {
if (!opened)
throw Closed("abortTransaction", name, storageName);
@ -198,7 +200,7 @@ void LMDBAL::Base::commitTransaction(LMDBAL::TransactionID id, const std::string
LMDBAL::TransactionID LMDBAL::Base::beginPrivateReadOnlyTransaction(const std::string& storageName) const {
MDB_txn* txn;
int rc = mdb_txn_begin(environment, NULL, MDB_RDONLY, &txn);
if (rc) {
if (rc != MDB_SUCCESS) {
mdb_txn_abort(txn);
throw Unknown(name, mdb_strerror(rc), storageName);
}
@ -208,7 +210,7 @@ LMDBAL::TransactionID LMDBAL::Base::beginPrivateReadOnlyTransaction(const std::s
LMDBAL::TransactionID LMDBAL::Base::beginPrivateTransaction(const std::string& storageName) const {
MDB_txn* txn;
int rc = mdb_txn_begin(environment, NULL, 0, &txn);
if (rc) {
if (rc != MDB_SUCCESS) {
mdb_txn_abort(txn);
throw Unknown(name, mdb_strerror(rc), storageName);
}
@ -220,7 +222,7 @@ void LMDBAL::Base::abortPrivateTransaction(LMDBAL::TransactionID id, const std::
mdb_txn_abort(id);
}
void LMDBAL::Base::commitPrivateTransaction(LMDBAL::TransactionID id, const std::string& storageName) const {
void LMDBAL::Base::commitPrivateTransaction(LMDBAL::TransactionID id, const std::string& storageName) {
int rc = mdb_txn_commit(id);
if (rc != MDB_SUCCESS)
throw Unknown(name, mdb_strerror(rc), storageName);

6
src/base.h
View File

@ -65,7 +65,7 @@ public:
TransactionID beginReadOnlyTransaction() const;
TransactionID beginTransaction() const;
void commitTransaction(TransactionID id) const;
void commitTransaction(TransactionID id);
void abortTransaction(TransactionID id) const;
template <class K, class V>
@ -86,12 +86,12 @@ private:
TransactionID beginReadOnlyTransaction(const std::string& storageName) const;
TransactionID beginTransaction(const std::string& storageName) const;
void commitTransaction(TransactionID id, const std::string& storageName) const;
void commitTransaction(TransactionID id, const std::string& storageName);
void abortTransaction(TransactionID id, const std::string& storageName) const;
TransactionID beginPrivateReadOnlyTransaction(const std::string& storageName) const;
TransactionID beginPrivateTransaction(const std::string& storageName) const;
void commitPrivateTransaction(TransactionID id, const std::string& storageName) const;
void commitPrivateTransaction(TransactionID id, const std::string& storageName);
void abortPrivateTransaction(TransactionID id, const std::string& storageName) const;
private:

41
src/cache.h
View File

@ -21,6 +21,7 @@
#include <map>
#include <set>
#include <tuple>
#include "storage.h"
@ -34,33 +35,63 @@ class Cache : public Storage<K, V> {
size, // - know just an amount of records
full // - shure that our cache is equal to the database on disk
};
enum class Operation {
add,
remove,
change,
force,
drop,
replace,
addMany
};
typedef std::pair<Operation, void*> Entry;
typedef std::list<Entry> Queue;
typedef std::map<TransactionID, Queue> TransactionCache;
protected:
Cache(const std::string& name, Base* parent);
~Cache() override;
virtual void transactionStarted(TransactionID txn, bool readOnly) const override;
virtual void transactionCommited(TransactionID txn) override;
virtual void transactionAborted(TransactionID txn) const override;
private:
void handleMode() const;
void handleTransactionEntry(const Entry& entry);
void destroyTransactionEntry(const Entry& entry) const;
void handleAddRecord(const K& key, const V& value);
void handleRemoveRecord(const K& key);
void handleChangeRecord(const K& key, const V& value);
void handleForceRecord(const K& key, const V& value, bool added);
void handleReplaceAll(std::map<K, V>* data);
void handleAddRecords(const std::map<K, V>& data, bool overwrite, SizeType newSize);
void handleDrop();
public:
using Storage<K, V>::drop;
virtual int drop(TransactionID transaction) override;
virtual void addRecord(const K& key, const V& value) override;
virtual bool forceRecord(const K& key, const V& value) override;
virtual void changeRecord(const K& key, const V& value) override;
virtual void removeRecord(const K& key) override;
virtual bool checkRecord(const K& key) const override;
virtual V getRecord(const K& key) const override;
virtual uint32_t count() const override;
virtual SizeType count() const override;
using Storage<K, V>::drop;
virtual int drop(MDB_txn * transaction) override;
virtual std::map<K, V> readAll() const override;
virtual void replaceAll(const std::map<K, V>& data) override;
virtual uint32_t addRecords(const std::map<K, V>& data, bool overwrite = false) override;
virtual SizeType addRecords(const std::map<K, V>& data, bool overwrite = false) override;
protected:
Mode* mode;
std::map<K, V>* cache;
std::set<K>* abscent;
uint32_t* sizeDifference;
SizeType* sizeDifference;
TransactionCache* transactionCache;
};
}

180
src/cache.hpp
View File

@ -28,7 +28,8 @@ LMDBAL::Cache<K, V>::Cache(const std::string& p_name, Base* parent):
mode(new Mode),
cache(new std::map<K, V>()),
abscent(new std::set<K>()),
sizeDifference(new uint32_t)
sizeDifference(new uint32_t),
transactionCache(new TransactionCache)
{
*mode = Mode::nothing;
*sizeDifference = 0;
@ -36,6 +37,7 @@ LMDBAL::Cache<K, V>::Cache(const std::string& p_name, Base* parent):
template<class K, class V>
LMDBAL::Cache<K, V>::~Cache() {
delete transactionCache;
delete sizeDifference;
delete mode;
delete cache;
@ -50,6 +52,11 @@ void LMDBAL::Cache<K, V>::addRecord(const K& key, const V& value) {
iStorage::throwDuplicate(iStorage::toString(key));
Storage<K, V>::addRecord(key, value);
handleAddRecord(key, value);
}
template<class K, class V>
void LMDBAL::Cache<K, V>::handleAddRecord(const K& key, const V& value) {
cache->insert(std::make_pair(key, value));
if (*mode != Mode::full)
@ -61,6 +68,13 @@ bool LMDBAL::Cache<K, V>::forceRecord(const K& key, const V& value) {
iStorage::ensureOpened(iStorage::forceRecordMethodName);
bool added = Storage<K, V>::forceRecord(key, value);
handleForceRecord(key, value, added);
return added;
}
template<class K, class V>
void LMDBAL::Cache<K, V>::handleForceRecord(const K& key, const V& value, bool added) {
if (*mode == Mode::full) {
(*cache)[key] = value;
} else {
@ -73,8 +87,6 @@ bool LMDBAL::Cache<K, V>::forceRecord(const K& key, const V& value) {
else if (!added) //this way database had value but cache didn't, so, need to decrease sizeDifference
handleMode();
}
return added;
}
template<class K, class V>
@ -106,6 +118,19 @@ void LMDBAL::Cache<K, V>::changeRecord(const K& key, const V& value) {
}
}
template<class K, class V>
void LMDBAL::Cache<K, V>::handleChangeRecord(const K& key, const V& value) {
if (*mode == Mode::full) {
cache->at(key) = value;
} else {
typename std::pair<typename std::map<K, V>::iterator, bool> res = cache->insert(std::make_pair(key, value));
if (!res.second)
res.first->second = value;
else
handleMode();
}
}
template<class K, class V>
V LMDBAL::Cache<K, V>::getRecord(const K& key) const {
iStorage::ensureOpened(iStorage::getRecordMethodName);
@ -181,9 +206,27 @@ void LMDBAL::Cache<K, V>::replaceAll(const std::map<K, V>& data) {
}
template<class K, class V>
uint32_t LMDBAL::Cache<K, V>::addRecords(const std::map<K, V>& data, bool overwrite) {
uint32_t newSize = Storage<K, V>::addRecords(data, overwrite);
void LMDBAL::Cache<K, V>::handleReplaceAll(std::map<K, V>* data) {
delete cache;
cache = data;
if (*mode != Mode::full) {
*mode = Mode::full;
abscent->clear();
*sizeDifference = 0;
}
}
template<class K, class V>
LMDBAL::SizeType LMDBAL::Cache<K, V>::addRecords(const std::map<K, V>& data, bool overwrite) {
SizeType newSize = Storage<K, V>::addRecords(data, overwrite);
handleAddRecords(data, overwrite, newSize);
return newSize;
}
template<class K, class V>
void LMDBAL::Cache<K, V>::handleAddRecords(const std::map<K, V>& data, bool overwrite, SizeType newSize) {
Mode& m = *mode;
if (m == Mode::nothing)
m = Mode::size;
@ -207,19 +250,27 @@ uint32_t LMDBAL::Cache<K, V>::addRecords(const std::map<K, V>& data, bool overwr
abscent->clear();
}
}
return newSize;
}
template<class K, class V>
void LMDBAL::Cache<K, V>::removeRecord(const K& key) {
iStorage::ensureOpened(iStorage::removeRecordMethodName);
typename std::pair<typename std::set<K>::const_iterator, bool> pair = abscent->insert(key);
if (!pair.second)
bool noKey = false;
if (*mode != Mode::full)
noKey = cache->count(key) == 0;
else
noKey = abscent->count(key) > 0;
if (noKey)
iStorage::throwNotFound(iStorage::toString(key));
Storage<K, V>::removeRecord(key);
handleRemoveRecord(key);
}
template<class K, class V>
void LMDBAL::Cache<K, V>::handleRemoveRecord(const K& key) {
if (cache->erase(key) == 0) //if it was not cached and we are now in size mode then the sizeDifference would decrease
handleMode();
@ -263,13 +314,120 @@ void LMDBAL::Cache<K, V>::handleMode() const {
}
template<class K, class V>
int LMDBAL::Cache<K, V>::drop(MDB_txn * transaction) {
int LMDBAL::Cache<K, V>::drop(TransactionID transaction) {
int res = Storage<K, V>::drop(transaction);
transactionCache->at(transaction).emplace_back(Operation::drop, nullptr);
return res;
}
template<class K, class V>
void LMDBAL::Cache<K, V>::handleDrop() {
cache->clear();
abscent->clear();
*mode = Mode::full;
*sizeDifference = 0;
return res;
}
template<class K, class V>
void LMDBAL::Cache<K, V>::transactionStarted(TransactionID txn, bool readOnly) const {
if (!readOnly)
transactionCache->emplace(txn, Queue());
}
template<class K, class V>
void LMDBAL::Cache<K, V>::transactionCommited(TransactionID txn) {
typename TransactionCache::iterator itr = transactionCache->find(txn);
if (itr != transactionCache->end()) {
Queue& queue = itr->second;
for (const Entry& entry : queue)
handleTransactionEntry(entry);
transactionCache->erase(itr);
}
}
template<class K, class V>
void LMDBAL::Cache<K, V>::transactionAborted(TransactionID txn) const {
typename TransactionCache::iterator itr = transactionCache->find(txn);
if (itr != transactionCache->end()) {
Queue& queue = itr->second;
for (const Entry& entry : queue)
destroyTransactionEntry(entry);
transactionCache->erase(itr);
}
}
template<class K, class V>
void LMDBAL::Cache<K, V>::handleTransactionEntry(const Entry& entry) {
switch (entry.first) {
case Operation::add: {
std::pair<K, V>* pair = static_cast<std::pair<K, V>*>(entry.second);
handleAddRecord(pair->first, pair->second);
delete pair;
}
break;
case Operation::remove: {
K* key = static_cast<K*>(entry.second);
handleRemoveRecord(*key);
delete key;
}
break;
case Operation::change: {
std::pair<K, V>* pair = static_cast<std::pair<K, V>*>(entry.second);
handleChangeRecord(pair->first, pair->second);
delete pair;
}
case Operation::force: {
std::tuple<bool, K, V>* tuple = static_cast<std::tuple<bool, K, V>*>(entry.second);
const std::tuple<bool, K, V>& t = *tuple;
handleForceRecord(std::get<1>(t), std::get<2>(t), std::get<0>(t));
delete tuple;
}
break;
case Operation::drop:
handleDrop();
break;
case Operation::replace:
handleReplaceAll(static_cast<std::map<K, V>*>(entry.second)); //I take ownership, no need to delete
break;
case Operation::addMany: {
std::tuple<bool, SizeType, std::map<K, V>>* tuple = static_cast<std::tuple<bool, SizeType, std::map<K, V>>*>(entry.second);
const std::tuple<bool, SizeType, std::map<K, V>>& t = * tuple;
handleAddRecords(std::get<2>(t), std::get<0>(t), std::get<1>(t));
delete tuple;
}
break;
}
}
template<class K, class V>
void LMDBAL::Cache<K, V>::destroyTransactionEntry(const Entry& entry) const {
switch (entry.first) {
case Operation::add:
delete static_cast<std::pair<K, V>*>(entry.second);
break;
case Operation::remove:
delete static_cast<K*>(entry.second);
break;
case Operation::change:
delete static_cast<std::pair<K, V>*>(entry.second);
break;
case Operation::force:
delete static_cast<std::tuple<bool, K, V>*>(entry.second);
break;
case Operation::drop:
break;
case Operation::replace:
delete static_cast<std::map<K, V>*>(entry.second);
break;
case Operation::addMany:
delete static_cast<std::tuple<bool, SizeType, std::map<K, V>>*>(entry.second);
break;
}
}
#endif //LMDBAL_CACHE_HPP

44
src/storage.cpp
View File

@ -31,31 +31,20 @@ LMDBAL::iStorage::~iStorage() {}
void LMDBAL::iStorage::drop() {
ensureOpened(dropMethodName);
MDB_txn *txn;
int rc = mdb_txn_begin(db->environment, NULL, 0, &txn);
if (rc) {
mdb_txn_abort(txn);
throw Unknown(db->name, mdb_strerror(rc), name);
}
rc = drop(txn);
if (rc) {
mdb_txn_abort(txn);
TransactionID txn = db->beginTransaction();
int rc = drop(txn);
if (rc != MDB_SUCCESS) {
abortTransaction(txn);
throw Unknown(db->name, mdb_strerror(rc), name);
}
mdb_txn_commit(txn);
db->commitTransaction(txn);
}
int LMDBAL::iStorage::drop(MDB_txn* transaction) {
int LMDBAL::iStorage::drop(TransactionID transaction) {
return mdb_drop(transaction, dbi, 0);
}
const std::string & LMDBAL::iStorage::dbName() const {
return db->name;}
bool LMDBAL::iStorage::isDBOpened() const {
return db->opened;}
void LMDBAL::iStorage::ensureOpened(const std::string& methodName) const {
if (!db->opened)
throw Closed(methodName, db->name, name);
@ -80,10 +69,8 @@ LMDBAL::SizeType LMDBAL::iStorage::count() const {
LMDBAL::SizeType LMDBAL::iStorage::count(TransactionID txn) const {
MDB_stat stat;
int rc = mdb_stat(txn, dbi, &stat);
if (rc) {
mdb_txn_abort(txn);
if (rc != MDB_SUCCESS)
throw Unknown(db->name, mdb_strerror(rc), name);
}
return stat.ms_entries;
}
@ -117,6 +104,12 @@ void LMDBAL::iStorage::throwUnknown(int rc, LMDBAL::TransactionID txn) const {
throwUnknown(rc);
}
const std::string & LMDBAL::iStorage::dbName() const {
return db->name;}
bool LMDBAL::iStorage::isDBOpened() const {
return db->opened;}
void LMDBAL::iStorage::throwUnknown(int rc) const {
throw Unknown(db->name, mdb_strerror(rc), name);}
@ -135,19 +128,18 @@ LMDBAL::TransactionID LMDBAL::iStorage::beginTransaction() const {
void LMDBAL::iStorage::abortTransaction(LMDBAL::TransactionID id) const {
db->abortPrivateTransaction(id, name);}
void LMDBAL::iStorage::commitTransaction(LMDBAL::TransactionID id) const {
void LMDBAL::iStorage::commitTransaction(LMDBAL::TransactionID id) {
db->commitPrivateTransaction(id, name);}
void LMDBAL::iStorage::transactionStarted(LMDBAL::TransactionID txn, bool readOnly) const {
UNUSED(txn);
UNUSED(readOnly);
}
void LMDBAL::iStorage::transactionCommited(LMDBAL::TransactionID txn) const {
UNUSED(txn);
}
void LMDBAL::iStorage::transactionCommited(LMDBAL::TransactionID txn) {
UNUSED(txn);}
void LMDBAL::iStorage::transactionAborted(LMDBAL::TransactionID txn) const {
UNUSED(txn);
}
UNUSED(txn);}

10
src/storage.h
View File

@ -35,7 +35,6 @@ protected:
virtual ~iStorage();
virtual int createStorage(MDB_txn * transaction) = 0;
virtual int drop(MDB_txn * transaction);
bool isDBOpened() const;
const std::string& dbName() const;
@ -52,14 +51,15 @@ protected:
TransactionID beginReadOnlyTransaction() const;
TransactionID beginTransaction() const;
void commitTransaction(TransactionID id) const;
void commitTransaction(TransactionID id);
void abortTransaction(TransactionID id) const;
void transactionStarted(TransactionID txn, bool readOnly) const;
void transactionCommited(TransactionID txn) const;
void transactionAborted(TransactionID txn) const;
virtual void transactionStarted(TransactionID txn, bool readOnly) const;
virtual void transactionCommited(TransactionID txn);
virtual void transactionAborted(TransactionID txn) const;
public:
virtual void drop();
virtual int drop(TransactionID transaction);
virtual SizeType count() const;
virtual SizeType count(TransactionID txn) const;