blue/lmdbal
1
0
Fork 1
Code Issues 1 Pull Requests Actions Packages Projects Releases 9 Wiki Activity

methods to bulk modify the storage, some transaction code hardening, operators to serialize std containers

Browse Source
This commit is contained in:
Blue 2022-12-18 17:45:12 +03:00
parent 6ae32e38b6
commit a79dae8fd0
Signed by: blue
GPG Key ID: 9B203B252A63EE38
8 changed files with 365 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@ -56,6 +56,7 @@ set(HEADERS
serializer_stdstring.hpp
serializer_qstring.hpp
serializer_qbytearray.hpp
operators.hpp
)
if (BUILD_STATIC)

4
cache.h
View File

@ -45,8 +45,12 @@ public:
virtual bool checkRecord(const K& key) const override;
virtual V getRecord(const K& key) const override;
virtual uint32_t count() const override;
using DataBase::Table<K, V>::drop;
virtual int drop(MDB_txn * transaction) override;
virtual std::map<K, V> readAll() const override;
virtual void replaceAll(const std::map<K, V>& data) override;
virtual uint32_t addRecords(const std::map<K, V>& data, bool overwrite = false) override;
protected:
Mode* mode;

45
cache.hpp
View File

@ -102,11 +102,10 @@ void DataBase::Cache<K, V>::changeRecord(const K& key, const V& value) {
try {
Table<K, V>::changeRecord(key, value);
typename std::map<K, V>::iterator itr = cache->find(key);
if (itr != cache->end()) {
itr->second = value;
typename std::pair<typename std::map<K, V>::iterator, bool> res = cache->insert(std::make_pair(key, value));
if (!res.second) {
res.first->second = value;
} else {
cache->insert(std::make_pair(key, value));
handleMode();
}
} catch (const NotFound& error) {
@ -188,6 +187,44 @@ std::map<K, V> DataBase::Cache<K, V>::readAll() const {
return *cache;
}
template<class K, class V>
void DataBase::Cache<K, V>::replaceAll(const std::map<K, V>& data) {
DataBase::Table<K, V>::replaceAll(data);
*cache = data;
}
template<class K, class V>
uint32_t DataBase::Cache<K, V>::addRecords(const std::map<K, V>& data, bool overwrite) {
uint32_t newSize = DataBase::Table<K, V>::addRecords(data, overwrite);
Mode& m = *mode;
if (m == Mode::nothing) {
m = Mode::size;
}
std::map<K, V>& c = *cache;
std::set<K>& a = *abscent;
for (const std::pair<const K, V>& pair : data) {
std::pair<typename std::map<K, V>::iterator, bool> res = c.insert(pair);
if (!res.second) {
if (overwrite) {
res.first->second = pair.second;
}
} else if (m != Mode::full) {
a.erase(pair.first);
}
}
if (m != Mode::full) {
*sizeDifference = newSize - c.size();
if (*sizeDifference == 0) {
*mode = Mode::full;
abscent->clear();
}
}
return newSize;
}
template<class K, class V>
void DataBase::Cache<K, V>::removeRecord(const K& key) {
if (!DataBase::Table<K, V>::db->opened) {

1
database.h
View File

@ -80,6 +80,7 @@ private:
};
#include "exceptions.h"
#include "operators.hpp"
template <class K, class V>
DataBase::Table<K, V>* DataBase::addTable(const std::string& p_name) {

209
operators.hpp Normal file
View File

@ -0,0 +1,209 @@
// Squawk messenger.
// Copyright (C) 2019 Yury Gubich <blue@macaw.me>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
#ifndef CORE_OPERATORS_HPP
#define CORE_OPERATORS_HPP
#include <map>
#include <set>
#include <vector>
#include <list>
#include <deque>
#include <QDataStream>
template <class K, class V>
QDataStream& operator << (QDataStream &out, const std::map<K, V>& container) {
uint32_t size = container.size();
out << size;
for (const std::pair<const K, V>& pair : container) {
out << pair;
}
return out;
}
template <class K, class V>
QDataStream& operator >> (QDataStream &in, std::map<K, V>& container) {
uint32_t size;
in >> size;
for (uint32_t i = 0; i < size; ++i) {
std::pair<K, V> pair;
in >> pair;
container.insert(pair);
}
return in;
}
template <class K, class V>
QDataStream& operator << (QDataStream &out, const std::multimap<K, V>& container) {
uint32_t size = container.size();
out << size;
for (const std::pair<const K, V>& pair : container) {
out << pair;
}
return out;
}
template <class K, class V>
QDataStream& operator >> (QDataStream &in, std::multimap<K, V>& container) {
uint32_t size;
in >> size;
for (uint32_t i = 0; i < size; ++i) {
std::pair<K, V> pair;
in >> pair;
container.insert(pair);
}
return in;
}
template <class K, class V>
QDataStream& operator << (QDataStream &out, const std::pair<K, V>& pair) {
out << pair.first;
out << pair.second;
return out;
}
template <class K, class V>
QDataStream& operator >> (QDataStream &in, std::pair<K, V>& container) {
in >> container.first;
in >> container.second;
return in;
}
template <class K>
QDataStream& operator << (QDataStream &out, const std::set<K>& container) {
uint32_t size = container.size();
out << size;
for (const K& value : container) {
out << value;
}
return out;
}
template <class K>
QDataStream& operator >> (QDataStream &in, std::set<K>& container) {
uint32_t size;
in >> size;
for (uint32_t i = 0; i < size; ++i) {
K value;
in >> value;
container.insert(value);
}
return in;
}
template <class K>
QDataStream& operator << (QDataStream &out, const std::multiset<K>& container) {
uint32_t size = container.size();
out << size;
for (const K& value : container) {
out << value;
}
return out;
}
template <class K>
QDataStream& operator >> (QDataStream &in, std::multiset<K>& container) {
uint32_t size;
in >> size;
for (uint32_t i = 0; i < size; ++i) {
K value;
in >> value;
container.insert(value);
}
return in;
}
template <class K>
QDataStream& operator << (QDataStream &out, const std::vector<K>& container) {
uint32_t size = container.size();
out << size;
for (const K& value : container) {
out << value;
}
return out;
}
template <class K>
QDataStream& operator >> (QDataStream &in, std::vector<K>& container) {
uint32_t size;
in >> size;
container.resize(size);
for (uint32_t i = 0; i < size; ++i) {
in >> container[i];
}
return in;
}
template <class K>
QDataStream& operator << (QDataStream &out, const std::deque<K>& container) {
uint32_t size = container.size();
out << size;
for (const K& value : container) {
out << value;
}
return out;
}
template <class K>
QDataStream& operator >> (QDataStream &in, std::deque<K>& container) {
uint32_t size;
in >> size;
container.resize(size);
for (uint32_t i = 0; i < size; ++i) {
in >> container[i];
}
return in;
}
template <class K>
QDataStream& operator << (QDataStream &out, const std::list<K>& container) {
uint32_t size = container.size();
out << size;
for (const K& value : container) {
out << value;
}
return out;
}
template <class K>
QDataStream& operator >> (QDataStream &in, std::list<K>& container) {
uint32_t size;
in >> size;
for (uint32_t i = 0; i < size; ++i) {
typename std::list<K>::iterator itr = container.emplace(container.end());
in >> *itr;
}
return in;
}
#endif //CORE_OPERATORS_HPP

3
table.cpp
View File

@ -35,10 +35,12 @@ void DataBase::_Table::drop()
MDB_txn *txn;
int rc = mdb_txn_begin(db->environment, NULL, 0, &txn);
if (rc) {
mdb_txn_abort(txn);
throw Unknown(db->name, mdb_strerror(rc), name);
}
rc = drop(txn);
if (rc) {
mdb_txn_abort(txn);
throw Unknown(db->name, mdb_strerror(rc), name);
}
@ -61,6 +63,7 @@ uint32_t DataBase::_Table::count() const
int rc = mdb_txn_begin(db->environment, NULL, 0, &txn);
rc = mdb_stat(txn, dbi, &stat);
if (rc) {
mdb_txn_abort(txn);
throw Unknown(db->name, mdb_strerror(rc), name);
}
uint32_t amount = stat.ms_entries;

3
table.h
View File

@ -54,6 +54,7 @@ protected:
~Table() override;
public:
using DataBase::_Table::drop;
virtual void addRecord(const K& key, const V& value);
virtual bool forceRecord(const K& key, const V& value); //returns true if there was addition, false if change
virtual void changeRecord(const K& key, const V& value);
@ -61,6 +62,8 @@ public:
virtual bool checkRecord(const K& key) const; //checks if there is a record with given key
virtual V getRecord(const K& key) const;
virtual std::map<K, V> readAll() const;
virtual void replaceAll(const std::map<K, V>& data);
virtual uint32_t addRecords(const std::map<K, V>& data, bool overwrite = false);
protected:
Serializer<K>* keySerializer;

113
table.hpp
View File

@ -43,8 +43,12 @@ void DataBase::Table<K, V>::addRecord(const K& key, const V& value) {
MDB_val lmdbKey = keySerializer->setData(key);
MDB_val lmdbData = valueSerializer->setData(value);
MDB_txn *txn;
mdb_txn_begin(db->environment, NULL, 0, &txn);
int rc;
int rc = mdb_txn_begin(db->environment, NULL, 0, &txn);
if (rc != MDB_SUCCESS) {
mdb_txn_abort(txn);
throw Unknown(db->name, mdb_strerror(rc), name);
}
rc = mdb_put(txn, dbi, &lmdbKey, &lmdbData, MDB_NOOVERWRITE);
if (rc != 0) {
mdb_txn_abort(txn);
@ -68,8 +72,12 @@ bool DataBase::Table<K, V>::forceRecord(const K& key, const V& value) {
MDB_val lmdbKey = keySerializer->setData(key);
MDB_val lmdbData;
MDB_txn *txn;
mdb_txn_begin(db->environment, NULL, 0, &txn);
int rc;
int rc = mdb_txn_begin(db->environment, NULL, 0, &txn);
if (rc != MDB_SUCCESS) {
mdb_txn_abort(txn);
throw Unknown(db->name, mdb_strerror(rc), name);
}
rc = mdb_get(txn, dbi, &lmdbKey, &lmdbData);
if (rc == 0) {
added = false;
@ -100,8 +108,13 @@ void DataBase::Table<K, V>::changeRecord(const K& key, const V& value) {
MDB_val lmdbKey = keySerializer->setData(key);
MDB_val lmdbData = valueSerializer->setData(value);
MDB_txn *txn;
mdb_txn_begin(db->environment, NULL, 0, &txn);
int rc;
int rc = mdb_txn_begin(db->environment, NULL, 0, &txn);
if (rc != MDB_SUCCESS) {
mdb_txn_abort(txn);
throw Unknown(db->name, mdb_strerror(rc), name);
}
rc = mdb_put(txn, dbi, &lmdbKey, &lmdbData, 0);
if (rc != 0) {
mdb_txn_abort(txn);
@ -123,8 +136,12 @@ V DataBase::Table<K, V>::getRecord(const K& key) const {
MDB_val lmdbData;
MDB_txn *txn;
int rc;
mdb_txn_begin(db->environment, NULL, MDB_RDONLY, &txn);
int rc = mdb_txn_begin(db->environment, NULL, MDB_RDONLY, &txn);
if (rc != MDB_SUCCESS) {
mdb_txn_abort(txn);
throw Unknown(db->name, mdb_strerror(rc), name);
}
rc = mdb_get(txn, dbi, &lmdbKey, &lmdbData);
if (rc) {
mdb_txn_abort(txn);
@ -151,8 +168,12 @@ bool DataBase::Table<K, V>::checkRecord(const K& key) const {
MDB_val lmdbData;
MDB_txn *txn;
int rc;
mdb_txn_begin(db->environment, NULL, MDB_RDONLY, &txn);
int rc = mdb_txn_begin(db->environment, NULL, MDB_RDONLY, &txn);
if (rc != MDB_SUCCESS) {
mdb_txn_abort(txn);
throw Unknown(db->name, mdb_strerror(rc), name);
}
rc = mdb_get(txn, dbi, &lmdbKey, &lmdbData);
mdb_txn_abort(txn);
@ -179,10 +200,12 @@ std::map<K, V> DataBase::Table<K, V>::readAll() const {
MDB_val lmdbKey, lmdbData;
int rc = mdb_txn_begin(db->environment, NULL, MDB_RDONLY, &txn);
if (rc != MDB_SUCCESS) {
mdb_txn_abort(txn);
throw Unknown(db->name, mdb_strerror(rc), name);
}
rc = mdb_cursor_open(txn, dbi, &cursor);
if (rc != MDB_SUCCESS) {
mdb_txn_abort(txn);
throw Unknown(db->name, mdb_strerror(rc), name);
}
rc = mdb_cursor_get(cursor, &lmdbKey, &lmdbData, MDB_FIRST);
@ -193,6 +216,7 @@ std::map<K, V> DataBase::Table<K, V>::readAll() const {
rc = mdb_cursor_get(cursor, &lmdbKey, &lmdbData, MDB_NEXT);
}
mdb_txn_abort(txn);
if (rc != MDB_NOTFOUND) {
throw Unknown(db->name, mdb_strerror(rc), name);
}
@ -200,6 +224,75 @@ std::map<K, V> DataBase::Table<K, V>::readAll() const {
return result;
}
template<class K, class V>
void DataBase::Table<K, V>::replaceAll(const std::map<K, V>& data) {
if (!db->opened) {
throw Closed("replaceAll", db->name, name);
}
MDB_txn *txn;
int rc = mdb_txn_begin(db->environment, NULL, 0, &txn);
if (rc) {
mdb_txn_abort(txn);
throw Unknown(db->name, mdb_strerror(rc), name);
}
rc = drop(txn);
if (rc) {
mdb_txn_abort(txn);
throw Unknown(db->name, mdb_strerror(rc), name);
}
MDB_val lmdbKey, lmdbData;
for (const std::pair<const K, V>& pair : data) {
lmdbKey = keySerializer->setData(pair.first);
lmdbData = valueSerializer->setData(pair.second);
rc = mdb_put(txn, dbi, &lmdbKey, &lmdbData, MDB_NOOVERWRITE);
if (rc != 0) {
mdb_txn_abort(txn);
throw Unknown(db->name, mdb_strerror(rc), name);
}
}
mdb_txn_commit(txn);
}
template<class K, class V>
uint32_t DataBase::Table<K, V>::addRecords(const std::map<K, V>& data, bool overwrite) {
if (!db->opened) {
throw Closed("addRecords", db->name, name);
}
MDB_txn *txn;
int rc = mdb_txn_begin(db->environment, NULL, 0, &txn);
if (rc) {
mdb_txn_abort(txn);
throw Unknown(db->name, mdb_strerror(rc), name);
}
MDB_val lmdbKey, lmdbData;
for (const std::pair<const K, V>& pair : data) {
lmdbKey = keySerializer->setData(pair.first);
lmdbData = valueSerializer->setData(pair.second);
rc = mdb_put(txn, dbi, &lmdbKey, &lmdbData, overwrite ? 0 : MDB_NOOVERWRITE);
if (rc != 0) {
mdb_txn_abort(txn);
throw Unknown(db->name, mdb_strerror(rc), name);
}
}
MDB_stat stat;
rc = mdb_stat(txn, dbi, &stat);
if (rc) {
mdb_txn_abort(txn);
throw Unknown(db->name, mdb_strerror(rc), name);
}
uint32_t amount = stat.ms_entries;
mdb_txn_commit(txn);
return amount;
}
template<class K, class V>
void DataBase::Table<K, V>::removeRecord(const K& key) {
if (!db->opened) {