started to work on duplicates support

This commit is contained in:
Blue 2023-08-15 15:48:19 -03:00
parent 7b26d57ab6
commit f0727aa73d
Signed by: blue
GPG Key ID: 9B203B252A63EE38
9 changed files with 132 additions and 108 deletions

View File

@ -1,14 +1,22 @@
# Changelog # Changelog
## LMDBAL 0.5.0 (UNRELEASED, 2023)
### New Features
- duplicates support
## LMDBAL 0.4.0 (August 13, 2023) ## LMDBAL 0.4.0 (August 13, 2023)
### New Features
- read only cursors
### Bug fixes ### Bug fixes
- possible cache unsync - possible cache unsync
- doxygen-awesome build bix
### Improvements ### Improvements
- read only cursors
- some more documentation - some more documentation
- more tests - more tests
- doxygen-awesome build bix
## LMDBAL 0.3.1 (April 14, 2023) ## LMDBAL 0.3.1 (April 14, 2023)
### Bug fixes ### Bug fixes
@ -17,6 +25,7 @@
### Improvements ### Improvements
- exception documentation - exception documentation
## LMDBAL 0.3.0 (April 12, 2023) ## LMDBAL 0.3.0 (April 12, 2023)
### New features ### New features
- transaction functions - transaction functions

View File

@ -69,16 +69,16 @@ public:
void abortTransaction(TransactionID id) const; void abortTransaction(TransactionID id) const;
template <class K, class V> template <class K, class V>
LMDBAL::Storage<K, V>* addStorage(const std::string& name); LMDBAL::Storage<K, V>* addStorage(const std::string& storageName, bool duplicates = false);
template <class K, class V> template <class K, class V>
LMDBAL::Cache<K, V>* addCache(const std::string& name); LMDBAL::Cache<K, V>* addCache(const std::string& storageName);
template <class K, class V> template <class K, class V>
LMDBAL::Storage<K, V>* getStorage(const std::string& name); LMDBAL::Storage<K, V>* getStorage(const std::string& storageName);
template <class K, class V> template <class K, class V>
LMDBAL::Cache<K, V>* getCache(const std::string& name); LMDBAL::Cache<K, V>* getCache(const std::string& storageName);
private: private:
typedef std::map<std::string, LMDBAL::iStorage*> Storages; /**<Storage and Cache pointers are saved in the std::map*/ typedef std::map<std::string, LMDBAL::iStorage*> Storages; /**<Storage and Cache pointers are saved in the std::map*/
@ -114,7 +114,8 @@ private:
* Defines that the database is going to have the following storage. * Defines that the database is going to have the following storage.
* The LMDBAL::Base must be closed * The LMDBAL::Base must be closed
* *
* \param[in] _name - storage name * \param[in] storageName - storage name
*
* \returns storage pointer. LMDBAL::Base keeps the ownership of the added storage, you don't need to destoroy it * \returns storage pointer. LMDBAL::Base keeps the ownership of the added storage, you don't need to destoroy it
* *
* \tparam K - key type of the storage * \tparam K - key type of the storage
@ -124,14 +125,14 @@ private:
* \exception LMDBAL::StorageDuplicate thrown if somebody tries to add storage with repeating name * \exception LMDBAL::StorageDuplicate thrown if somebody tries to add storage with repeating name
*/ */
template <class K, class V> template <class K, class V>
LMDBAL::Storage<K, V>* LMDBAL::Base::addStorage(const std::string& _name) { LMDBAL::Storage<K, V>* LMDBAL::Base::addStorage(const std::string& storageName, bool duplicates) {
if (opened) { if (opened) {
throw Opened(name, "add storage " + _name); throw Opened(name, "add storage " + storageName);
} }
Storage<K, V>* storage = new Storage<K, V>(_name, this); Storage<K, V>* storage = new Storage<K, V>(this, storageName, duplicates);
std::pair<Storages::const_iterator, bool> pair = storages.insert(std::make_pair(_name, (iStorage*)storage)); std::pair<Storages::const_iterator, bool> pair = storages.insert(std::make_pair(storageName, (iStorage*)storage));
if (!pair.second) if (!pair.second)
throw StorageDuplicate(name, _name); throw StorageDuplicate(name, storageName);
return storage; return storage;
} }
@ -142,7 +143,7 @@ LMDBAL::Storage<K, V>* LMDBAL::Base::addStorage(const std::string& _name) {
* Defines that the database is going to have the following cache. * Defines that the database is going to have the following cache.
* The LMDBAL::Base must be closed * The LMDBAL::Base must be closed
* *
* \param[in] _name - cache name * \param[in] storageName - cache name
* \returns cache pointer. LMDBAL::Base keeps the ownership of the added cache, you don't need to destoroy it * \returns cache pointer. LMDBAL::Base keeps the ownership of the added cache, you don't need to destoroy it
* *
* \tparam K - key type of the cache * \tparam K - key type of the cache
@ -152,14 +153,14 @@ LMDBAL::Storage<K, V>* LMDBAL::Base::addStorage(const std::string& _name) {
* \exception LMDBAL::StorageDuplicate thrown if somebody tries to add cache with repeating name * \exception LMDBAL::StorageDuplicate thrown if somebody tries to add cache with repeating name
*/ */
template<class K, class V> template<class K, class V>
LMDBAL::Cache<K, V> * LMDBAL::Base::addCache(const std::string& _name) { LMDBAL::Cache<K, V> * LMDBAL::Base::addCache(const std::string& storageName) {
if (opened) { if (opened) {
throw Opened(name, "add cache " + _name); throw Opened(name, "add cache " + storageName);
} }
Cache<K, V>* cache = new Cache<K, V>(_name, this); Cache<K, V>* cache = new Cache<K, V>(this, storageName, false);
std::pair<Storages::const_iterator, bool> pair = storages.insert(std::make_pair(_name, (iStorage*)cache)); std::pair<Storages::const_iterator, bool> pair = storages.insert(std::make_pair(storageName, (iStorage*)cache));
if (!pair.second) if (!pair.second)
throw StorageDuplicate(name, _name); throw StorageDuplicate(name, storageName);
return cache; return cache;
} }
@ -173,7 +174,7 @@ LMDBAL::Cache<K, V> * LMDBAL::Base::addCache(const std::string& _name) {
* this method with template parameters <std::string, std::string> * this method with template parameters <std::string, std::string>
* on the same name of the previously added storage, or calling it on cache - the behaviour is undefined * on the same name of the previously added storage, or calling it on cache - the behaviour is undefined
* *
* \param[in] _name - storage name * \param[in] storageName - storage name
* \returns storage pointer. LMDBAL::Base keeps the ownership of the added storage, you don't need to destoroy it * \returns storage pointer. LMDBAL::Base keeps the ownership of the added storage, you don't need to destoroy it
* *
* \tparam K - key type of the storage * \tparam K - key type of the storage
@ -182,8 +183,8 @@ LMDBAL::Cache<K, V> * LMDBAL::Base::addCache(const std::string& _name) {
* \exception std::out_of_range thrown if storage with the given name was not found * \exception std::out_of_range thrown if storage with the given name was not found
*/ */
template <class K, class V> template <class K, class V>
LMDBAL::Storage<K, V>* LMDBAL::Base::getStorage(const std::string& _name) { LMDBAL::Storage<K, V>* LMDBAL::Base::getStorage(const std::string& storageName) {
return static_cast<Storage<K, V>*>(storages.at(_name)); return static_cast<Storage<K, V>*>(storages.at(storageName));
} }
/** /**
@ -195,7 +196,7 @@ LMDBAL::Storage<K, V>* LMDBAL::Base::getStorage(const std::string& _name) {
* this method with template parameters <std::string, std::string> * this method with template parameters <std::string, std::string>
* on the same name of the previously added cache, or calling it on storage - the behaviour is undefined * on the same name of the previously added cache, or calling it on storage - the behaviour is undefined
* *
* \param[in] _name - cache name * \param[in] storageName - cache name
* \returns cache pointer. LMDBAL::Base keeps the ownership of the added cache, you don't need to destoroy it * \returns cache pointer. LMDBAL::Base keeps the ownership of the added cache, you don't need to destoroy it
* *
* \tparam K - key type of the cache * \tparam K - key type of the cache
@ -204,8 +205,8 @@ LMDBAL::Storage<K, V>* LMDBAL::Base::getStorage(const std::string& _name) {
* \exception std::out_of_range thrown if cache with the given name was not found * \exception std::out_of_range thrown if cache with the given name was not found
*/ */
template <class K, class V> template <class K, class V>
LMDBAL::Cache<K, V>* LMDBAL::Base::getCache(const std::string& _name) { LMDBAL::Cache<K, V>* LMDBAL::Base::getCache(const std::string& storageName) {
return static_cast<Cache<K, V>*>(storages.at(_name)); return static_cast<Cache<K, V>*>(storages.at(storageName));
} }
#endif //LMDBAL_BASE_H #endif //LMDBAL_BASE_H

View File

@ -51,7 +51,7 @@ class Cache : public Storage<K, V> {
typedef std::map<TransactionID, Queue> TransactionCache; typedef std::map<TransactionID, Queue> TransactionCache;
protected: protected:
Cache(const std::string& name, Base* parent); Cache(Base* parent, const std::string& name, bool duplicates = false);
~Cache() override; ~Cache() override;
virtual void transactionStarted(TransactionID txn, bool readOnly) const override; virtual void transactionStarted(TransactionID txn, bool readOnly) const override;

View File

@ -39,12 +39,13 @@
/** /**
* \brief Creates a cache * \brief Creates a cache
* *
* \param[in] _name - name of the new cache * \param[in] parent - LMDBAL::Base pointer for the owning database (borrowed)
* \param[in] parent - parent database pointed (borrowed) * \param[in] name - the name of the storage
* \param[in] duplicates - true if storage supports duplicates, false otherwise (false by default)
*/ */
template<class K, class V> template<class K, class V>
LMDBAL::Cache<K, V>::Cache(const std::string& _name, Base* parent): LMDBAL::Cache<K, V>::Cache(Base* parent, const std::string& name, bool duplicates):
Storage<K, V>(_name, parent), Storage<K, V>(parent, name, duplicates),
mode(Mode::nothing), mode(Mode::nothing),
cache(new std::map<K, V>()), cache(new std::map<K, V>()),
abscent(new std::set<K>()), abscent(new std::set<K>()),

View File

@ -23,11 +23,12 @@
/** /**
* \class LMDBAL::Serializer * \class LMDBAL::Serializer
* \brief A class handling serialization/deserialization
* *
* A class that is constructed in every LMDBAL::Storage * A class that is constructed in every LMDBAL::Storage
* to serialize or deserialize keys and values. * to serialize or deserialize keys and values.
* *
* It serializes to and deserializes from <a class="el" href="http://www.lmdb.tech/doc/group__mdb.html#structMDB__val">MDB_val</a>MDB_val * It serializes to and deserializes from <a class="el" href="http://www.lmdb.tech/doc/group__mdb.html#structMDB__val">MDB_val</a>
* *
* \tparam K type of the keys of the storage * \tparam K type of the keys of the storage
* \tparam V type of the values of the storage * \tparam V type of the values of the storage

View File

@ -30,11 +30,16 @@
/** /**
* \brief Constructs a storage interface * \brief Constructs a storage interface
*
* \param[in] parent - LMDBAL::Base pointer for the owning database (borrowed)
* \param[in] name - the name of the storage
* \param[in] duplicates - true if storage supports duplicates, false otherwise (false by default)
*/ */
LMDBAL::iStorage::iStorage(const std::string& p_name, Base* parent): LMDBAL::iStorage::iStorage(Base* parent, const std::string& name, bool duplicates):
dbi(), dbi(),
db(parent), db(parent),
name(p_name) name(name),
duplicates(duplicates)
{} {}
/** /**

View File

@ -19,10 +19,14 @@
#ifndef LMDBAL_STORAGE_H #ifndef LMDBAL_STORAGE_H
#define LMDBAL_STORAGE_H #define LMDBAL_STORAGE_H
#include <type_traits>
#include "base.h" #include "base.h"
#include "serializer.h" #include "serializer.h"
#include "cursor.h" #include "cursor.h"
class BaseTest;
namespace LMDBAL { namespace LMDBAL {
typedef uint32_t SizeType; typedef uint32_t SizeType;
@ -32,7 +36,7 @@ class iStorage {
public: public:
protected: protected:
iStorage(const std::string& name, Base* parent); iStorage(Base* parent, const std::string& name, bool duplicates = false);
virtual ~iStorage(); virtual ~iStorage();
/** /**
@ -77,9 +81,11 @@ protected:
MDB_dbi dbi; /**<\brief lmdb storage handle*/ MDB_dbi dbi; /**<\brief lmdb storage handle*/
Base* db; /**<\brief parent database pointer (borrowed)*/ Base* db; /**<\brief parent database pointer (borrowed)*/
const std::string name; /**<\brief this storage name*/ const std::string name; /**<\brief this storage name*/
const bool duplicates; /**<\brief true if storage supports duplicates*/
inline static const std::string dropMethodName = "drop"; /**<\brief member function name, just for exceptions*/ inline static const std::string dropMethodName = "drop"; /**<\brief member function name, just for exceptions*/
inline static const std::string countMethodName = "count"; /**<\brief member function name, just for exceptions*/ inline static const std::string countMethodName = "count"; /**<\brief member function name, just for exceptions*/
inline static const std::string flagsMethodName = "flags"; /**<\brief member function name, just for exceptions*/
inline static const std::string addRecordMethodName = "addRecord"; /**<\brief member function name, just for exceptions*/ inline static const std::string addRecordMethodName = "addRecord"; /**<\brief member function name, just for exceptions*/
inline static const std::string forceRecordMethodName = "forceRecord"; /**<\brief member function name, just for exceptions*/ inline static const std::string forceRecordMethodName = "forceRecord"; /**<\brief member function name, just for exceptions*/
@ -92,8 +98,8 @@ protected:
inline static const std::string addRecordsMethodName = "addRecords"; /**<\brief member function name, just for exceptions*/ inline static const std::string addRecordsMethodName = "addRecords"; /**<\brief member function name, just for exceptions*/
protected: protected:
template <class T> template <class K, class V>
int makeStorage(MDB_txn* transaction); int makeStorage(MDB_txn* transaction, bool duplicates);
template <class T> template <class T>
static std::string toString(const T& value); static std::string toString(const T& value);
@ -101,14 +107,16 @@ protected:
template <class K, class V> template <class K, class V>
class Storage : public iStorage { class Storage : public iStorage {
friend class ::BaseTest;
friend class Base; friend class Base;
friend class Cursor<K, V>; friend class Cursor<K, V>;
protected: protected:
Storage(const std::string& name, Base* parent); Storage(Base* parent, const std::string& name, bool duplicates = false);
~Storage() override; ~Storage() override;
virtual void discoveredRecord(const K& key, const V& value) const; virtual void discoveredRecord(const K& key, const V& value) const;
virtual void discoveredRecord(const K& key, const V& value, TransactionID txn) const; virtual void discoveredRecord(const K& key, const V& value, TransactionID txn) const;
uint32_t flags() const;
public: public:
using iStorage::drop; using iStorage::drop;

View File

@ -41,12 +41,13 @@
/** /**
* \brief Creates a storage * \brief Creates a storage
* *
* \param[in] _name - name of the new storage * \param[in] parent - LMDBAL::Base pointer for the owning database (borrowed)
* \param[in] parent - parent database pointed (borrowed) * \param[in] name - the name of the storage
* \param[in] duplicates - true if storage supports duplicates, false otherwise (false by default)
*/ */
template<class K, class V> template<class K, class V>
LMDBAL::Storage<K, V>::Storage(const std::string& _name, Base* parent): LMDBAL::Storage<K, V>::Storage(Base* parent, const std::string& name, bool duplicates):
iStorage(_name, parent), iStorage(parent, name, duplicates),
keySerializer(), keySerializer(),
valueSerializer(), valueSerializer(),
cursors() cursors()
@ -692,7 +693,7 @@ void LMDBAL::Storage<K, V>::removeRecord(const K& key, TransactionID txn) {
*/ */
template<class K, class V> template<class K, class V>
int LMDBAL::Storage<K, V>::open(MDB_txn* transaction) { int LMDBAL::Storage<K, V>::open(MDB_txn* transaction) {
return makeStorage<K>(transaction); return makeStorage<K, V>(transaction, duplicates);
} }
/** /**
@ -719,6 +720,30 @@ LMDBAL::Cursor<K, V>* LMDBAL::Storage<K, V>::createCursor() {
return cursor; return cursor;
} }
/**
* \brief Reads current storage flags it was opened with
*
* This function exists mostly for testing purposes
*
* \returns Third out parameter of <a href="http://www.lmdb.tech/doc/group__internal.html#ga95ba4cb721035478a8705e57b91ae4d4">mdb_dbi_flags</a> function
*
* \exception LMDBAL::Closed thrown if the database was not opened
* \exception LMDBAL::Unknown if the result of <a href="http://www.lmdb.tech/doc/group__internal.html#ga95ba4cb721035478a8705e57b91ae4d4">mdb_dbi_flags</a> was not successfull
*/
template<class K, class V>
uint32_t LMDBAL::Storage<K, V>::flags() const {
ensureOpened(flagsMethodName);
uint32_t result;
TransactionID txn = beginReadOnlyTransaction();
int res = mdb_dbi_flags(txn, dbi, &result);
abortTransaction(txn);
if (res != MDB_SUCCESS)
throwUnknown(res);
return result;
}
/** /**
* \brief Destroys cursor * \brief Destroys cursor
* *
@ -726,7 +751,7 @@ LMDBAL::Cursor<K, V>* LMDBAL::Storage<K, V>::createCursor() {
* *
* \param[in] cursor a pointer to a cursor you want to destroy * \param[in] cursor a pointer to a cursor you want to destroy
* *
* \throws LMDBAL::Unknown thrown if you try to destroy something that this storage didn't create * \exception LMDBAL::Unknown thrown if you try to destroy something that this storage didn't create
*/ */
template<class K, class V> template<class K, class V>
void LMDBAL::Storage<K, V>::destroyCursor(Cursor<K, V>* cursor) { void LMDBAL::Storage<K, V>::destroyCursor(Cursor<K, V>* cursor) {
@ -770,79 +795,28 @@ void LMDBAL::Storage<K, V>::discoveredRecord(const K& key, const V& value, Trans
* \tparam K type of keys in opening storage * \tparam K type of keys in opening storage
* *
* \param[in] transaction - lmdb transaction to call <a class="el" href="http://www.lmdb.tech/doc/group__mdb.html#gac08cad5b096925642ca359a6d6f0562a">mdb_dbi_open</a>, must be a writable transaction! * \param[in] transaction - lmdb transaction to call <a class="el" href="http://www.lmdb.tech/doc/group__mdb.html#gac08cad5b096925642ca359a6d6f0562a">mdb_dbi_open</a>, must be a writable transaction!
* \param[in] duplicates - true if you wish to enable duplicates support for the storage
*
* \returns MDB_SUCCESS if everything went smooth or MDB_<error> -like error code * \returns MDB_SUCCESS if everything went smooth or MDB_<error> -like error code
* *
* This and the following collection of specializations are a way to optimise database using * This is a way to optimise database using MDB_INTEGERKEY flag,
* MDB_INTEGERKEY flag, when the key is actually kind of an integer * when the key is actually kind of an integer
* This infrastructure also allowes us to customize mdb_dbi_open call in the future * This infrastructure also allowes us to customize mdb_dbi_open call in the future
*/ */
template<class K> template<class K, class V>
inline int LMDBAL::iStorage::makeStorage(MDB_txn* transaction) { inline int LMDBAL::iStorage::makeStorage(MDB_txn* transaction, bool duplicates) {
return mdb_dbi_open(transaction, name.c_str(), MDB_CREATE, &dbi); unsigned int flags = MDB_CREATE;
} if constexpr (std::is_integral<K>::value)
flags |= MDB_INTEGERKEY;
/** if (duplicates) {
* \brief Opening database function specialization for uint64_t flags |= MDB_DUPSORT;
*/
template<>
inline int LMDBAL::iStorage::makeStorage<uint64_t>(MDB_txn* transaction) {
return mdb_dbi_open(transaction, name.c_str(), MDB_CREATE | MDB_INTEGERKEY, &dbi);
}
/** if constexpr (std::is_integral<V>::value)
* \brief Opening database function specialization for uint32_t flags |= MDB_INTEGERDUP | MDB_DUPFIXED;
*/ }
template<>
inline int LMDBAL::iStorage::makeStorage<uint32_t>(MDB_txn* transaction) {
return mdb_dbi_open(transaction, name.c_str(), MDB_CREATE | MDB_INTEGERKEY, &dbi);
}
/** return mdb_dbi_open(transaction, name.c_str(), flags, &dbi);
* \brief Opening database function specialization for uint16_t
*/
template<>
inline int LMDBAL::iStorage::makeStorage<uint16_t>(MDB_txn* transaction) {
return mdb_dbi_open(transaction, name.c_str(), MDB_CREATE | MDB_INTEGERKEY, &dbi);
}
/**
* \brief Opening database function specialization for uint8_t
*/
template<>
inline int LMDBAL::iStorage::makeStorage<uint8_t>(MDB_txn* transaction) {
return mdb_dbi_open(transaction, name.c_str(), MDB_CREATE | MDB_INTEGERKEY, &dbi);
}
/**
* \brief Opening database function specialization for int64_t
*/
template<>
inline int LMDBAL::iStorage::makeStorage<int64_t>(MDB_txn* transaction) {
return mdb_dbi_open(transaction, name.c_str(), MDB_CREATE | MDB_INTEGERKEY, &dbi);
}
/**
* \brief Opening database function specialization for int32_t
*/
template<>
inline int LMDBAL::iStorage::makeStorage<int32_t>(MDB_txn* transaction) {
return mdb_dbi_open(transaction, name.c_str(), MDB_CREATE | MDB_INTEGERKEY, &dbi);
}
/**
* \brief Opening database function specialization for int16_t
*/
template<>
inline int LMDBAL::iStorage::makeStorage<int16_t>(MDB_txn* transaction) {
return mdb_dbi_open(transaction, name.c_str(), MDB_CREATE | MDB_INTEGERKEY, &dbi);
}
/**
* \brief Opening database function specialization for int8_t
*/
template<>
inline int LMDBAL::iStorage::makeStorage<int8_t>(MDB_txn* transaction) {
return mdb_dbi_open(transaction, name.c_str(), MDB_CREATE | MDB_INTEGERKEY, &dbi);
} }
/** /**

View File

@ -16,6 +16,10 @@ protected:
~BaseTest() {} ~BaseTest() {}
uint32_t getT1Flags() const {return t1->flags();}
uint32_t getT2Flags() const {return t2->flags();}
uint32_t getC1Flags() const {return c1->flags();}
static void SetUpTestSuite() { static void SetUpTestSuite() {
if (db == nullptr) { if (db == nullptr) {
db = new LMDBAL::Base("testBase"); db = new LMDBAL::Base("testBase");
@ -56,6 +60,27 @@ TEST_F(BaseTest, OpeningClosingDatabase) {
EXPECT_EQ(db->ready(), true); EXPECT_EQ(db->ready(), true);
} }
TEST_F(BaseTest, Flags) {
uint32_t t1Flags = getT1Flags();
uint32_t t2Flags = getT2Flags();
uint32_t c1Flags = getC1Flags();
EXPECT_TRUE(t1Flags & MDB_INTEGERKEY);
EXPECT_FALSE(t1Flags & MDB_DUPSORT);
EXPECT_FALSE(t1Flags & MDB_DUPFIXED);
EXPECT_FALSE(t1Flags & MDB_INTEGERDUP);
EXPECT_FALSE(t2Flags & MDB_INTEGERKEY);
EXPECT_FALSE(t2Flags & MDB_DUPSORT);
EXPECT_FALSE(t2Flags & MDB_DUPFIXED);
EXPECT_FALSE(t2Flags & MDB_INTEGERDUP);
EXPECT_TRUE(c1Flags & MDB_INTEGERKEY);
EXPECT_FALSE(c1Flags & MDB_DUPSORT);
EXPECT_FALSE(c1Flags & MDB_DUPFIXED);
EXPECT_FALSE(c1Flags & MDB_INTEGERDUP);
}
TEST_F(BaseTest, AddingIntegerKey) { TEST_F(BaseTest, AddingIntegerKey) {
EXPECT_EQ(db->ready(), true); EXPECT_EQ(db->ready(), true);
t1->addRecord(1, 2); t1->addRecord(1, 2);