From f0727aa73d38f7122cfd002522e1629b30974c21 Mon Sep 17 00:00:00 2001 From: blue Date: Tue, 15 Aug 2023 15:48:19 -0300 Subject: [PATCH] started to work on duplicates support --- CHANGELOG.md | 13 ++++- src/base.h | 45 ++++++++--------- src/cache.h | 2 +- src/cache.hpp | 9 ++-- src/serializer.hpp | 3 +- src/storage.cpp | 9 +++- src/storage.h | 16 ++++-- src/storage.hpp | 118 ++++++++++++++++++--------------------------- test/basic.cpp | 25 ++++++++++ 9 files changed, 132 insertions(+), 108 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c27023..035262c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,22 @@ # Changelog +## LMDBAL 0.5.0 (UNRELEASED, 2023) +### New Features +- duplicates support + + ## LMDBAL 0.4.0 (August 13, 2023) +### New Features +- read only cursors + ### Bug fixes - possible cache unsync +- doxygen-awesome build bix ### Improvements -- read only cursors - some more documentation - more tests -- doxygen-awesome build bix + ## LMDBAL 0.3.1 (April 14, 2023) ### Bug fixes @@ -17,6 +25,7 @@ ### Improvements - exception documentation + ## LMDBAL 0.3.0 (April 12, 2023) ### New features - transaction functions diff --git a/src/base.h b/src/base.h index 3cee862..85f75c4 100644 --- a/src/base.h +++ b/src/base.h @@ -69,16 +69,16 @@ public: void abortTransaction(TransactionID id) const; template - LMDBAL::Storage* addStorage(const std::string& name); + LMDBAL::Storage* addStorage(const std::string& storageName, bool duplicates = false); template - LMDBAL::Cache* addCache(const std::string& name); + LMDBAL::Cache* addCache(const std::string& storageName); template - LMDBAL::Storage* getStorage(const std::string& name); + LMDBAL::Storage* getStorage(const std::string& storageName); template - LMDBAL::Cache* getCache(const std::string& name); + LMDBAL::Cache* getCache(const std::string& storageName); private: typedef std::map Storages; /** -LMDBAL::Storage* LMDBAL::Base::addStorage(const std::string& _name) { +LMDBAL::Storage* LMDBAL::Base::addStorage(const std::string& storageName, bool duplicates) { if (opened) { - throw Opened(name, "add storage " + _name); + throw Opened(name, "add storage " + storageName); } - Storage* storage = new Storage(_name, this); - std::pair pair = storages.insert(std::make_pair(_name, (iStorage*)storage)); + Storage* storage = new Storage(this, storageName, duplicates); + std::pair pair = storages.insert(std::make_pair(storageName, (iStorage*)storage)); if (!pair.second) - throw StorageDuplicate(name, _name); + throw StorageDuplicate(name, storageName); return storage; } @@ -142,7 +143,7 @@ LMDBAL::Storage* LMDBAL::Base::addStorage(const std::string& _name) { * Defines that the database is going to have the following cache. * The LMDBAL::Base must be closed * - * \param[in] _name - cache name + * \param[in] storageName - cache name * \returns cache pointer. LMDBAL::Base keeps the ownership of the added cache, you don't need to destoroy it * * \tparam K - key type of the cache @@ -152,14 +153,14 @@ LMDBAL::Storage* LMDBAL::Base::addStorage(const std::string& _name) { * \exception LMDBAL::StorageDuplicate thrown if somebody tries to add cache with repeating name */ template -LMDBAL::Cache * LMDBAL::Base::addCache(const std::string& _name) { +LMDBAL::Cache * LMDBAL::Base::addCache(const std::string& storageName) { if (opened) { - throw Opened(name, "add cache " + _name); + throw Opened(name, "add cache " + storageName); } - Cache* cache = new Cache(_name, this); - std::pair pair = storages.insert(std::make_pair(_name, (iStorage*)cache)); + Cache* cache = new Cache(this, storageName, false); + std::pair pair = storages.insert(std::make_pair(storageName, (iStorage*)cache)); if (!pair.second) - throw StorageDuplicate(name, _name); + throw StorageDuplicate(name, storageName); return cache; } @@ -173,7 +174,7 @@ LMDBAL::Cache * LMDBAL::Base::addCache(const std::string& _name) { * this method with template parameters * on the same name of the previously added storage, or calling it on cache - the behaviour is undefined * - * \param[in] _name - storage name + * \param[in] storageName - storage name * \returns storage pointer. LMDBAL::Base keeps the ownership of the added storage, you don't need to destoroy it * * \tparam K - key type of the storage @@ -182,8 +183,8 @@ LMDBAL::Cache * LMDBAL::Base::addCache(const std::string& _name) { * \exception std::out_of_range thrown if storage with the given name was not found */ template -LMDBAL::Storage* LMDBAL::Base::getStorage(const std::string& _name) { - return static_cast*>(storages.at(_name)); +LMDBAL::Storage* LMDBAL::Base::getStorage(const std::string& storageName) { + return static_cast*>(storages.at(storageName)); } /** @@ -195,7 +196,7 @@ LMDBAL::Storage* LMDBAL::Base::getStorage(const std::string& _name) { * this method with template parameters * on the same name of the previously added cache, or calling it on storage - the behaviour is undefined * - * \param[in] _name - cache name + * \param[in] storageName - cache name * \returns cache pointer. LMDBAL::Base keeps the ownership of the added cache, you don't need to destoroy it * * \tparam K - key type of the cache @@ -204,8 +205,8 @@ LMDBAL::Storage* LMDBAL::Base::getStorage(const std::string& _name) { * \exception std::out_of_range thrown if cache with the given name was not found */ template -LMDBAL::Cache* LMDBAL::Base::getCache(const std::string& _name) { - return static_cast*>(storages.at(_name)); +LMDBAL::Cache* LMDBAL::Base::getCache(const std::string& storageName) { + return static_cast*>(storages.at(storageName)); } #endif //LMDBAL_BASE_H diff --git a/src/cache.h b/src/cache.h index 41b3e06..ff0d996 100644 --- a/src/cache.h +++ b/src/cache.h @@ -51,7 +51,7 @@ class Cache : public Storage { typedef std::map TransactionCache; protected: - Cache(const std::string& name, Base* parent); + Cache(Base* parent, const std::string& name, bool duplicates = false); ~Cache() override; virtual void transactionStarted(TransactionID txn, bool readOnly) const override; diff --git a/src/cache.hpp b/src/cache.hpp index c82820d..063b1bd 100644 --- a/src/cache.hpp +++ b/src/cache.hpp @@ -39,12 +39,13 @@ /** * \brief Creates a cache * - * \param[in] _name - name of the new cache - * \param[in] parent - parent database pointed (borrowed) + * \param[in] parent - LMDBAL::Base pointer for the owning database (borrowed) + * \param[in] name - the name of the storage + * \param[in] duplicates - true if storage supports duplicates, false otherwise (false by default) */ template -LMDBAL::Cache::Cache(const std::string& _name, Base* parent): - Storage(_name, parent), +LMDBAL::Cache::Cache(Base* parent, const std::string& name, bool duplicates): + Storage(parent, name, duplicates), mode(Mode::nothing), cache(new std::map()), abscent(new std::set()), diff --git a/src/serializer.hpp b/src/serializer.hpp index 9b10cbd..80f4d90 100644 --- a/src/serializer.hpp +++ b/src/serializer.hpp @@ -23,11 +23,12 @@ /** * \class LMDBAL::Serializer + * \brief A class handling serialization/deserialization * * A class that is constructed in every LMDBAL::Storage * to serialize or deserialize keys and values. * - * It serializes to and deserializes from MDB_valMDB_val + * It serializes to and deserializes from MDB_val * * \tparam K type of the keys of the storage * \tparam V type of the values of the storage diff --git a/src/storage.cpp b/src/storage.cpp index d65923c..0f09886 100644 --- a/src/storage.cpp +++ b/src/storage.cpp @@ -30,11 +30,16 @@ /** * \brief Constructs a storage interface + * + * \param[in] parent - LMDBAL::Base pointer for the owning database (borrowed) + * \param[in] name - the name of the storage + * \param[in] duplicates - true if storage supports duplicates, false otherwise (false by default) */ -LMDBAL::iStorage::iStorage(const std::string& p_name, Base* parent): +LMDBAL::iStorage::iStorage(Base* parent, const std::string& name, bool duplicates): dbi(), db(parent), - name(p_name) + name(name), + duplicates(duplicates) {} /** diff --git a/src/storage.h b/src/storage.h index fa9516c..bd1229a 100644 --- a/src/storage.h +++ b/src/storage.h @@ -19,10 +19,14 @@ #ifndef LMDBAL_STORAGE_H #define LMDBAL_STORAGE_H +#include + #include "base.h" #include "serializer.h" #include "cursor.h" +class BaseTest; + namespace LMDBAL { typedef uint32_t SizeType; @@ -32,7 +36,7 @@ class iStorage { public: protected: - iStorage(const std::string& name, Base* parent); + iStorage(Base* parent, const std::string& name, bool duplicates = false); virtual ~iStorage(); /** @@ -77,9 +81,11 @@ protected: MDB_dbi dbi; /**<\brief lmdb storage handle*/ Base* db; /**<\brief parent database pointer (borrowed)*/ const std::string name; /**<\brief this storage name*/ + const bool duplicates; /**<\brief true if storage supports duplicates*/ inline static const std::string dropMethodName = "drop"; /**<\brief member function name, just for exceptions*/ inline static const std::string countMethodName = "count"; /**<\brief member function name, just for exceptions*/ + inline static const std::string flagsMethodName = "flags"; /**<\brief member function name, just for exceptions*/ inline static const std::string addRecordMethodName = "addRecord"; /**<\brief member function name, just for exceptions*/ inline static const std::string forceRecordMethodName = "forceRecord"; /**<\brief member function name, just for exceptions*/ @@ -92,8 +98,8 @@ protected: inline static const std::string addRecordsMethodName = "addRecords"; /**<\brief member function name, just for exceptions*/ protected: - template - int makeStorage(MDB_txn* transaction); + template + int makeStorage(MDB_txn* transaction, bool duplicates); template static std::string toString(const T& value); @@ -101,14 +107,16 @@ protected: template class Storage : public iStorage { + friend class ::BaseTest; friend class Base; friend class Cursor; protected: - Storage(const std::string& name, Base* parent); + Storage(Base* parent, const std::string& name, bool duplicates = false); ~Storage() override; virtual void discoveredRecord(const K& key, const V& value) const; virtual void discoveredRecord(const K& key, const V& value, TransactionID txn) const; + uint32_t flags() const; public: using iStorage::drop; diff --git a/src/storage.hpp b/src/storage.hpp index 5e33f08..d1dcbae 100644 --- a/src/storage.hpp +++ b/src/storage.hpp @@ -41,12 +41,13 @@ /** * \brief Creates a storage * - * \param[in] _name - name of the new storage - * \param[in] parent - parent database pointed (borrowed) + * \param[in] parent - LMDBAL::Base pointer for the owning database (borrowed) + * \param[in] name - the name of the storage + * \param[in] duplicates - true if storage supports duplicates, false otherwise (false by default) */ template -LMDBAL::Storage::Storage(const std::string& _name, Base* parent): - iStorage(_name, parent), +LMDBAL::Storage::Storage(Base* parent, const std::string& name, bool duplicates): + iStorage(parent, name, duplicates), keySerializer(), valueSerializer(), cursors() @@ -692,7 +693,7 @@ void LMDBAL::Storage::removeRecord(const K& key, TransactionID txn) { */ template int LMDBAL::Storage::open(MDB_txn* transaction) { - return makeStorage(transaction); + return makeStorage(transaction, duplicates); } /** @@ -719,6 +720,30 @@ LMDBAL::Cursor* LMDBAL::Storage::createCursor() { return cursor; } +/** + * \brief Reads current storage flags it was opened with + * + * This function exists mostly for testing purposes + * + * \returns Third out parameter of mdb_dbi_flags function + * + * \exception LMDBAL::Closed thrown if the database was not opened + * \exception LMDBAL::Unknown if the result of mdb_dbi_flags was not successfull + */ +template +uint32_t LMDBAL::Storage::flags() const { + ensureOpened(flagsMethodName); + uint32_t result; + TransactionID txn = beginReadOnlyTransaction(); + + int res = mdb_dbi_flags(txn, dbi, &result); + abortTransaction(txn); + if (res != MDB_SUCCESS) + throwUnknown(res); + + return result; +} + /** * \brief Destroys cursor * @@ -726,7 +751,7 @@ LMDBAL::Cursor* LMDBAL::Storage::createCursor() { * * \param[in] cursor a pointer to a cursor you want to destroy * - * \throws LMDBAL::Unknown thrown if you try to destroy something that this storage didn't create + * \exception LMDBAL::Unknown thrown if you try to destroy something that this storage didn't create */ template void LMDBAL::Storage::destroyCursor(Cursor* cursor) { @@ -770,79 +795,28 @@ void LMDBAL::Storage::discoveredRecord(const K& key, const V& value, Trans * \tparam K type of keys in opening storage * * \param[in] transaction - lmdb transaction to call mdb_dbi_open, must be a writable transaction! + * \param[in] duplicates - true if you wish to enable duplicates support for the storage + * * \returns MDB_SUCCESS if everything went smooth or MDB_ -like error code * - * This and the following collection of specializations are a way to optimise database using - * MDB_INTEGERKEY flag, when the key is actually kind of an integer + * This is a way to optimise database using MDB_INTEGERKEY flag, + * when the key is actually kind of an integer * This infrastructure also allowes us to customize mdb_dbi_open call in the future */ -template -inline int LMDBAL::iStorage::makeStorage(MDB_txn* transaction) { - return mdb_dbi_open(transaction, name.c_str(), MDB_CREATE, &dbi); -} +template +inline int LMDBAL::iStorage::makeStorage(MDB_txn* transaction, bool duplicates) { + unsigned int flags = MDB_CREATE; + if constexpr (std::is_integral::value) + flags |= MDB_INTEGERKEY; -/** - * \brief Opening database function specialization for uint64_t - */ -template<> -inline int LMDBAL::iStorage::makeStorage(MDB_txn* transaction) { - return mdb_dbi_open(transaction, name.c_str(), MDB_CREATE | MDB_INTEGERKEY, &dbi); -} + if (duplicates) { + flags |= MDB_DUPSORT; -/** - * \brief Opening database function specialization for uint32_t - */ -template<> -inline int LMDBAL::iStorage::makeStorage(MDB_txn* transaction) { - return mdb_dbi_open(transaction, name.c_str(), MDB_CREATE | MDB_INTEGERKEY, &dbi); -} + if constexpr (std::is_integral::value) + flags |= MDB_INTEGERDUP | MDB_DUPFIXED; + } -/** - * \brief Opening database function specialization for uint16_t - */ -template<> -inline int LMDBAL::iStorage::makeStorage(MDB_txn* transaction) { - return mdb_dbi_open(transaction, name.c_str(), MDB_CREATE | MDB_INTEGERKEY, &dbi); -} - -/** - * \brief Opening database function specialization for uint8_t - */ -template<> -inline int LMDBAL::iStorage::makeStorage(MDB_txn* transaction) { - return mdb_dbi_open(transaction, name.c_str(), MDB_CREATE | MDB_INTEGERKEY, &dbi); -} - -/** - * \brief Opening database function specialization for int64_t - */ -template<> -inline int LMDBAL::iStorage::makeStorage(MDB_txn* transaction) { - return mdb_dbi_open(transaction, name.c_str(), MDB_CREATE | MDB_INTEGERKEY, &dbi); -} - -/** - * \brief Opening database function specialization for int32_t - */ -template<> -inline int LMDBAL::iStorage::makeStorage(MDB_txn* transaction) { - return mdb_dbi_open(transaction, name.c_str(), MDB_CREATE | MDB_INTEGERKEY, &dbi); -} - -/** - * \brief Opening database function specialization for int16_t - */ -template<> -inline int LMDBAL::iStorage::makeStorage(MDB_txn* transaction) { - return mdb_dbi_open(transaction, name.c_str(), MDB_CREATE | MDB_INTEGERKEY, &dbi); -} - -/** - * \brief Opening database function specialization for int8_t - */ -template<> -inline int LMDBAL::iStorage::makeStorage(MDB_txn* transaction) { - return mdb_dbi_open(transaction, name.c_str(), MDB_CREATE | MDB_INTEGERKEY, &dbi); + return mdb_dbi_open(transaction, name.c_str(), flags, &dbi); } /** diff --git a/test/basic.cpp b/test/basic.cpp index 7369de6..460ce36 100644 --- a/test/basic.cpp +++ b/test/basic.cpp @@ -16,6 +16,10 @@ protected: ~BaseTest() {} + uint32_t getT1Flags() const {return t1->flags();} + uint32_t getT2Flags() const {return t2->flags();} + uint32_t getC1Flags() const {return c1->flags();} + static void SetUpTestSuite() { if (db == nullptr) { db = new LMDBAL::Base("testBase"); @@ -56,6 +60,27 @@ TEST_F(BaseTest, OpeningClosingDatabase) { EXPECT_EQ(db->ready(), true); } +TEST_F(BaseTest, Flags) { + uint32_t t1Flags = getT1Flags(); + uint32_t t2Flags = getT2Flags(); + uint32_t c1Flags = getC1Flags(); + + EXPECT_TRUE(t1Flags & MDB_INTEGERKEY); + EXPECT_FALSE(t1Flags & MDB_DUPSORT); + EXPECT_FALSE(t1Flags & MDB_DUPFIXED); + EXPECT_FALSE(t1Flags & MDB_INTEGERDUP); + + EXPECT_FALSE(t2Flags & MDB_INTEGERKEY); + EXPECT_FALSE(t2Flags & MDB_DUPSORT); + EXPECT_FALSE(t2Flags & MDB_DUPFIXED); + EXPECT_FALSE(t2Flags & MDB_INTEGERDUP); + + EXPECT_TRUE(c1Flags & MDB_INTEGERKEY); + EXPECT_FALSE(c1Flags & MDB_DUPSORT); + EXPECT_FALSE(c1Flags & MDB_DUPFIXED); + EXPECT_FALSE(c1Flags & MDB_INTEGERDUP); +} + TEST_F(BaseTest, AddingIntegerKey) { EXPECT_EQ(db->ready(), true); t1->addRecord(1, 2);