From de210b44f5985d0bc8028b9ca1692ffb076ceab9 Mon Sep 17 00:00:00 2001 From: blue Date: Sun, 20 Aug 2023 13:38:29 -0300 Subject: [PATCH] readAll method now works correctly for duplicates mode, testing, some doc fixes --- doc/mainpage.dox | 2 +- src/base.h | 8 +-- src/storage.cpp | 2 + src/storage.hpp | 20 ++++++- test/duplicates.cpp | 139 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 163 insertions(+), 8 deletions(-) diff --git a/doc/mainpage.dox b/doc/mainpage.dox index 7cd5015..2192aa4 100644 --- a/doc/mainpage.dox +++ b/doc/mainpage.dox @@ -15,7 +15,7 @@ * std::map * to speed up the access. * - * You can obtain handlers by calling LMDBAL::Base::addStorage(const std::string&) or LMDBAL::Base::addCache(const std::string& name). + * You can obtain handlers by calling LMDBAL::Base::addStorage(const std::string&, bool) or LMDBAL::Base::addCache(const std::string& name). * Note that the handlers still belong to the LMDBAL::Base and it's his responsibility to destroy them. * You are not obliged to save those handlers, * you can obtain them at any time later using methods LMDBAL::Base::getStorage(const std::string&) or LMDBAL::Base::getCache(const std::string&) diff --git a/src/base.h b/src/base.h index fd5ba18..8b21c77 100644 --- a/src/base.h +++ b/src/base.h @@ -46,8 +46,8 @@ class Storage; template class Cache; -typedef MDB_txn* TransactionID; /*** getCache(const std::string& storageName); private: - typedef std::map Storages; /** Transactions; /** Storages; /**<\brief Storage and Cache pointers are saved in the std::map*/ + typedef std::set Transactions; /**<\brief Piblic transaction IDs are saved in the std::set*/ TransactionID beginReadOnlyTransaction(const std::string& storageName) const; TransactionID beginTransaction(const std::string& storageName) const; diff --git a/src/storage.cpp b/src/storage.cpp index 2f50cd4..b96ad4c 100644 --- a/src/storage.cpp +++ b/src/storage.cpp @@ -23,6 +23,8 @@ /** * \class LMDBAL::iStorage * + * \brief Storage interface + * * This is a interface-like class, it's designed to be an inner database interface to * be used as a polymorphic entity, and provide protected interaction with the database * from the heirs code diff --git a/src/storage.hpp b/src/storage.hpp index 8a8ef31..b3ef621 100644 --- a/src/storage.hpp +++ b/src/storage.hpp @@ -31,7 +31,7 @@ * \tparam K type of the keys of the storage * \tparam V type of the values of the storage * - * You can receive an instance of this class calling LMDBAL::Base::addStorage(const std::string&) + * You can receive an instance of this class calling LMDBAL::Base::addStorage(const std::string&, bool) * if the database is yet closed and you're defining the storages you're going to need. * Or you can call LMDBAL::Base::getStorage(const std::string&) if you didn't save a pointer to the storage at first * @@ -501,6 +501,9 @@ bool LMDBAL::Storage::checkRecord(const K& key, TransactionID txn) const { * * Basically just reads all database in an std::map, usefull when you store small storages * + * In case storage supports duplicates only what lmdb considered to be lowest value + * (see LMDBAL::Storage::getRecord() description) is returned in the resulting map + * * \exception LMDBAL::Closed thrown if the database was not opened * \exception LMDBAL::Unknown thrown if something unexpected happend within lmdb */ @@ -518,6 +521,9 @@ std::map LMDBAL::Storage::readAll() const { * * Basically just reads all database in an std::map, usefull when you store small storages * + * In case storage supports duplicates only what lmdb considered to be lowest value + * (see LMDBAL::Storage::getRecord() description) is returned in the resulting map + * * \param[out] result a map that is going to contain all data * * \exception LMDBAL::Closed thrown if the database was not opened @@ -545,6 +551,9 @@ void LMDBAL::Storage::readAll(std::map& result) const { * You can obtain LMDBAL::TransactionID calling LMDBAL::Base::beginReadOnlyTransaction() or LMDBAL::Base::beginTransaction(). * If you just want to read data you should prefer LMDBAL::Base::beginReadOnlyTransaction(). * + * In case storage supports duplicates only what lmdb considered to be lowest value + * (see LMDBAL::Storage::getRecord() description) is returned in the resulting map + * * \param[in] txn transaction ID, can be read only transaction * * \exception LMDBAL::Closed thrown if the database was not opened @@ -566,6 +575,9 @@ std::map LMDBAL::Storage::readAll(TransactionID txn) const { * You can obtain LMDBAL::TransactionID calling LMDBAL::Base::beginReadOnlyTransaction() or LMDBAL::Base::beginTransaction(). * If you just want to read data you should prefer LMDBAL::Base::beginReadOnlyTransaction(). * + * In case storage supports duplicates only what lmdb considered to be lowest value + * (see LMDBAL::Storage::getRecord() description) is returned in the resulting map + * * \param[out] result a map that is going to contain all data * \param[in] txn transaction ID, can be read only transaction * @@ -587,8 +599,10 @@ void LMDBAL::Storage::readAll(std::map& result, TransactionID txn) c while (rc == MDB_SUCCESS) { K key; keySerializer.deserialize(lmdbKey, key); - V& value = result[key]; - valueSerializer.deserialize(lmdbData, value); + std::pair::iterator, bool> probe = result.emplace(key, V{}); + if (probe.second) //I do this to avoid overwrites in case duplicates are enabled + valueSerializer.deserialize(lmdbData, probe.first->second); + rc = mdb_cursor_get(cursor, &lmdbKey, &lmdbData, MDB_NEXT); } mdb_cursor_close(cursor); diff --git a/test/duplicates.cpp b/test/duplicates.cpp index d84ba2b..56b63fd 100644 --- a/test/duplicates.cpp +++ b/test/duplicates.cpp @@ -1,9 +1,12 @@ #include #include +#include +#include #include "base.h" #include "storage.h" +#include "cursor.h" class DuplicatesTest : public ::testing::Test { protected: @@ -326,3 +329,139 @@ TEST_F(DuplicatesTest, Changing) { EXPECT_EQ(tu4->getRecord(852), 213.85); EXPECT_THROW(tu4->changeRecord(852, 46324.1135), LMDBAL::Exist); } + +TEST_F(DuplicatesTest, GettingAllRecords) { + LMDBAL::TransactionID txn = db->beginReadOnlyTransaction(); + bool cycle; + LMDBAL::SizeType iterations; + + std::map m1; + std::set k1; + LMDBAL::Cursor* c1 = tu1->createCursor(); + tu1->readAll(m1, txn); + c1->open(txn); + + cycle = false; + iterations = 0; + do { + try { + std::pair pair = c1->next(); + cycle = true; + std::pair::const_iterator, bool> probe = k1.insert(pair.first); + if (probe.second) { + uint16_t valueAll = m1.at(pair.first); + uint16_t valueGet; + EXPECT_NO_THROW(tu1->getRecord(pair.first, valueGet, txn)); + EXPECT_EQ(valueAll, valueGet); + } + ++iterations; + } catch (const LMDBAL::NotFound& e) { + cycle = false; + } + } while (cycle); + tu1->destroyCursor(c1); + + EXPECT_EQ(iterations, tu1->count(txn)); + EXPECT_EQ(k1.size(), m1.size()); + EXPECT_NE(iterations, 0); + EXPECT_NE(k1.size(), 0); + + + std::map m2; + std::set k2; + LMDBAL::Cursor* c2 = tu2->createCursor(); + tu2->readAll(m2, txn); + c2->open(txn); + + cycle = false; + iterations = 0; + do { + try { + std::pair pair = c2->next(); + cycle = true; + std::pair::const_iterator, bool> probe = k2.insert(pair.first); + if (probe.second) { + int8_t valueAll = m2.at(pair.first); + int8_t valueGet; + EXPECT_NO_THROW(tu2->getRecord(pair.first, valueGet, txn)); + EXPECT_EQ(valueAll, valueGet); + } + ++iterations; + } catch (const LMDBAL::NotFound& e) { + cycle = false; + } + } while (cycle); + tu2->destroyCursor(c2); + + EXPECT_EQ(iterations, tu2->count(txn)); + EXPECT_EQ(k2.size(), m2.size()); + EXPECT_NE(iterations, 0); + EXPECT_NE(k2.size(), 0); + + + std::map m3; + std::set k3; + LMDBAL::Cursor* c3 = tu3->createCursor(); + tu3->readAll(m3, txn); + c3->open(txn); + + cycle = false; + iterations = 0; + do { + try { + std::pair pair = c3->next(); + cycle = true; + std::pair::const_iterator, bool> probe = k3.insert(pair.first); + if (probe.second) { + float valueAll = m3.at(pair.first); + float valueGet; + EXPECT_NO_THROW(tu3->getRecord(pair.first, valueGet, txn)); + EXPECT_EQ(valueAll, valueGet); + } + ++iterations; + } catch (const LMDBAL::NotFound& e) { + cycle = false; + } + } while (cycle); + tu3->destroyCursor(c3); + + EXPECT_EQ(iterations, tu3->count(txn)); + EXPECT_EQ(k3.size(), m3.size()); + EXPECT_NE(iterations, 0); + EXPECT_NE(k3.size(), 0); + + + std::map m4; + std::set k4; + LMDBAL::Cursor* c4 = tu4->createCursor(); + tu4->readAll(m4, txn); + c4->open(txn); + + cycle = false; + iterations = 0; + do { + try { + std::pair pair = c4->next(); + cycle = true; + std::pair::const_iterator, bool> probe = k4.insert(pair.first); + if (probe.second) { + double valueAll = m4.at(pair.first); + double valueGet; + EXPECT_NO_THROW(tu4->getRecord(pair.first, valueGet, txn)); + EXPECT_EQ(valueAll, valueGet); + } + ++iterations; + } catch (const LMDBAL::NotFound& e) { + cycle = false; + } + } while (cycle); + tu4->destroyCursor(c4); + + EXPECT_EQ(iterations, tu4->count(txn)); + EXPECT_EQ(k4.size(), m4.size()); + EXPECT_NE(iterations, 0); + EXPECT_NE(k4.size(), 0); + + + db->abortTransaction(txn); +}