lmdbal/src/storage.hpp

/*
 * LMDB Abstraction Layer.
 * Copyright (C) 2023  Yury Gubich <blue@macaw.me>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#ifndef LMDBAL_STORAGE_HPP
#define LMDBAL_STORAGE_HPP

#include "storage.h"
#include "exceptions.h"

#define UNUSED(x) (void)(x)

/**
 * \class LMDBAL::Storage
 * \brief This is a basic key value storage.
 *
 * \tparam K type of the keys of the storage
 * \tparam V type of the values of the storage
 *
 * You can receive an instance of this class calling LMDBAL::Base::addStorage(const std::string&, bool)
 * if the database is yet closed and you're defining the storages you're going to need.
 * Or you can call LMDBAL::Base::getStorage(const std::string&) if you didn't save a pointer to the storage at first
 *
 * You are not supposed to instantiate or destory instances of this class yourself!
 */

/**
 * \brief Creates a storage
 *
 * \param[in]   parent      - LMDBAL::Base pointer for the owning database (borrowed)
 * \param[in]   name        - the name of the storage
 * \param[in]   duplicates  - true if key duplicates are allowed (false by default)
 */
template<class K, class V>
LMDBAL::Storage<K, V>::Storage(Base* parent, const std::string& name, bool duplicates):
    iStorage(parent, name, duplicates),
    keySerializer(),
    valueSerializer(),
    cursors()
{}

/**
 * \brief Destroys a storage
 */
template<class K, class V>
LMDBAL::Storage<K, V>::~Storage() {
    for (Cursor<K, V>* cursor : cursors)
        delete cursor;
}

/**
 * \brief Adds a key-value record to the storage
 *
 * Take a note that if the storage already had a record you want to add LMDBAL::Exist is thrown.
 * If your storage doesn't support duplicates LMDBAL::Exist is thrown if the record with the same key already exists in the database.
 * If your storage supports duplicates LMDBAL::Exist is thrown only if the record with the same key <b>AND</b> already exists in the database.
 *
 * \param[in] key      key of the record
 * \param[in] value    value of the record
 *
 * \exception LMDBAL::Exist     thrown if the storage already has a record with the given key
 * \exception LMDBAL::Closed    thrown if the database was not opened
 * \exception LMDBAL::Unknown   thrown if something unexpected happend within lmdb
 */
template<class K, class V>
void LMDBAL::Storage<K, V>::addRecord(const K& key, const V& value) {
    ensureOpened(addRecordMethodName);
    TransactionID txn = beginTransaction();
    try {
        Storage<K, V>::addRecord(key, value, txn);
    } catch (...) {
        abortTransaction(txn);
        throw;
    }

    commitTransaction(txn);
}

/**
 * \brief Adds a key-value record to the storage (transaction variant)
 *
 * This method schedules an addition of a key-value record, but doesn't immidiately adds it.
 * You can obtain LMDBAL::TransactionID calling LMDBAL::Base::beginTransaction().
 *
 * Take a note that if the storage already had a record you want to add LMDBAL::Exist is thrown.
 * If your storage doesn't support duplicates LMDBAL::Exist is thrown if the record with the same key already exists in the database.
 * If your storage supports duplicates LMDBAL::Exist is thrown only if the record with the same key <b>AND</b> already exists in the database.
 *
 * \param[in] key      key of the record
 * \param[in] value    value of the record
 * \param[in] txn      transaction ID, needs to be a writable transaction!
 *
 * \exception LMDBAL::Exist     thrown if the storage already has a record with the given key
 * \exception LMDBAL::Closed    thrown if the database was not opened
 * \exception LMDBAL::Unknown   thrown if something unexpected happend within lmdb
 */
template<class K, class V>
void LMDBAL::Storage<K, V>::addRecord(const K& key, const V& value, TransactionID txn) {
    ensureOpened(addRecordMethodName);

    MDB_val lmdbKey = keySerializer.setData(key);
    MDB_val lmdbData = valueSerializer.setData(value);

    unsigned int flags = 0;
    if (duplicates)
        flags |= MDB_NODUPDATA;
    else
        flags |= MDB_NOOVERWRITE;

    int rc = mdb_put(txn, dbi, &lmdbKey, &lmdbData, flags);
    if (rc != MDB_SUCCESS)
        throwDuplicateOrUnknown(rc, toString(key));
}

/**
 * \brief Adds a key-value record to the storage, overwrites if it already exists
 *
 * This method is mostly useless in duplicates mode.
 * In this mode it basically does the same thing LMDBAL::Storage::addRecord() does,
 * but suppresses LMDBAL::Exist exception if the record with the same key-value pair existed in the storage.
 * In this case just false is returned from the method.
 *
 * \param[in] key      key of the record
 * \param[in] value    value of the record
 *
 * \returns true if the record was added, false otherwise
 *
 * \exception LMDBAL::Closed    thrown if the database was not opened
 * \exception LMDBAL::Unknown   thrown if something unexpected happend within lmdb
 */
template<class K, class V>
bool LMDBAL::Storage<K, V>::forceRecord(const K& key, const V& value) {
    ensureOpened(forceRecordMethodName);

    TransactionID txn = beginTransaction();
    bool added;
    try {
        added = Storage<K, V>::forceRecord(key, value, txn);
    } catch (...) {
        abortTransaction(txn);
        throw;
    }

    commitTransaction(txn);
    return added;
}

/**
 * \brief Adds a key-value record to the storage, overwrites if it already exists (transaction variant)
 *
 * This method schedules an addition of a key-value record, but doesn't immidiately add it.
 * You can obtain LMDBAL::TransactionID calling LMDBAL::Base::beginTransaction().
 * If the record did already exist in the database the actual overwrite will be done only after calling LMDBAL::Base::commitTransaction().
 *
 * This method is mostly useless in duplicates mode.
 * In this mode it basically does the same thing LMDBAL::Storage::addRecord() does,
 * but suppresses LMDBAL::Exist exception if the record with the same key-value pair existed in the storage.
 * In this case just false is returned from the method.
 *
 * \param[in] key      key of the record
 * \param[in] value    value of the record
 * \param[in] txn      transaction ID, needs to be a writable transaction!
 * \returns true if the record was added, false otherwise
 *
 * \exception LMDBAL::Closed    thrown if the database was not opened
 * \exception LMDBAL::Unknown   thrown if something unexpected happend within lmdb
 */
template<class K, class V>
bool LMDBAL::Storage<K, V>::forceRecord(const K& key, const V& value, TransactionID txn) {
    ensureOpened(forceRecordMethodName);

    bool added;
    if (duplicates) {
        try {
            addRecord(key, value, txn);
            added = true;
        } catch (const LMDBAL::Exist& e) {
            added = false;
        }
    } else {
        MDB_val lmdbKey = keySerializer.setData(key);
        MDB_val lmdbData;

        int rc = mdb_get(txn, dbi, &lmdbKey, &lmdbData);
        switch (rc) {
            case MDB_SUCCESS:
                added = false;
                break;
            case MDB_NOTFOUND:
                added = true;
                break;
            default:
                added = false;
                throwUnknown(rc);
        }

        lmdbData = valueSerializer.setData(value);
        rc = mdb_put(txn, dbi, &lmdbKey, &lmdbData, 0);
        if (rc != MDB_SUCCESS)
            throwUnknown(rc);
    }
    return added;
}

/**
 * \brief Changes key-value record to the storage.
 *
 * Take a note that if the storage didn't have a record you want to change LMDBAL::NotFound is thrown
 *
 * If duplicates mode is enabled this function will find the first entry of the key
 * (which is pretty couterintuitive, see LMDBAL::Storage::getRecord() description)
 * and change it's value to the given one.
 * If the given value matches some of the other values for given key the method will throw LMDBAL::Exist,
 * if no key was found it will still throw LMDBAL::NotFound.
 *
 * \param[in] key      key of the record
 * \param[in] value    new value of the record
 *
 * \exception LMDBAL::NotFound  thrown if the storage doesn't have a record with the given key
 * \exception LMDBAL::Exist     thrown in duplicates mode when the given value matches some of existing values for the given key
 * \exception LMDBAL::Closed    thrown if the database was not opened
 * \exception LMDBAL::Unknown   thrown if something unexpected happend within lmdb
 */
template<class K, class V>
void LMDBAL::Storage<K, V>::changeRecord(const K& key, const V& value) {
    ensureOpened(changeRecordMethodName);

    TransactionID txn = beginTransaction();
    try {
        Storage<K, V>::changeRecord(key, value, txn);
    } catch (...) {
        abortTransaction(txn);
        throw;
    }

    commitTransaction(txn);
}

/**
 * \brief Changes key-value record to the storage (transaction variant)
 *
 * This method schedules a modification of a key-value record, but doesn't immidiately changes it.
 * You can obtain LMDBAL::TransactionID calling LMDBAL::Base::beginTransaction().
 *
 * Take a note that if the storage didn't have a record you want to change LMDBAL::NotFound is thrown
 *
 * If duplicates mode is enabled this function will find the first entry of the key
 * (which is pretty couterintuitive, see LMDBAL::Storage::getRecord() description)
 * and change it's value to the given one.
 * If the given value matches some of the other values for given key the method will throw LMDBAL::Exist,
 * if no key was found it will still throw LMDBAL::NotFound.
 *
 * \param[in] key      key of the record
 * \param[in] value    new value of the record
 * \param[in] txn      transaction ID, needs to be a writable transaction!
 *
 * \exception LMDBAL::NotFound  thrown if the storage doesn't have a record with the given key
 * \exception LMDBAL::Exist     thrown in duplicates mode when the given value matches some of existing values for the given key
 * \exception LMDBAL::Closed    thrown if the database was not opened
 * \exception LMDBAL::Unknown   thrown if something unexpected happend within lmdb
 */
template<class K, class V>
void LMDBAL::Storage<K, V>::changeRecord(const K& key, const V& value, TransactionID txn) {
    ensureOpened(changeRecordMethodName);

    MDB_cursor* cursor;
    int rc = mdb_cursor_open(txn, dbi, &cursor);
    if (rc != MDB_SUCCESS)
        throwUnknown(rc);

    MDB_val lmdbKey = keySerializer.setData(key);
    MDB_val lmdbData;
    rc = mdb_cursor_get(cursor, &lmdbKey, &lmdbData, MDB_SET);
    if (rc != MDB_SUCCESS)
        throwNotFoundOrUnknown(rc, toString(key));

    MDB_val lmdbNewData = valueSerializer.setData(value);
    bool sameSize = lmdbData.mv_size == lmdbNewData.mv_size;
    int firstDifferentByte = 0;
    if (sameSize) {                                             //can compare only if they are the same size
        firstDifferentByte = memcmp(lmdbData.mv_data, lmdbNewData.mv_data, lmdbData.mv_size);
        if (firstDifferentByte == 0) {                          //old and new is the same, nothing to do
            mdb_cursor_close(cursor);
            return;
        }
    }

    unsigned int flags = MDB_CURRENT;
    if (duplicates && (!sameSize || firstDifferentByte < 0)) {  //if new value is greater than the old one
        rc = mdb_cursor_del(cursor, 0);                         //we need to initiate duplicates sort, for it to be in the correct place
        flags = MDB_NODUPDATA;
    }

    if (rc == MDB_SUCCESS)
        rc = mdb_cursor_put(cursor, &lmdbKey, &lmdbNewData, flags);

    mdb_cursor_close(cursor);
    if (rc != MDB_SUCCESS)
        throwDuplicateOrUnknown(rc, toString(key));
}

/**
 * \brief Gets the record from the database

 * Take a note that if the storage didn't have a record you want to get LMDBAL::NotFound is thrown
 *
 * If the storage supports duplicates the exact value returned from it depends on comparison function of lmdb.
 * It's not very straight forward, so, you shouldn't really use this method if you use duplicates and you rely on exact result.
 * Anyway:
 * - if your values are signed or unsigned integer of any size the LOWEST value is returned compared as <b>UNSIGNED</b>. For example for storage with int32_t as value, from the same key, from the set of values {-33, -1, 5573, 77753} 5573 is returned as it is the lowest by <b>UNSIGNED</b> comparison.
 * - if your values are anything else - they are compared byte by byte as if they are strings, it makes it especially complicated to predict the exact value for float or double templated storages. For strings if makes a bit more sence: if the choise is from "50" and "100" - "100" is returned, because the first byte of the "100" is lower than the first byte of the "50"
 *
 * \param[in] key      key of the record you look for
 * \returns the value from the storage
 *
 * \exception LMDBAL::NotFound  thrown if the storage doesn't have a record with the given key
 * \exception LMDBAL::Closed    thrown if the database was not opened
 * \exception LMDBAL::Unknown   thrown if something unexpected happend within lmdb
 */
template<class K, class V>
V LMDBAL::Storage<K, V>::getRecord(const K& key) const {
    ensureOpened(getRecordMethodName);

    V value;
    Storage<K, V>::getRecord(key, value);
    return value;
}

/**
 * \brief Gets the record from the database (reference variant)
 *
 * Take a note that if the storage didn't have a record you want to get LMDBAL::NotFound is thrown
 *
 * If the storage supports duplicates the exact value returned from it depends on comparison function of lmdb.
 * It's not very straight forward, so, you shouldn't really use this method if you use duplicates and you rely on exact result.
 * Anyway:
 * - if your values are signed or unsigned integer of any size the LOWEST value is returned compared as <b>UNSIGNED</b>. For example for storage with int32_t as value, from the same key, from the set of values {-33, -1, 5573, 77753} 5573 is returned as it is the lowest by <b>UNSIGNED</b> comparison.
 * - if your values are anything else - they are compared byte by byte as if they are strings, it makes it especially complicated to predict the exact value for float or double templated storages. For strings if makes a bit more sence: if the choise is from "50" and "100" - "100" is returned, because the first byte of the "100" is lower than the first byte of the "50"
 *
 * \param[in] key      key of the record you look for
 * \param[out] value   the value from the storage
 *
 * \exception LMDBAL::NotFound  thrown if the storage doesn't have a record with the given key
 * \exception LMDBAL::Closed    thrown if the database was not opened
 * \exception LMDBAL::Unknown   thrown if something unexpected happend within lmdb
 */
template<class K, class V>
void LMDBAL::Storage<K, V>::getRecord(const K& key, V& value) const {
    ensureOpened(getRecordMethodName);

    TransactionID txn = beginReadOnlyTransaction();
    try {
        Storage<K, V>::getRecord(key, value, txn);
    } catch (...) {
        abortTransaction(txn);
        throw;
    }

    abortTransaction(txn);
}

/**
 * \brief Gets the record from the database (transaction variant)
 *
 * You can obtain LMDBAL::TransactionID calling LMDBAL::Base::beginReadOnlyTransaction() or LMDBAL::Base::beginTransaction().
 * If you just want to read data you should prefer LMDBAL::Base::beginReadOnlyTransaction().
 *
 * Take a note that if the storage didn't have a record you want to get LMDBAL::NotFound is thrown
 *
 * If the storage supports duplicates the exact value returned from it depends on comparison function of lmdb.
 * It's not very straight forward, so, you shouldn't really use this method if you use duplicates and you rely on exact result.
 * Anyway:
 * - if your values are signed or unsigned integer of any size the LOWEST value is returned compared as <b>UNSIGNED</b>. For example for storage with int32_t as value, from the same key, from the set of values {-33, -1, 5573, 77753} 5573 is returned as it is the lowest by <b>UNSIGNED</b> comparison.
 * - if your values are anything else - they are compared byte by byte as if they are strings, it makes it especially complicated to predict the exact value for float or double templated storages. For strings if makes a bit more sence: if the choise is from "50" and "100" - "100" is returned, because the first byte of the "100" is lower than the first byte of the "50"
 *
 * \param[in] key      key of the record you look for
 * \param[in] txn      transaction ID, can be read only transaction
 * \returns the value from the storage
 *
 * \exception LMDBAL::NotFound  thrown if the storage doesn't have a record with the given key
 * \exception LMDBAL::Closed    thrown if the database was not opened
 * \exception LMDBAL::Unknown   thrown if something unexpected happend within lmdb
 */
template<class K, class V>
V LMDBAL::Storage<K, V>::getRecord(const K& key, TransactionID txn) const {
    ensureOpened(getRecordMethodName);

    V value;
    Storage<K, V>::getRecord(key, value, txn);
    return value;
}

/**
 * \brief Gets the record from the database (transaction, reference variant)
 *
 * You can obtain LMDBAL::TransactionID calling LMDBAL::Base::beginReadOnlyTransaction() or LMDBAL::Base::beginTransaction().
 * If you just want to read data you should prefer LMDBAL::Base::beginReadOnlyTransaction().
 *
 * Take a note that if the storage didn't have a record you want to get LMDBAL::NotFound is thrown
 *
 * If the storage supports duplicates the exact value returned from it depends on comparison function of lmdb.
 * It's not very straight forward, so, you shouldn't really use this method if you use duplicates and you rely on exact result.
 * Anyway:
 * - if your values are signed or unsigned integer of any size the LOWEST value is returned compared as <b>UNSIGNED</b>. For example for storage with int32_t as value, from the same key, from the set of values {-33, -1, 5573, 77753} 5573 is returned as it is the lowest by <b>UNSIGNED</b> comparison.
 * - if your values are anything else - they are compared byte by byte as if they are strings, it makes it especially complicated to predict the exact value for float or double templated storages. For strings if makes a bit more sence: if the choise is from "50" and "100" - "100" is returned, because the first byte of the "100" is lower than the first byte of the "50"
 *
 * \param[in] key      key of the record you look for
 * \param[out] value   the value from the storage
 * \param[in] txn      transaction ID, can be read only transaction
 *
 * \exception LMDBAL::NotFound  thrown if the storage doesn't have a record with the given key
 * \exception LMDBAL::Closed    thrown if the database was not opened
 * \exception LMDBAL::Unknown   thrown if something unexpected happend within lmdb
 */
template<class K, class V>
void LMDBAL::Storage<K, V>::getRecord(const K& key, V& value, TransactionID txn) const {
    ensureOpened(getRecordMethodName);

    MDB_val lmdbKey = keySerializer.setData(key);
    MDB_val lmdbData;

    int rc = mdb_get(txn, dbi, &lmdbKey, &lmdbData);
    if (rc != MDB_SUCCESS)
        throwNotFoundOrUnknown(rc, toString(key));

    valueSerializer.deserialize(lmdbData, value);
}

/**
 * \brief Chechs if storage has value
 *
 * \param[in] key      key of the record you look for
 * \returns true if there was a record with given key, false otherwise
 *
 * \exception LMDBAL::Closed    thrown if the database was not opened
 * \exception LMDBAL::Unknown   thrown if something unexpected happend within lmdb
 */
template<class K, class V>
bool LMDBAL::Storage<K, V>::checkRecord(const K& key) const {
    ensureOpened(checkRecordMethodName);

    TransactionID txn = beginReadOnlyTransaction();
    bool result;
    try {
        result = Storage<K, V>::checkRecord(key, txn);
    } catch (...) {
        abortTransaction(txn);
        throw;
    }

    abortTransaction(txn);
    return result;
}

/**
 * \brief Chechs if storage has value (transaction variant)
 *
 * You can obtain LMDBAL::TransactionID calling LMDBAL::Base::beginReadOnlyTransaction() or LMDBAL::Base::beginTransaction().
 * If you just want to read data you should prefer LMDBAL::Base::beginReadOnlyTransaction().
 *
 * \param[in] key      key of the record you look for
 * \param[in] txn      transaction ID, can be read only transaction
 * \returns true if there was a record with given key, false otherwise
 *
 * \exception LMDBAL::Closed    thrown if the database was not opened
 * \exception LMDBAL::Unknown   thrown if something unexpected happend within lmdb
 */
template<class K, class V>
bool LMDBAL::Storage<K, V>::checkRecord(const K& key, TransactionID txn) const {
    ensureOpened(checkRecordMethodName);

    MDB_val lmdbKey = keySerializer.setData(key);
    MDB_val lmdbData;

    int rc = mdb_get(txn, dbi, &lmdbKey, &lmdbData);
    if (rc == MDB_SUCCESS)
        return true;

    if (rc != MDB_NOTFOUND)
        throwUnknown(rc);

    return false;
}

/**
 * \brief Reads whole storage into a map
 *
 * Basically just reads all database in an std::map, usefull when you store small storages
 *
 * In case storage supports duplicates <b>only what lmdb considered to be lowest value</b>
 * (see LMDBAL::Storage::getRecord() description) is returned in the resulting map
 *
 * \exception LMDBAL::Closed    thrown if the database was not opened
 * \exception LMDBAL::Unknown   thrown if something unexpected happend within lmdb
 */
template<class K, class V>
std::map<K, V> LMDBAL::Storage<K, V>::readAll() const {
    ensureOpened(readAllMethodName);

    std::map<K, V> result;
    Storage<K, V>::readAll(result);
    return result;
}

/**
 * \brief Reads whole storage into a map (reference variant)
 *
 * Basically just reads all database in an std::map, usefull when you store small storages
 *
 * In case storage supports duplicates <b>only what lmdb considered to be lowest value</b>
 * (see LMDBAL::Storage::getRecord() description) is returned in the resulting map
 *
 * \param[out] result   a map that is going to contain all data
 *
 * \exception LMDBAL::Closed    thrown if the database was not opened
 * \exception LMDBAL::Unknown   thrown if something unexpected happend within lmdb
 */
template<class K, class V>
void LMDBAL::Storage<K, V>::readAll(std::map<K, V>& result) const {
    ensureOpened(readAllMethodName);

    TransactionID txn = beginReadOnlyTransaction();
    try {
        Storage<K, V>::readAll(result, txn);
    } catch (...) {
        abortTransaction(txn);
        throw;
    }

    abortTransaction(txn);
}

/**
 * \brief Reads whole storage into a map (transaction variant)
 *
 * Basically just reads all database in an std::map, usefull when you store small storages
 * You can obtain LMDBAL::TransactionID calling LMDBAL::Base::beginReadOnlyTransaction() or LMDBAL::Base::beginTransaction().
 * If you just want to read data you should prefer LMDBAL::Base::beginReadOnlyTransaction().
 *
 * In case storage supports duplicates <b>only what lmdb considered to be lowest value</b>
 * (see LMDBAL::Storage::getRecord() description) is returned in the resulting map
 *
 * \param[in] txn      transaction ID, can be read only transaction
 *
 * \exception LMDBAL::Closed    thrown if the database was not opened
 * \exception LMDBAL::Unknown   thrown if something unexpected happend within lmdb
 */
template<class K, class V>
std::map<K, V> LMDBAL::Storage<K, V>::readAll(TransactionID txn) const {
    ensureOpened(readAllMethodName);

    std::map<K, V> result;
    Storage<K, V>::readAll(result, txn);
    return result;
}

/**
 * \brief Reads whole storage into a map (transaction, reference variant)
 *
 * Basically just reads all database in an std::map, usefull when you store small storages
 * You can obtain LMDBAL::TransactionID calling LMDBAL::Base::beginReadOnlyTransaction() or LMDBAL::Base::beginTransaction().
 * If you just want to read data you should prefer LMDBAL::Base::beginReadOnlyTransaction().
 *
 * In case storage supports duplicates <b>only what lmdb considered to be lowest value</b>
 * (see LMDBAL::Storage::getRecord() description) is returned in the resulting map
 *
 * \param[out] result   a map that is going to contain all data
 * \param[in] txn      transaction ID, can be read only transaction
 *
 * \exception LMDBAL::Closed    thrown if the database was not opened
 * \exception LMDBAL::Unknown   thrown if something unexpected happend within lmdb
 */
template<class K, class V>
void LMDBAL::Storage<K, V>::readAll(std::map<K, V>& result, TransactionID txn) const {
    ensureOpened(readAllMethodName);

    MDB_cursor* cursor;
    MDB_val lmdbKey, lmdbData;

    int rc = mdb_cursor_open(txn, dbi, &cursor);
    if (rc != MDB_SUCCESS)
        throwUnknown(rc);

    rc = mdb_cursor_get(cursor, &lmdbKey, &lmdbData, MDB_FIRST);
    while (rc == MDB_SUCCESS) {
        K key;
        keySerializer.deserialize(lmdbKey, key);
        std::pair<typename std::map<K, V>::iterator, bool> probe = result.emplace(key, V{});
        if (probe.second)                                               //I do this to avoid overwrites in case duplicates are enabled
            valueSerializer.deserialize(lmdbData, probe.first->second);

        rc = mdb_cursor_get(cursor, &lmdbKey, &lmdbData, MDB_NEXT);
    }
    mdb_cursor_close(cursor);
    if (rc != MDB_NOTFOUND)
        throwUnknown(rc);
}

/**
 * \brief Replaces the content of the whole storage with the given
 *
 * Basically this method drops the database and adds all the records from the given map
 *
 * \param[in] data      new data of the storage
 *
 * \exception LMDBAL::Closed    thrown if the database was not opened
 * \exception LMDBAL::Unknown   thrown if something unexpected happend within lmdb
 */
template<class K, class V>
void LMDBAL::Storage<K, V>::replaceAll(const std::map<K, V>& data) {
    ensureOpened(replaceAllMethodName);

    TransactionID txn = beginTransaction();
    try {
        Storage<K, V>::replaceAll(data, txn);
    } catch (...) {
        abortTransaction(txn);
        throw;
    }

    commitTransaction(txn);
}

/**
 * \brief Replaces the content of the whole storage with the given (transaction variant)
 *
 * Basically this method drops the database and adds all the records from the given map
 * This method schedules a data replacement, but doesn't immidiately execute it.
 * You can obtain LMDBAL::TransactionID calling LMDBAL::Base::beginTransaction().
 *
 * \param[in] data      new data of the storage
 * \param[in] txn       transaction ID, needs to be a writable transaction!
 *
 * \exception LMDBAL::Closed    thrown if the database was not opened
 * \exception LMDBAL::Unknown   thrown if something unexpected happend within lmdb
 */
template<class K, class V>
void LMDBAL::Storage<K, V>::replaceAll(const std::map<K, V>& data, TransactionID txn) {
    ensureOpened(replaceAllMethodName);

    int rc = drop(txn);
    if (rc != MDB_SUCCESS)
        throwUnknown(rc);

    MDB_val lmdbKey, lmdbData;
    for (const std::pair<const K, V>& pair : data) {
        lmdbKey = keySerializer.setData(pair.first);
        lmdbData = valueSerializer.setData(pair.second);

        rc = mdb_put(txn, dbi, &lmdbKey, &lmdbData, MDB_NOOVERWRITE);   //TODO may be appending with cursor makes sence here?
        if (rc != MDB_SUCCESS)
            throwUnknown(rc);
    }
}

/**
 * \brief Adds records in bulk
 *
 * \param[in]       data        the data to be added
 * \param[in]       overwrite   if false method throws LMDBAL::Exist on repeated key, if true - overwrites it
 * \returns         new actual amount of records in the storage
 *
 * \exception LMDBAL::Closed    thrown if the database was not opened
 * \exception LMDBAL::Exist     thrown if overwrite==false and at least one of the keys of data already exists in the storage
 * \exception LMDBAL::Unknown   thrown if something unexpected happend within lmdb
 */
template<class K, class V>
uint32_t LMDBAL::Storage<K, V>::addRecords(const std::map<K, V>& data, bool overwrite) {
    ensureOpened(addRecordsMethodName);

    TransactionID txn = beginTransaction();
    uint32_t amount;
    try {
        amount = Storage<K, V>::addRecords(data, txn, overwrite);
    } catch (...) {
        abortTransaction(txn);
        throw;
    }

    commitTransaction(txn);
    return amount;
}

/**
 * \brief Adds records in bulk (transaction variant)
 *
 * This method schedules a data addition, but doesn't immidiately execute it.
 * You can obtain LMDBAL::TransactionID calling LMDBAL::Base::beginTransaction().
 *
 * \param[in]       data        the data to be added
 * \param[in]       txn         transaction ID, needs to be a writable transaction!
 * \param[in]       overwrite   if false method throws LMDBAL::Exist on repeated key, if true - overwrites it
 * \returns         new actual amount of records in the storage
 *
 * \exception LMDBAL::Closed    thrown if the database was not opened
 * \exception LMDBAL::Exist     thrown if overwrite==false and at least one of the keys of data already exists in the storage
 * \exception LMDBAL::Unknown   thrown if something unexpected happend within lmdb
 */
template<class K, class V>
uint32_t LMDBAL::Storage<K, V>::addRecords(const std::map<K, V>& data, TransactionID txn, bool overwrite) {
    ensureOpened(addRecordsMethodName);

    MDB_val lmdbKey, lmdbData;
    int rc;
    for (const std::pair<const K, V>& pair : data) {
        lmdbKey = keySerializer.setData(pair.first);
        lmdbData = valueSerializer.setData(pair.second);

        rc = mdb_put(txn, dbi, &lmdbKey, &lmdbData, overwrite ? 0 : MDB_NOOVERWRITE);
        if (rc == MDB_KEYEXIST)
            throwDuplicate(toString(pair.first));

        if (rc != MDB_SUCCESS)
            throwUnknown(rc);
    }

    MDB_stat stat;
    rc = mdb_stat(txn, dbi, &stat);
    if (rc != MDB_SUCCESS)
        throwUnknown(rc);

    return stat.ms_entries;
}

/**
 * \brief Removes one of the records
 *
 * Take a note that if the storage didn't have a record you want to remove LMDBAL::NotFound is thrown
 *
 * \param[in]       key     key of the record you wish to be removed
 *
 * \exception LMDBAL::Closed    thrown if the database was not opened
 * \exception LMDBAL::NotFound  thrown if the record with given key wasn't found
 * \exception LMDBAL::Unknown   thrown if something unexpected happend within lmdb
 */
template<class K, class V>
void LMDBAL::Storage<K, V>::removeRecord(const K& key) {
    ensureOpened(removeRecordMethodName);

    TransactionID txn = beginTransaction();
    try {
        Storage<K, V>::removeRecord(key, txn);
    } catch (...) {
        abortTransaction(txn);
        throw;
    }

    commitTransaction(txn);
}

/**
 * \brief Removes one of the records (transaction variant)
 *
 * Take a note that if the storage didn't have a record you want to remove LMDBAL::NotFound is thrown
 * This method schedules a record removal, but doesn't immidiately execute it.
 * You can obtain LMDBAL::TransactionID calling LMDBAL::Base::beginTransaction().
 *
 * \param[in]       key         key of the record you wish to be removed
 * \param[in]       txn         transaction ID, needs to be a writable transaction!
 *
 * \exception LMDBAL::Closed    thrown if the database was not opened
 * \exception LMDBAL::NotFound  thrown if the record with given key wasn't found
 * \exception LMDBAL::Unknown   thrown if something unexpected happend within lmdb
 */
template<class K, class V>
void LMDBAL::Storage<K, V>::removeRecord(const K& key, TransactionID txn) {
    ensureOpened(removeRecordMethodName);

    MDB_val lmdbKey = keySerializer.setData(key);
    int rc = mdb_del(txn, dbi, &lmdbKey, NULL);
    if (rc != MDB_SUCCESS)
        throwNotFoundOrUnknown(rc, toString(key));
}

/**
 * \brief A private virtual method I need to open each storage in the database
 *
 * \param[in] transaction   - lmdb transaction to call <a class="el" href="http://www.lmdb.tech/doc/group__mdb.html#gac08cad5b096925642ca359a6d6f0562a">mdb_dbi_open</a>
 * \returns MDB_SUCCESS if everything went smooth or MDB_<error> -like error code
 */
template<class K, class V>
int LMDBAL::Storage<K, V>::open(MDB_txn* transaction) {
    return makeStorage<K, V>(transaction, duplicates);
}

/**
 * \brief A private virtual method I need to close each storage in the database
 */
template<class K, class V>
void LMDBAL::Storage<K, V>::close() {
    for (Cursor<K, V>* cursor : cursors)
        cursor->terminated();

    iStorage::close();
}

/**
 * \brief Creates cursor
 *
 * \returns LMDBAL::Cursor for this storage and returs you a pointer to a created cursor
 */
template<class K, class V>
LMDBAL::Cursor<K, V>* LMDBAL::Storage<K, V>::createCursor() {
    Cursor<K, V>* cursor = new Cursor<K, V>(this);
    cursors.insert(cursor);

    return cursor;
}

/**
 * \brief Reads current storage flags it was opened with
 *
 * This method exists mostly for testing purposes
 *
 * \returns Third out parameter of <a href="http://www.lmdb.tech/doc/group__internal.html#ga95ba4cb721035478a8705e57b91ae4d4">mdb_dbi_flags</a> function
 *
 * \exception LMDBAL::Closed    thrown if the database was not opened
 * \exception LMDBAL::Unknown   if the result of <a href="http://www.lmdb.tech/doc/group__internal.html#ga95ba4cb721035478a8705e57b91ae4d4">mdb_dbi_flags</a> was not successfull
 */
template<class K, class V>
uint32_t LMDBAL::Storage<K, V>::flags() const {
    ensureOpened(flagsMethodName);
    uint32_t result;
    TransactionID txn = beginReadOnlyTransaction();

    int res = mdb_dbi_flags(txn, dbi, &result);
    abortTransaction(txn);
    if (res != MDB_SUCCESS)
        throwUnknown(res);

    return result;
}

/**
 * \brief Destroys cursor
 *
 * This a normal way to discard a cursor you don't need anymore
 *
 * \param[in] cursor    a pointer to a cursor you want to destroy
 *
 * \exception LMDBAL::Unknown  thrown if you try to destroy something that this storage didn't create
 */
template<class K, class V>
void LMDBAL::Storage<K, V>::destroyCursor(Cursor<K, V>* cursor) {
    typename std::set<Cursor<K, V>*>::const_iterator itr = cursors.find(cursor);
    if (itr == cursors.end())
        throwUnknown("An attempt to destroy a cursor the storage doesn't own");

    cursors.erase(itr);
    delete cursor;
}

/**
 * \brief A private virtual method that cursor calls when he reads a record, does nothing here but populates the LMDBAL::Cache
 *
 * \param[in]   key     a key of discovered record
 * \param[in]   value   a value of discovered record
 */
template<class K, class V>
void LMDBAL::Storage<K, V>::discoveredRecord(const K& key, const V& value) const {
    UNUSED(key);
    UNUSED(value);
}

/**
 * \brief A private virtual method that cursor calls when he reads a record, does nothing here but populates the LMDBAL::Cache
 *
 * \param[in]   key     a key of discovered record
 * \param[in]   value   a value of discovered record
 * \param[in]   txn     TransactionID under which the dicovery happened, to avoid not commited changes collisions
 */
template<class K, class V>
void LMDBAL::Storage<K, V>::discoveredRecord(const K& key, const V& value, TransactionID txn) const {
    UNUSED(key);
    UNUSED(value);
    UNUSED(txn);
}

/**
 * \brief A functiion to actually open <a class="el" href="http://www.lmdb.tech/doc/group__mdb.html#gadbe68a06c448dfb62da16443d251a78b">MDB_dbi</a> storage
 *
 * \tparam K type of keys in opening storage
 *
 * \param[in] transaction   - lmdb transaction to call <a class="el" href="http://www.lmdb.tech/doc/group__mdb.html#gac08cad5b096925642ca359a6d6f0562a">mdb_dbi_open</a>, must be a writable transaction!
 * \param[in] duplicates    - true if key duplicates are allowed (false by default)
 *
 * \returns MDB_SUCCESS if everything went smooth or MDB_<error> -like error code
 *
 * This is a way to optimise database using MDB_INTEGERKEY flag,
 * when the key is actually kind of an integer
 * This infrastructure also allowes us to customize mdb_dbi_open call in the future
 */
template<class K, class V>
inline int LMDBAL::iStorage::makeStorage(MDB_txn* transaction, bool duplicates) {
    unsigned int flags = MDB_CREATE;
    if constexpr (std::is_integral<K>::value)
        flags |= MDB_INTEGERKEY;

    if (duplicates) {
        flags |= MDB_DUPSORT;

        if constexpr (std::is_scalar<V>::value)
            flags |= MDB_DUPFIXED;

        if constexpr (
            std::is_same<V, uint32_t>::value ||
            std::is_same<V, int32_t>::value ||
            std::is_same<V, uint64_t>::value ||
            std::is_same<V, int64_t>::value
        )                                   //for some reason lmdb breaks if it's not one of these types in MDB_DUPFIXED mode
            flags |= MDB_INTEGERDUP;
    }

    return mdb_dbi_open(transaction, name.c_str(), flags, &dbi);
}

/**
 * \brief A method to cast a value (which can be a value or a key) to string.
 *
 * This function is mainly used in exceptions, to report which key was duplicated or not found.
 * You can define your own specializations to this function in case std::to_string doesn't cover your case
 *
 * \param[in]   value   a value that should be converted to string
 * \returns     a string presentation of value
 */
template<class T>
inline std::string LMDBAL::iStorage::toString(const T& value) {
    return std::to_string(value);
}

/**
 * \brief A method to cast a value (which can be a value or a key) to string.
 *
 * QString spectialization
 *
 * \param[in]   value   a value that should be converted to string
 * \returns     a string presentation of value
 */
template<>
inline std::string LMDBAL::iStorage::toString(const QString& value) {
    return value.toStdString();
}

/**
 * \brief A method to cast a value (which can be a value or a key) to string.
 *
 * std::string spectialization
 *
 * \param[in]   value   a value that should be converted to string
 * \returns     a string presentation of value
 */
template<>
inline std::string LMDBAL::iStorage::toString(const std::string& value) {
    return value;
}

#endif //LMDBAL_STORAGE_HPP