initial commit

lib/wDatabase/CMakeLists.txt

@@ -0,0 +1,25 @@
+cmake_minimum_required(VERSION 2.8.12)
+project(database)
+
+find_package(Qt5Core REQUIRED)
+
+set(CMAKE_INCLUDE_CURRENT_DIR ON)
+set(CMAKE_AUTOMOC ON)
+
+set(HEADERS
+    database.h
+    resourcecache.h
+)
+
+set(SOURCES
+    database.cpp
+    resourcecache.cpp
+)
+
+
+add_library(wDatabase STATIC ${HEADERS} ${SOURCES})
+
+target_link_libraries(wDatabase Qt5::Core)
+target_link_libraries(wDatabase lmdb)
+target_link_libraries(wDatabase wType)
+target_link_libraries(wDatabase wModel)

lib/wDatabase/database.cpp

@@ -0,0 +1,230 @@
+#include "database.h"
+
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#include <wType/bytearray.h>
+#include <algorithm>
+
+Database::Database(const W::String& dbName, QObject* parent):
+    M::ICatalogue(W::Address({dbName}), parent),
+    name(dbName),
+    opened(false),
+    environment(lmdb::env::create()),
+    dbi(0),
+    elements()
+{
+}
+
+Database::~Database()
+{
+}
+
+void Database::open()
+{
+    if (!opened) {
+        checkDirAndOpenEnvironment();
+        index();
+        opened = true;
+    }
+}
+
+void Database::addIndex(const W::String& fieldName, W::Object::objectType fieldType)
+{
+    ICatalogue::addIndex(fieldName, fieldType);
+    
+    if (opened) {
+        lmdb::txn rtxn = lmdb::txn::begin(environment, nullptr, MDB_RDONLY);
+        lmdb::cursor cursor = lmdb::cursor::open(rtxn, dbi);
+        lmdb::val key;
+        lmdb::val value;
+        while (cursor.get(key, value, MDB_NEXT)) {
+            uint64_t iKey = *((uint64_t*) key.data());
+            W::ByteArray ba(value.size());
+            ba.fill(value.data(), value.size());
+            W::Vocabulary* wVal = static_cast<W::Vocabulary*>(W::Object::fromByteArray(ba));
+            
+            IndexMap::const_iterator itr = indexes.find(fieldName);
+            itr->second->add(wVal->at(itr->first), iKey);
+            
+            delete wVal;
+        }
+        cursor.close();
+        rtxn.abort();
+    }
+}
+
+uint64_t Database::addElement(const W::Vocabulary& record)
+{
+    if (!opened) {
+        throw 6;    //TODO
+    }
+    uint64_t id = ICatalogue::addElement(record);
+    
+    elements.insert(id);
+    
+    int size = record.size();
+    W::ByteArray ba(size + 1);
+    ba.push8(record.getType());
+    record.serialize(ba);
+    
+    lmdb::val key((uint8_t*) &id, 8);
+    lmdb::val value(ba.getData(), ba.size());
+    lmdb::txn wTrans = lmdb::txn::begin(environment);
+    dbi.put(wTrans, key, value);
+    wTrans.commit();
+    
+    return id;
+}
+
+
+void Database::checkDirAndOpenEnvironment()
+{
+    int state1 = mkdir("./db", S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
+    if (state1 != 0 && errno != EEXIST) {
+        emit serviceMessage("Failed to create a root database folder");
+        throw 1;
+    }
+    
+    W::String path("./db/");
+    path += name;
+    
+    int state2 = mkdir(path.toString().c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
+    if (state2 != 0 && errno != EEXIST) {
+        emit serviceMessage(QString("Failed to create ") + name.toString().c_str() + " database folder");
+        throw 1;
+    }
+    
+    environment.set_mapsize(1UL * 1024UL * 1024UL * 1024UL);
+    environment.set_max_dbs(10);
+    environment.open(path.toString().c_str(), 0, 0664);
+    
+    lmdb::txn wTrans = lmdb::txn::begin(environment);
+    dbi = lmdb::dbi::open(wTrans, "main", MDB_CREATE | MDB_INTEGERKEY);
+    wTrans.commit();
+}
+
+void Database::index()
+{
+    lmdb::txn rtxn = lmdb::txn::begin(environment, nullptr, MDB_RDONLY);
+    lmdb::cursor cursor = lmdb::cursor::open(rtxn, dbi);
+    lmdb::val key;
+    lmdb::val value;
+    while (cursor.get(key, value, MDB_NEXT)) {
+        uint64_t iKey = *((uint64_t*) key.data());
+        W::ByteArray ba(value.size());
+        ba.fill(value.data(), value.size());
+        W::Vocabulary* wVal = static_cast<W::Vocabulary*>(W::Object::fromByteArray(ba));
+        ICatalogue::addElement(*wVal);
+        
+        elements.insert(iKey);
+        delete wVal;
+    }
+    cursor.close();
+    rtxn.abort();
+    
+    emit countChange(elements.size());
+}
+
+W::Vocabulary* Database::getElement(uint64_t id)
+{
+    lmdb::txn rtxn = lmdb::txn::begin(environment, nullptr, MDB_RDONLY);
+    lmdb::val key((uint8_t*) &id, 8);
+    lmdb::val value;
+    
+    if (dbi.get(rtxn, key, value)) {
+        W::ByteArray ba(value.size());
+        ba.fill(value.data(), value.size());
+        
+        W::Vocabulary* wVal = static_cast<W::Vocabulary*>(W::Object::fromByteArray(ba));
+        rtxn.abort();
+        
+        return wVal;
+    } else {
+        rtxn.abort();
+        throw 3;
+    }
+}
+
+std::set<uint64_t> Database::getAll() const
+{
+    return elements;
+}
+
+void Database::removeElement(uint64_t id)
+{
+    if (!opened) {
+        throw 6;    //TODO
+    }
+    ICatalogue::removeElement(id);
+    
+    lmdb::txn transaction = lmdb::txn::begin(environment);
+    lmdb::val key((uint8_t*) &id, 8);
+    dbi.del(transaction, key);
+    transaction.commit();
+    elements.erase(id);
+}
+
+void Database::clear()
+{
+    if (!opened) {
+        throw 6;    //TODO
+    }
+    M::ICatalogue::clear();
+    
+    lmdb::txn transaction = lmdb::txn::begin(environment);
+    dbi.drop(transaction);
+    transaction.commit();
+}
+
+void Database::addModel(M::Model* model)
+{
+    connect(model, SIGNAL(subscribersCountChange(uint64_t)), this, SLOT(onChildSubscribersCountChange(uint64_t)));
+    
+    M::ICatalogue::addModel(model);
+}
+
+void Database::removeModel(M::Model* model)
+{
+    disconnect(model, SIGNAL(subscribersCountChange(uint64_t)), this, SLOT(onChildSubscribersCountChange(uint64_t)));
+    
+    M::ICatalogue::removeModel(model);
+}
+
+
+void Database::onChildSubscribersCountChange(uint64_t count)
+{
+    if (count == 0) {
+        M::Model* model = static_cast<M::Model*>(sender());
+        
+        removeModel(model);
+        emit serviceMessage(QString("Unregistered model ") + model->getAddress().toString().c_str() + " because there no subscribers left");
+        
+        model->deleteLater();
+    }
+}
+
+void Database::modifyElement(uint64_t id, const W::Vocabulary& newValue)
+{
+    if (!opened) {
+        throw 6;    //TODO
+    }
+    int size = newValue.size();
+    W::ByteArray ba(size + 1);
+    
+    ba.push8(newValue.getType());
+    newValue.serialize(ba);
+    lmdb::val key((uint8_t*) &id, 8);
+    lmdb::val value(ba.getData(), ba.size());
+    lmdb::txn wTrans = lmdb::txn::begin(environment);
+    dbi.put(wTrans, key, value);
+    wTrans.commit();
+
+}
+
+uint64_t Database::size() const
+{
+    return elements.size();
+}
+
+

lib/wDatabase/database.h

@@ -0,0 +1,56 @@
+#ifndef DATABASE_H
+#define DATABASE_H
+
+#include "lmdb++.h"
+
+#include <map>
+#include <set>
+
+#include <wModel/icatalogue.h>
+
+#include <wType/string.h>
+#include <wType/address.h>
+
+class Database: public M::ICatalogue
+{
+    Q_OBJECT
+    class AbstractIndex;
+public:
+    Database(const W::String& dbName, QObject* parent = 0);
+    ~Database();
+    
+    void open();
+    
+    uint64_t addElement(const W::Vocabulary& record) override;
+    W::Vocabulary* getElement(uint64_t id) override;
+    void removeElement(uint64_t id) override;
+    void clear() override;
+    void modifyElement(uint64_t id, const W::Vocabulary & newValue) override;
+    uint64_t size() const override;
+    
+    void addIndex(const W::String& fieldName, W::Object::objectType fieldType) override;
+    
+    void addModel(M::Model* model);
+    void removeModel(M::Model* model);
+    
+protected:
+    std::set<uint64_t> getAll() const override;
+    
+private:
+    void checkDirAndOpenEnvironment();
+    void index();
+    
+private slots:
+    void onChildSubscribersCountChange(uint64_t count);
+    
+public:
+    const W::String name;
+    
+private:
+    bool opened;
+    lmdb::env environment;
+    lmdb::dbi dbi;
+    std::set<uint64_t> elements;
+};
+
+#endif // DATABASE_H

lib/wDatabase/resourcecache.cpp

@@ -0,0 +1,277 @@
+#include "resourcecache.h"
+
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#include <fstream>
+
+ResourceCache::ResourceCache(const W::String& dbName, QObject* parent):
+    M::Model(W::Address{dbName}, parent),
+    subscribeMember(W::Handler::create(W::Address({}), this, &ResourceCache::_h_subscribeMember)),
+    name(dbName),
+    opened(false),
+    environment(lmdb::env::create()),
+    dbi(0),
+    elements(),
+    lastIndex(0)
+{
+    W::Handler* get = W::Handler::create(address + W::Address({u"get"}), this, &ResourceCache::_h_get);
+    addHandler(get);
+}
+
+ResourceCache::~ResourceCache()
+{
+    delete subscribeMember;
+}
+
+void ResourceCache::open()
+{
+    if (!opened) {
+        checkDirAndOpenEnvironment();
+        
+        lmdb::txn rtxn = lmdb::txn::begin(environment, nullptr, MDB_RDONLY);
+        lmdb::cursor cursor = lmdb::cursor::open(rtxn, dbi);
+        lmdb::val key;
+        lmdb::val value;
+        while (cursor.get(key, value, MDB_NEXT)) {
+            uint64_t iKey = *((uint64_t*) key.data());
+            
+            elements.insert(iKey);
+            if (iKey > lastIndex) {
+                lastIndex = iKey;
+            }
+        }
+        cursor.close();
+        rtxn.abort();
+        
+        opened = true;
+        emit countChange(elements.size());
+    }
+}
+
+void ResourceCache::checkDirAndOpenEnvironment()
+{
+    int state1 = mkdir("./dbMedia", S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
+    if (state1 != 0 && errno != EEXIST) {
+        emit serviceMessage("Failed to create a root database folder");
+        throw 1;
+    }
+    
+    W::String path("./dbMedia/");
+    path += name;
+    
+    int state2 = mkdir(path.toString().c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
+    if (state2 != 0 && errno != EEXIST) {
+        emit serviceMessage(QString("Failed to create ") + name.toString().c_str() + " database folder");
+        throw 1;
+    }
+    
+    environment.set_mapsize(1UL * 1024UL * 1024UL * 1024UL);
+    environment.set_max_dbs(10);
+    environment.open(path.toString().c_str(), 0, 0664);
+    
+    lmdb::txn wTrans = lmdb::txn::begin(environment);
+    dbi = lmdb::dbi::open(wTrans, "main", MDB_CREATE | MDB_INTEGERKEY);
+    wTrans.commit();
+}
+
+uint64_t ResourceCache::addResource(const W::String& path)
+{
+    uint64_t id = ++lastIndex;
+    
+    elements.insert(id);
+    
+    W::ByteArray ba(path.size() + 1);
+    ba.push8(path.getType());
+    path.serialize(ba);
+    
+    lmdb::val key((uint8_t*) &id, 8);
+    lmdb::val value(ba.getData(), ba.size());
+    lmdb::txn wTrans = lmdb::txn::begin(environment);
+    dbi.put(wTrans, key, value);
+    wTrans.commit();
+    
+    W::Vocabulary* vc = new W::Vocabulary();
+    vc->insert(u"id", new W::Uint64(id));
+    
+    broadcast(vc, W::Address({u"addElement"}));
+    emit countChange(elements.size());
+    
+    return id;
+}
+
+void ResourceCache::removeResource(uint64_t id)
+{
+    if (!opened) {
+        throw ClosedDB("removeResource");
+    }
+    
+    lmdb::txn transaction = lmdb::txn::begin(environment);
+    lmdb::val key((uint8_t*) &id, 8);
+    dbi.del(transaction, key);
+    transaction.commit();
+    elements.erase(id);
+    
+    W::Vocabulary* vc = new W::Vocabulary();
+    vc->insert(u"id", new W::Uint64(id));
+    
+    broadcast(vc, W::Address({u"removeElement"}));
+    emit countChange(elements.size());
+    
+    //TODO not sure, may be it's better to also destroy resource model?
+}
+
+void ResourceCache::clear()
+{
+    if (!opened) {
+        throw new ClosedDB("clear");
+    }
+    
+    lmdb::txn transaction = lmdb::txn::begin(environment);
+    dbi.drop(transaction);
+    transaction.commit();
+    
+    elements.clear();
+    lastIndex = 0;
+    
+    W::Vocabulary* vc = new W::Vocabulary();
+    
+    broadcast(vc, W::Address({u"clear"}));
+    emit countChange(elements.size());
+}
+
+void ResourceCache::h_get(const W::Event& ev)
+{
+    W::Vector* order = new W::Vector();
+    std::set<uint64_t>::const_iterator itr = elements.begin();
+    std::set<uint64_t>::const_iterator end = elements.end();
+    
+    for (; itr != end; ++itr) {
+        order->push(W::Uint64(*itr));
+    }
+    
+    W::Vocabulary* rvc = new W::Vocabulary;
+    rvc->insert(u"data", order);
+    
+    response(rvc, W::Address({u"get"}), ev);
+}
+
+
+M::Model::ModelType ResourceCache::getType() const
+{
+    return type;
+}
+
+void ResourceCache::set(W::Object* value)
+{
+    set(*value);
+}
+
+void ResourceCache::set(const W::Object& value)
+{
+    throw 14;   //what do you expect here? not implemented, and not sure it ever would be
+}
+
+void ResourceCache::h_subscribe(const W::Event& ev)
+{
+    M::Model::h_subscribe(ev);
+    
+    h_get(ev);
+}
+
+W::String * ResourceCache::getElement(uint64_t id) const
+{
+    lmdb::txn rtxn = lmdb::txn::begin(environment, nullptr, MDB_RDONLY);
+    lmdb::val key((uint8_t*) &id, 8);
+    lmdb::val value;
+    
+    if (lmdb::dbi_get(rtxn, dbi.handle(), key, value)) {
+        W::ByteArray ba(value.size());
+        ba.fill(value.data(), value.size());
+        
+        W::String* wVal = static_cast<W::String*>(W::Object::fromByteArray(ba));
+        rtxn.abort();
+        
+        return wVal;
+    } else {
+        rtxn.abort();
+        throw 3;
+    }
+}
+
+
+void ResourceCache::h_subscribeMember(const W::Event& ev)
+{
+    const W::Address& addr = ev.getDestination();
+    W::Address lastHops = addr << address.length();
+    
+    if (lastHops.length() == 2 && (lastHops.ends(W::Address{u"subscribe"}) 
+        || lastHops.ends(W::Address{u"get"}) 
+        || lastHops.ends(W::Address{u"getAdditional"}))
+    ) {
+        W::String* record;
+        try {
+            record = getElement(lastHops.front().toUint64());
+            M::File* modelRecord = M::File::create(readFile(*record), address + lastHops >> 1);
+            delete record;
+            addModel(modelRecord);
+            passToHandler(ev);
+        } catch (int err) {
+            if (err == 3) {
+                emit serviceMessage(QString("An attempt to create and subscribe record model in resourcecache, but it is not found. Event: ") + ev.toString().c_str());
+            } else if (err == 10) {
+                serviceMessage(QString("Can't open file ") + record->toString().c_str());
+                delete record;
+            } else {
+                throw err;
+            }
+        } catch (const std::invalid_argument& err) {
+            emit serviceMessage(QString("Strange event in custom handler of resourcecache ") + ev.toString().c_str());
+        }
+    } else {
+        emit serviceMessage(QString("Strange event in custom handler of resourcecache ") + ev.toString().c_str());
+    }
+}
+
+
+W::Blob * ResourceCache::readFile(const W::String& path) const
+{
+    std::ifstream file (path.toString(), std::ios::in|std::ios::binary|std::ios::ate);
+    
+    if (file.is_open()) {
+        char * memblock;
+        uint32_t size;
+        size = file.tellg();
+        file.seekg(0, std::ios::beg);
+        memblock = new char[size];
+        file.read(memblock, size);
+        file.close();
+        
+        return new W::Blob(size, memblock);
+    } else {
+        throw 10;   //TODO
+    }
+}
+
+std::set<uint64_t> ResourceCache::getAllIdentificators() const
+{
+    if (!opened) {
+        throw new ClosedDB("getAllIdentificators");
+    }
+    
+    return elements;
+}
+
+W::String ResourceCache::getPath(uint64_t id) const
+{
+    return *(getElement(id));
+}
+
+bool ResourceCache::has(uint64_t id) const
+{
+    if (!opened) {
+        throw new ClosedDB("has");
+    }
+    
+    return elements.find(id) != elements.end();
+}

lib/wDatabase/resourcecache.h

@@ -0,0 +1,84 @@
+#ifndef RESOURCECACHE_H
+#define RESOURCECACHE_H
+
+#include <set>
+#include <map>
+
+#include "lmdb++.h"
+
+#include <wModel/model.h>
+#include <wModel/file/file.h>
+#include <wType/address.h>
+#include <wType/string.h>
+#include <wType/bytearray.h>
+#include <wType/uint64.h>
+#include <wType/blob.h>
+#include <utils/exception.h>
+
+
+class ResourceCache : public M::Model
+{
+    Q_OBJECT
+public:
+    ResourceCache(const W::String& dbName, QObject* parent = 0);
+    ~ResourceCache();
+    
+    void open();
+    uint64_t addResource(const W::String& path);
+    void removeResource(uint64_t id);
+    void clear();
+    bool has(uint64_t id) const;
+    W::String getPath(uint64_t id) const;
+    std::set<uint64_t> getAllIdentificators() const;
+    
+    W::Handler* subscribeMember;
+    handler(subscribeMember);
+    
+    M::Model::ModelType getType() const override;
+    static const M::Model::ModelType type = resourceCache;
+    void set(const W::Object & value) override;
+    void set(W::Object * value) override;
+
+signals:
+    void countChange(uint64_t count);
+    
+protected:
+    void h_subscribe(const W::Event & ev) override;
+    
+    handler(get);
+    
+    W::String* getElement(uint64_t id) const;
+    
+public:
+    const W::String name;
+    
+private:
+    bool opened;
+    lmdb::env environment;
+    lmdb::dbi dbi;
+    std::set<uint64_t> elements;
+    uint64_t lastIndex;
+    
+    void checkDirAndOpenEnvironment();
+    W::Blob* readFile(const W::String& path) const;
+    
+    class ClosedDB: 
+    public Utils::Exception
+    {
+    public:
+        ClosedDB(const std::string& p_op):Exception(), operation(p_op){}
+        
+        std::string getMessage() const {
+            std::string str = "An attempt to perform method ResourceCache::";
+            str += operation;
+            str += " but the database is not open";
+            return str;
+            
+        }
+        
+    private:
+        std::string operation;
+    };
+};
+
+#endif // RESOURCECACHE_H