diff --git a/src/collection.cpp b/src/collection.cpp index 2e339f6..5a015c5 100644 --- a/src/collection.cpp +++ b/src/collection.cpp @@ -5,6 +5,7 @@ constexpr std::string_view downloads("downloads"); static const std::regex remote("(^https?):\\/\\/"); Collection::Collection( + const std::filesystem::path& location, const std::filesystem::path& destination, const std::string& target, const std::shared_ptr& logger, @@ -16,8 +17,11 @@ Collection::Collection( destination(destination), target(target), taskManager(taskManager), + root(std::make_unique(location, shared_from_this(), logger)), + librariesPath(), knownSources(), successSources(), + pendingSources(), downloadingSources(), readingSources(), buildingSources() @@ -29,16 +33,52 @@ Collection::Collection( throw e; } + + root->read(); + if (root->successfullyRead()) + throw std::runtime_error("Error reading project in " + location.string()); + + if (root->getType() != Component::mason) + throw std::runtime_error("Project in " + location.string() + " doesn't seem to be a valid mason project"); + + librariesPath = root->getLibrariesPath(); + std::lock_guard lock(mutex); + while (pendingSources.size() > 0) { + std::string source = pendingSources.front(); + pendingSources.pop(); + processSource(source); + } + + taskManager->queue( + std::bind(&Component::build, root.get(), destination, target), + std::bind(&Collection::rootBuilt, this, std::placeholders::_1) + ); } void Collection::addSource(const std::string& source) { std::lock_guard lock(mutex); - if (_hasSource(source)) { + std::pair result = knownSources.emplace(source); + if (!result.second) { major("Source " + source + " is already present, skipping as duplicate"); return; } debug("Adding source " + source); + switch (root->getState()) { + case Component::initial: + case Component::reading: + pendingSources.emplace(source); + break; + case Component::ready: + case Component::building: + case Component::done: + processSource(source); + case Component::error: + throw std::runtime_error("Project in " + root->getLocation().string() + " doesn't seem to be a valid mason project"); + } +} + +void Collection::processSource(const std::string& source) { if (isRemote(source)) queueDownload(source); else @@ -62,7 +102,6 @@ void Collection::sourceDownaloded(const std::string& source, TaskManager::Error void Collection::sourceRead(const std::string& source, TaskManager::Error err) { std::lock_guard lock(mutex); - debug("Source " + source + " has been read"); Components::iterator itr = readingSources.find(source); if (err.has_value()) { error("Coundn't read " + source + ": " + err.value()->what()); @@ -70,12 +109,18 @@ void Collection::sourceRead(const std::string& source, TaskManager::Error err) { return; } - debug("Queuing build of " + source); + if (itr->second->successfullyRead()) { + error("Source " + source + " had an error during read process, canceling build of this component"); + readingSources.erase(itr); + return; + } + + debug("Source " + source + " has been read, scheduling build"); std::pair res = buildingSources.emplace(source, std::move(itr->second)); readingSources.erase(itr); taskManager->queue( - std::bind(&Component::build, res.first->second.get(), destination, target), + std::bind(&Component::build, res.first->second.get(), destination/librariesPath, target), std::bind(&Collection::sourceBuilt, this, source, std::placeholders::_1) ); } @@ -93,12 +138,16 @@ void Collection::sourceBuilt(const std::string& source, TaskManager::Error err) successSources.emplace(source); } +void Collection::rootBuilt(TaskManager::Error err) { + +} + bool Collection::isRemote(const std::string& source) { return std::regex_search(source, remote); } void Collection::queueDownload(const std::string& source) { - debug("Queuing download of " + source); + debug("Scheduling download of " + source); std::pair res = downloadingSources.emplace(std::piecewise_construct, std::forward_as_tuple(source), std::forward_as_tuple( @@ -113,7 +162,7 @@ void Collection::queueDownload(const std::string& source) { } void Collection::queueRead(const std::string& source, const std::filesystem::path& location) { - debug("Queuing read of " + source); + debug("Scheduling read of " + source); std::pair res = readingSources.emplace(std::piecewise_construct, std::forward_as_tuple(source), std::forward_as_tuple( @@ -132,6 +181,42 @@ bool Collection::hasSource(const std::string& source) const { return _hasSource(source); } +unsigned int Collection::sourcesTotal() const { + std::lock_guard lock(mutex); + return _sourcesTotal(); +} + +unsigned int Collection::sourcesSuccess() const { + std::lock_guard lock(mutex); + return _sourcesSuccess(); +} + +unsigned int Collection::sourcesPending() const { + std::lock_guard lock(mutex); + return _sourcesPending(); +} + +unsigned int Collection::sourcesError() const { + std::lock_guard lock(mutex); + return _sourcesError(); +} + bool Collection::_hasSource(const std::string& source) const { return knownSources.count(source) > 0; } + +unsigned int Collection::_sourcesTotal() const { + return knownSources.size(); +} + +unsigned int Collection::_sourcesSuccess() const { + return successSources.size(); +} + +unsigned int Collection::_sourcesPending() const { + return downloadingSources.size() + readingSources.size() + buildingSources.size() + pendingSources.size(); +} + +unsigned int Collection::_sourcesError() const { + return _sourcesTotal() - _sourcesPending() - _sourcesSuccess(); +} diff --git a/src/collection.h b/src/collection.h index 7aaaa44..0d555c7 100644 --- a/src/collection.h +++ b/src/collection.h @@ -8,6 +8,7 @@ #include #include #include +#include #include "loggable.h" #include "component.h" @@ -16,11 +17,13 @@ class Collection : protected Loggable, public std::enable_shared_from_this { using Sources = std::set; + using SourcesQueue = std::queue; using Downloads = std::map>; using Components = std::map>; public: Collection( + const std::filesystem::path& location, const std::filesystem::path& destination, const std::string& target, const std::shared_ptr& logger, @@ -39,21 +42,30 @@ private: void sourceDownaloded(const std::string& source, TaskManager::Error err); void sourceRead(const std::string& source, TaskManager::Error err); void sourceBuilt(const std::string& source, TaskManager::Error err); + void rootBuilt(TaskManager::Error err); static bool isRemote(const std::string& source); + void processSource(const std::string& source); void queueDownload(const std::string& source); void queueRead(const std::string& source, const std::filesystem::path& location); bool _hasSource(const std::string& source) const; + unsigned int _sourcesTotal() const; + unsigned int _sourcesPending() const; + unsigned int _sourcesError() const; + unsigned int _sourcesSuccess() const; private: mutable std::mutex mutex; std::filesystem::path destination; std::string target; std::shared_ptr taskManager; + std::unique_ptr root; + std::string librariesPath; Sources knownSources; Sources successSources; + SourcesQueue pendingSources; Downloads downloadingSources; Components readingSources; Components buildingSources; diff --git a/src/component.cpp b/src/component.cpp index bc337be..9f624dc 100644 --- a/src/component.cpp +++ b/src/component.cpp @@ -13,6 +13,13 @@ constexpr std::array stringStates { "done" }; +constexpr std::array stringTypes { + "file", + "directory", + "mason", + "unknown" +}; + Component::Component( const std::filesystem::path& path, const std::weak_ptr& collection, @@ -23,7 +30,7 @@ Component::Component( type(unknown), collection(collection), location(path), - scenario(std::nullopt) + manifest(std::nullopt) {} void Component::read() { @@ -56,65 +63,73 @@ void Component::tryReadingBuildScenarios() { } bool Component::readAsMason() { - std::filesystem::path masonScenarion = location/masonJSON; + std::filesystem::path masonManifest = location/masonJSON; try { - std::ifstream file(masonScenarion); + std::ifstream file(masonManifest); if (file.is_open()) - scenario = nlohmann::json::parse(file); + manifest = nlohmann::json::parse(file); else - minor("Couldn't open " + masonScenarion.string()); + minor("Couldn't open " + masonManifest.string()); } catch (const nlohmann::json::exception& e) { - major("Couldn't parse " + masonScenarion.string()); + major("Couldn't parse " + masonManifest.string()); } - if (scenario.has_value()) { - const nlohmann::json& sc = scenario.value(); - if (!sc.is_object()) - return errorScenario(masonScenarion.string() + " is unexpected root type"); + if (manifest.has_value()) { + const nlohmann::json& mnfst = manifest.value(); + if (!mnfst.is_object()) + return errorScenario(masonManifest.string() + " is unexpected root type"); - nlohmann::json::const_iterator itr = sc.find(dependencies); - if (itr == sc.end()) { - info(masonScenarion.string() + " doesn't have dependencies"); + nlohmann::json::const_iterator itr = mnfst.find(dependencies); + if (itr == mnfst.end()) { + info(masonManifest.string() + " doesn't have dependencies"); } else { const nlohmann::json& deps = *itr; - if (!deps.is_array()) - return errorScenario(masonScenarion.string() + " dependencies are not organized as an array"); - - for (const nlohmann::json& dep : deps) { - std::shared_ptr col = collection.lock(); - switch (dep.type()) { - case nlohmann::json::value_t::string: - col->addSource(dep); - break; - case nlohmann::json::value_t::object: { - nlohmann::json::const_iterator ditr = dep.find("path"); - if (ditr == dep.end()) - return errorScenario(masonScenarion.string() + " object of unexpected format in dependecies"); - - nlohmann::json path = *ditr; - if (!path.is_string()) - return errorScenario(masonScenarion.string() + " object of unexpected format in dependecies"); - - col->addSource(path); - } - break; - default: - return errorScenario(masonScenarion.string() + " has unexpected entries in dependecies"); - } - } + if (!readMasonDependencies(deps, masonManifest)) + return false; } type = mason; - info(location.string() + " seems to be a mason project"); + info(location.string() + " seems to be a valid mason project"); return true; } return false; } +bool Component::readMasonDependencies(const nlohmann::json& deps, const std::string& manifestPath) { + if (!deps.is_array()) + return errorScenario(manifestPath + " dependencies are not organized as an array"); + + for (const nlohmann::json& dep : deps) { + std::shared_ptr col = collection.lock(); + switch (dep.type()) { + case nlohmann::json::value_t::string: + col->addSource(dep); + break; + case nlohmann::json::value_t::object: { + nlohmann::json::const_iterator ditr = dep.find("path"); + if (ditr == dep.end()) + return errorScenario(manifestPath + " object of unexpected format in dependecies"); + + nlohmann::json path = *ditr; + if (!path.is_string()) + return errorScenario(manifestPath + " object of unexpected format in dependecies"); + + col->addSource(path); + } + break; + default: + return errorScenario(manifestPath + " has unexpected entries in dependecies"); + } + } + + return true; +} + + bool Component::errorScenario(const std::string& message) { major(message); - scenario = std::nullopt; + manifest = std::nullopt; return false; return false; } @@ -124,11 +139,67 @@ void Component::build(const std::filesystem::path& destination, const std::strin throw WrongState(state, "build"); state = building; - info("Building " + location.string() + " to " + destination.string()); + + switch (type) { + case file: + buildAsFile(destination); + break; + case directory: + buildAsDiractory(destination); + break; + case mason: + buildAsMason(destination, target); + break; + default: + break; + } + state = done; } +void Component::buildAsFile(const std::filesystem::path& destination) { + std::filesystem::copy(location, destination/location.filename()); +} + +void Component::buildAsDiractory(const std::filesystem::path& destination) { + std::filesystem::copy(location, destination/location.filename(), + std::filesystem::copy_options::recursive | + std::filesystem::copy_options::overwrite_existing + ); +} + +std::string Component::getLibrariesPath() const { + //TODO may be it makes sence to cache + if (type != mason) + throw WrongType(type, "getLibrariesPath"); + + const nlohmann::json& mnfst = manifest.value(); + nlohmann::json::const_iterator itr = mnfst.find("libraries"); + if (itr == mnfst.end()) + return ""; + + const nlohmann::json& libs = *itr; + if (libs.is_string()) + return libs; + + warn("Mason manifest " + (location/masonJSON).string() + + " has field \"libraries\" which is not string, ignoring it"); + return ""; +} + +std::filesystem::path Component::getLocation() const { + return location; +} + +bool Component::successfullyRead() const { + return state > reading && state != error; +} + +void Component::buildAsMason(const std::filesystem::path& destination, const std::string& target) { + warn("mason build is not implemented yet"); +} + Component::State Component::getState() const { return state; } @@ -139,5 +210,10 @@ Component::Type Component::getType() const { Component::WrongState::WrongState(State state, const std::string& action): std::runtime_error("An attempt to perform action \"" + action - + "\" on a wrong state \"" + stringStates[state].data() + '\"') + + "\" on a wrong component state \"" + stringStates[state].data() + '\"') +{} + +Component::WrongType::WrongType(Type type, const std::string& action): + std::runtime_error("An attempt to perform an action \"" + action + + "\" on a wrong component type \"" + stringTypes[type].data() + '\"') {} diff --git a/src/component.h b/src/component.h index 2554e72..736d294 100644 --- a/src/component.h +++ b/src/component.h @@ -19,6 +19,7 @@ class Collection; class Component : protected Loggable { public: class WrongState; + class WrongType; enum State { initial, reading, @@ -29,10 +30,10 @@ public: }; enum Type { - unknown, file, directory, - mason + mason, + unknown }; Component( @@ -43,6 +44,9 @@ public: Type getType() const; State getState() const; + std::filesystem::path getLocation() const; + std::string getLibrariesPath() const; + bool successfullyRead() const; void read(); void build(const std::filesystem::path& destination, const std::string& target); @@ -52,15 +56,26 @@ private: bool readAsMason(); bool errorScenario(const std::string& message); + void buildAsFile(const std::filesystem::path& destination); + void buildAsDiractory(const std::filesystem::path& destination); + void buildAsMason(const std::filesystem::path& destination, const std::string& target); + + bool readMasonDependencies(const nlohmann::json& deps, const std::string& manifestPath); + private: State state; Type type; std::weak_ptr collection; std::filesystem::path location; - std::optional scenario; + std::optional manifest; }; class Component::WrongState : public std::runtime_error { public: explicit WrongState(State state, const std::string& action); }; + +class Component::WrongType : public std::runtime_error { +public: + explicit WrongType(Type type, const std::string& action); +}; diff --git a/src/download.cpp b/src/download.cpp index 8b61c39..feeb69e 100644 --- a/src/download.cpp +++ b/src/download.cpp @@ -71,18 +71,16 @@ void Download::proceed() { const FileInfo& fl = file.value(); std::optional path = downloadAsFile(fl); if (path.has_value()) { - bool success = true; info("Successfully downloaded " + url); if (fl.archive) { info(url + " appears to be an archive, extracting"); - success = extract(path.value(), destination/fl.name); - if (success) + bool success = extract(path.value(), destination/fl.name); + if (success) { info("Successfully extracted " + url); - } - - if (success) { - location = destination/fl.name; - return; + location = destination/fl.name; + } + } else { + location = path; } } diff --git a/src/main.cpp b/src/main.cpp index 5a3f0e4..c15074d 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -20,25 +20,31 @@ int main(int argc, char *argv[]) { std::shared_ptr logger = std::make_shared(Logger::Severity::debug); std::shared_ptr taskManager = std::make_shared(logger); - std::shared_ptr collection = std::make_shared(secondArg, "", logger, taskManager); + std::shared_ptr collection = std::make_shared( + firstArg, + secondArg, + "", + logger, + taskManager + ); taskManager->start(); - collection->addSource(firstArg); - taskManager->wait(); taskManager->stop(); - // int result = -1; - // if (success) { - // root.info("successfully parsed project " + root.getName()); - // root.info("dependencies count is " + std::to_string(root.dependenciesCount())); - // - // root.discover(); - // result = 0; - // } - // - // root.printLog(); + unsigned int success = collection->sourcesSuccess(); + unsigned int total = collection->sourcesTotal(); + if (total == success) { + logger->log(Logger::Severity::info, "Successfully built " + firstArg); + return 0; + } else { + logger->log(Logger::Severity::error, "Failed to build " + firstArg); + logger->log(Logger::Severity::info, "Success: " + std::to_string(success)); + logger->log(Logger::Severity::info, "Failed: " + std::to_string(collection->sourcesError())); + logger->log(Logger::Severity::info, "Unfinished: " + std::to_string(collection->sourcesPending())); + logger->log(Logger::Severity::info, "Total: " + std::to_string(total)); + return -1; + } - - return 0; + return -2; } diff --git a/test/mason.json b/test/mason.json index 2340e64..a4415fd 100644 --- a/test/mason.json +++ b/test/mason.json @@ -10,5 +10,6 @@ }, "https://git.macaw.me/blue/mason/archive/main.tar.gz", "https://git.macaw.me/blue/mason/raw/branch/main/src2/atomicmutex.h" - ] + ], + "libraries": "lib" }