libraries directory
This commit is contained in:
parent
2b913d1d42
commit
0fa8ba6918
@ -5,6 +5,7 @@ constexpr std::string_view downloads("downloads");
|
||||
static const std::regex remote("(^https?):\\/\\/");
|
||||
|
||||
Collection::Collection(
|
||||
const std::filesystem::path& location,
|
||||
const std::filesystem::path& destination,
|
||||
const std::string& target,
|
||||
const std::shared_ptr<Logger>& logger,
|
||||
@ -16,8 +17,11 @@ Collection::Collection(
|
||||
destination(destination),
|
||||
target(target),
|
||||
taskManager(taskManager),
|
||||
root(std::make_unique<Component>(location, shared_from_this(), logger)),
|
||||
librariesPath(),
|
||||
knownSources(),
|
||||
successSources(),
|
||||
pendingSources(),
|
||||
downloadingSources(),
|
||||
readingSources(),
|
||||
buildingSources()
|
||||
@ -29,16 +33,52 @@ Collection::Collection(
|
||||
|
||||
throw e;
|
||||
}
|
||||
|
||||
root->read();
|
||||
if (root->successfullyRead())
|
||||
throw std::runtime_error("Error reading project in " + location.string());
|
||||
|
||||
if (root->getType() != Component::mason)
|
||||
throw std::runtime_error("Project in " + location.string() + " doesn't seem to be a valid mason project");
|
||||
|
||||
librariesPath = root->getLibrariesPath();
|
||||
std::lock_guard lock(mutex);
|
||||
while (pendingSources.size() > 0) {
|
||||
std::string source = pendingSources.front();
|
||||
pendingSources.pop();
|
||||
processSource(source);
|
||||
}
|
||||
|
||||
taskManager->queue(
|
||||
std::bind(&Component::build, root.get(), destination, target),
|
||||
std::bind(&Collection::rootBuilt, this, std::placeholders::_1)
|
||||
);
|
||||
}
|
||||
|
||||
void Collection::addSource(const std::string& source) {
|
||||
std::lock_guard lock(mutex);
|
||||
if (_hasSource(source)) {
|
||||
std::pair<Sources::const_iterator, bool> result = knownSources.emplace(source);
|
||||
if (!result.second) {
|
||||
major("Source " + source + " is already present, skipping as duplicate");
|
||||
return;
|
||||
}
|
||||
|
||||
debug("Adding source " + source);
|
||||
switch (root->getState()) {
|
||||
case Component::initial:
|
||||
case Component::reading:
|
||||
pendingSources.emplace(source);
|
||||
break;
|
||||
case Component::ready:
|
||||
case Component::building:
|
||||
case Component::done:
|
||||
processSource(source);
|
||||
case Component::error:
|
||||
throw std::runtime_error("Project in " + root->getLocation().string() + " doesn't seem to be a valid mason project");
|
||||
}
|
||||
}
|
||||
|
||||
void Collection::processSource(const std::string& source) {
|
||||
if (isRemote(source))
|
||||
queueDownload(source);
|
||||
else
|
||||
@ -62,7 +102,6 @@ void Collection::sourceDownaloded(const std::string& source, TaskManager::Error
|
||||
void Collection::sourceRead(const std::string& source, TaskManager::Error err) {
|
||||
std::lock_guard lock(mutex);
|
||||
|
||||
debug("Source " + source + " has been read");
|
||||
Components::iterator itr = readingSources.find(source);
|
||||
if (err.has_value()) {
|
||||
error("Coundn't read " + source + ": " + err.value()->what());
|
||||
@ -70,12 +109,18 @@ void Collection::sourceRead(const std::string& source, TaskManager::Error err) {
|
||||
return;
|
||||
}
|
||||
|
||||
debug("Queuing build of " + source);
|
||||
if (itr->second->successfullyRead()) {
|
||||
error("Source " + source + " had an error during read process, canceling build of this component");
|
||||
readingSources.erase(itr);
|
||||
return;
|
||||
}
|
||||
|
||||
debug("Source " + source + " has been read, scheduling build");
|
||||
std::pair<Components::iterator, bool> res = buildingSources.emplace(source, std::move(itr->second));
|
||||
readingSources.erase(itr);
|
||||
|
||||
taskManager->queue(
|
||||
std::bind(&Component::build, res.first->second.get(), destination, target),
|
||||
std::bind(&Component::build, res.first->second.get(), destination/librariesPath, target),
|
||||
std::bind(&Collection::sourceBuilt, this, source, std::placeholders::_1)
|
||||
);
|
||||
}
|
||||
@ -93,12 +138,16 @@ void Collection::sourceBuilt(const std::string& source, TaskManager::Error err)
|
||||
successSources.emplace(source);
|
||||
}
|
||||
|
||||
void Collection::rootBuilt(TaskManager::Error err) {
|
||||
|
||||
}
|
||||
|
||||
bool Collection::isRemote(const std::string& source) {
|
||||
return std::regex_search(source, remote);
|
||||
}
|
||||
|
||||
void Collection::queueDownload(const std::string& source) {
|
||||
debug("Queuing download of " + source);
|
||||
debug("Scheduling download of " + source);
|
||||
std::pair<Downloads::iterator, bool> res = downloadingSources.emplace(std::piecewise_construct,
|
||||
std::forward_as_tuple(source),
|
||||
std::forward_as_tuple(
|
||||
@ -113,7 +162,7 @@ void Collection::queueDownload(const std::string& source) {
|
||||
}
|
||||
|
||||
void Collection::queueRead(const std::string& source, const std::filesystem::path& location) {
|
||||
debug("Queuing read of " + source);
|
||||
debug("Scheduling read of " + source);
|
||||
std::pair<Components::iterator, bool> res = readingSources.emplace(std::piecewise_construct,
|
||||
std::forward_as_tuple(source),
|
||||
std::forward_as_tuple(
|
||||
@ -132,6 +181,42 @@ bool Collection::hasSource(const std::string& source) const {
|
||||
return _hasSource(source);
|
||||
}
|
||||
|
||||
unsigned int Collection::sourcesTotal() const {
|
||||
std::lock_guard lock(mutex);
|
||||
return _sourcesTotal();
|
||||
}
|
||||
|
||||
unsigned int Collection::sourcesSuccess() const {
|
||||
std::lock_guard lock(mutex);
|
||||
return _sourcesSuccess();
|
||||
}
|
||||
|
||||
unsigned int Collection::sourcesPending() const {
|
||||
std::lock_guard lock(mutex);
|
||||
return _sourcesPending();
|
||||
}
|
||||
|
||||
unsigned int Collection::sourcesError() const {
|
||||
std::lock_guard lock(mutex);
|
||||
return _sourcesError();
|
||||
}
|
||||
|
||||
bool Collection::_hasSource(const std::string& source) const {
|
||||
return knownSources.count(source) > 0;
|
||||
}
|
||||
|
||||
unsigned int Collection::_sourcesTotal() const {
|
||||
return knownSources.size();
|
||||
}
|
||||
|
||||
unsigned int Collection::_sourcesSuccess() const {
|
||||
return successSources.size();
|
||||
}
|
||||
|
||||
unsigned int Collection::_sourcesPending() const {
|
||||
return downloadingSources.size() + readingSources.size() + buildingSources.size() + pendingSources.size();
|
||||
}
|
||||
|
||||
unsigned int Collection::_sourcesError() const {
|
||||
return _sourcesTotal() - _sourcesPending() - _sourcesSuccess();
|
||||
}
|
||||
|
@ -8,6 +8,7 @@
|
||||
#include <string_view>
|
||||
#include <regex>
|
||||
#include <mutex>
|
||||
#include <queue>
|
||||
|
||||
#include "loggable.h"
|
||||
#include "component.h"
|
||||
@ -16,11 +17,13 @@
|
||||
|
||||
class Collection : protected Loggable, public std::enable_shared_from_this<Collection> {
|
||||
using Sources = std::set<std::string>;
|
||||
using SourcesQueue = std::queue<std::string>;
|
||||
using Downloads = std::map<std::string, std::unique_ptr<Download>>;
|
||||
using Components = std::map<std::string, std::unique_ptr<Component>>;
|
||||
|
||||
public:
|
||||
Collection(
|
||||
const std::filesystem::path& location,
|
||||
const std::filesystem::path& destination,
|
||||
const std::string& target,
|
||||
const std::shared_ptr<Logger>& logger,
|
||||
@ -39,21 +42,30 @@ private:
|
||||
void sourceDownaloded(const std::string& source, TaskManager::Error err);
|
||||
void sourceRead(const std::string& source, TaskManager::Error err);
|
||||
void sourceBuilt(const std::string& source, TaskManager::Error err);
|
||||
void rootBuilt(TaskManager::Error err);
|
||||
|
||||
static bool isRemote(const std::string& source);
|
||||
|
||||
void processSource(const std::string& source);
|
||||
void queueDownload(const std::string& source);
|
||||
void queueRead(const std::string& source, const std::filesystem::path& location);
|
||||
|
||||
bool _hasSource(const std::string& source) const;
|
||||
unsigned int _sourcesTotal() const;
|
||||
unsigned int _sourcesPending() const;
|
||||
unsigned int _sourcesError() const;
|
||||
unsigned int _sourcesSuccess() const;
|
||||
|
||||
private:
|
||||
mutable std::mutex mutex;
|
||||
std::filesystem::path destination;
|
||||
std::string target;
|
||||
std::shared_ptr<TaskManager> taskManager;
|
||||
std::unique_ptr<Component> root;
|
||||
std::string librariesPath;
|
||||
Sources knownSources;
|
||||
Sources successSources;
|
||||
SourcesQueue pendingSources;
|
||||
Downloads downloadingSources;
|
||||
Components readingSources;
|
||||
Components buildingSources;
|
||||
|
@ -13,6 +13,13 @@ constexpr std::array<std::string_view, Component::done + 1> stringStates {
|
||||
"done"
|
||||
};
|
||||
|
||||
constexpr std::array<std::string_view, Component::unknown + 1> stringTypes {
|
||||
"file",
|
||||
"directory",
|
||||
"mason",
|
||||
"unknown"
|
||||
};
|
||||
|
||||
Component::Component(
|
||||
const std::filesystem::path& path,
|
||||
const std::weak_ptr<Collection>& collection,
|
||||
@ -23,7 +30,7 @@ Component::Component(
|
||||
type(unknown),
|
||||
collection(collection),
|
||||
location(path),
|
||||
scenario(std::nullopt)
|
||||
manifest(std::nullopt)
|
||||
{}
|
||||
|
||||
void Component::read() {
|
||||
@ -56,65 +63,73 @@ void Component::tryReadingBuildScenarios() {
|
||||
}
|
||||
|
||||
bool Component::readAsMason() {
|
||||
std::filesystem::path masonScenarion = location/masonJSON;
|
||||
std::filesystem::path masonManifest = location/masonJSON;
|
||||
try {
|
||||
std::ifstream file(masonScenarion);
|
||||
std::ifstream file(masonManifest);
|
||||
if (file.is_open())
|
||||
scenario = nlohmann::json::parse(file);
|
||||
manifest = nlohmann::json::parse(file);
|
||||
else
|
||||
minor("Couldn't open " + masonScenarion.string());
|
||||
minor("Couldn't open " + masonManifest.string());
|
||||
} catch (const nlohmann::json::exception& e) {
|
||||
major("Couldn't parse " + masonScenarion.string());
|
||||
major("Couldn't parse " + masonManifest.string());
|
||||
}
|
||||
|
||||
if (scenario.has_value()) {
|
||||
const nlohmann::json& sc = scenario.value();
|
||||
if (!sc.is_object())
|
||||
return errorScenario(masonScenarion.string() + " is unexpected root type");
|
||||
if (manifest.has_value()) {
|
||||
const nlohmann::json& mnfst = manifest.value();
|
||||
if (!mnfst.is_object())
|
||||
return errorScenario(masonManifest.string() + " is unexpected root type");
|
||||
|
||||
nlohmann::json::const_iterator itr = sc.find(dependencies);
|
||||
if (itr == sc.end()) {
|
||||
info(masonScenarion.string() + " doesn't have dependencies");
|
||||
nlohmann::json::const_iterator itr = mnfst.find(dependencies);
|
||||
if (itr == mnfst.end()) {
|
||||
info(masonManifest.string() + " doesn't have dependencies");
|
||||
} else {
|
||||
const nlohmann::json& deps = *itr;
|
||||
if (!deps.is_array())
|
||||
return errorScenario(masonScenarion.string() + " dependencies are not organized as an array");
|
||||
|
||||
for (const nlohmann::json& dep : deps) {
|
||||
std::shared_ptr<Collection> col = collection.lock();
|
||||
switch (dep.type()) {
|
||||
case nlohmann::json::value_t::string:
|
||||
col->addSource(dep);
|
||||
break;
|
||||
case nlohmann::json::value_t::object: {
|
||||
nlohmann::json::const_iterator ditr = dep.find("path");
|
||||
if (ditr == dep.end())
|
||||
return errorScenario(masonScenarion.string() + " object of unexpected format in dependecies");
|
||||
|
||||
nlohmann::json path = *ditr;
|
||||
if (!path.is_string())
|
||||
return errorScenario(masonScenarion.string() + " object of unexpected format in dependecies");
|
||||
|
||||
col->addSource(path);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
return errorScenario(masonScenarion.string() + " has unexpected entries in dependecies");
|
||||
}
|
||||
}
|
||||
if (!readMasonDependencies(deps, masonManifest))
|
||||
return false;
|
||||
}
|
||||
|
||||
type = mason;
|
||||
info(location.string() + " seems to be a mason project");
|
||||
info(location.string() + " seems to be a valid mason project");
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
bool Component::readMasonDependencies(const nlohmann::json& deps, const std::string& manifestPath) {
|
||||
if (!deps.is_array())
|
||||
return errorScenario(manifestPath + " dependencies are not organized as an array");
|
||||
|
||||
for (const nlohmann::json& dep : deps) {
|
||||
std::shared_ptr<Collection> col = collection.lock();
|
||||
switch (dep.type()) {
|
||||
case nlohmann::json::value_t::string:
|
||||
col->addSource(dep);
|
||||
break;
|
||||
case nlohmann::json::value_t::object: {
|
||||
nlohmann::json::const_iterator ditr = dep.find("path");
|
||||
if (ditr == dep.end())
|
||||
return errorScenario(manifestPath + " object of unexpected format in dependecies");
|
||||
|
||||
nlohmann::json path = *ditr;
|
||||
if (!path.is_string())
|
||||
return errorScenario(manifestPath + " object of unexpected format in dependecies");
|
||||
|
||||
col->addSource(path);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
return errorScenario(manifestPath + " has unexpected entries in dependecies");
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
bool Component::errorScenario(const std::string& message) {
|
||||
major(message);
|
||||
scenario = std::nullopt;
|
||||
manifest = std::nullopt;
|
||||
return false;
|
||||
return false;
|
||||
}
|
||||
@ -124,11 +139,67 @@ void Component::build(const std::filesystem::path& destination, const std::strin
|
||||
throw WrongState(state, "build");
|
||||
|
||||
state = building;
|
||||
|
||||
info("Building " + location.string() + " to " + destination.string());
|
||||
|
||||
switch (type) {
|
||||
case file:
|
||||
buildAsFile(destination);
|
||||
break;
|
||||
case directory:
|
||||
buildAsDiractory(destination);
|
||||
break;
|
||||
case mason:
|
||||
buildAsMason(destination, target);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
state = done;
|
||||
}
|
||||
|
||||
void Component::buildAsFile(const std::filesystem::path& destination) {
|
||||
std::filesystem::copy(location, destination/location.filename());
|
||||
}
|
||||
|
||||
void Component::buildAsDiractory(const std::filesystem::path& destination) {
|
||||
std::filesystem::copy(location, destination/location.filename(),
|
||||
std::filesystem::copy_options::recursive |
|
||||
std::filesystem::copy_options::overwrite_existing
|
||||
);
|
||||
}
|
||||
|
||||
std::string Component::getLibrariesPath() const {
|
||||
//TODO may be it makes sence to cache
|
||||
if (type != mason)
|
||||
throw WrongType(type, "getLibrariesPath");
|
||||
|
||||
const nlohmann::json& mnfst = manifest.value();
|
||||
nlohmann::json::const_iterator itr = mnfst.find("libraries");
|
||||
if (itr == mnfst.end())
|
||||
return "";
|
||||
|
||||
const nlohmann::json& libs = *itr;
|
||||
if (libs.is_string())
|
||||
return libs;
|
||||
|
||||
warn("Mason manifest " + (location/masonJSON).string()
|
||||
+ " has field \"libraries\" which is not string, ignoring it");
|
||||
return "";
|
||||
}
|
||||
|
||||
std::filesystem::path Component::getLocation() const {
|
||||
return location;
|
||||
}
|
||||
|
||||
bool Component::successfullyRead() const {
|
||||
return state > reading && state != error;
|
||||
}
|
||||
|
||||
void Component::buildAsMason(const std::filesystem::path& destination, const std::string& target) {
|
||||
warn("mason build is not implemented yet");
|
||||
}
|
||||
|
||||
Component::State Component::getState() const {
|
||||
return state;
|
||||
}
|
||||
@ -139,5 +210,10 @@ Component::Type Component::getType() const {
|
||||
|
||||
Component::WrongState::WrongState(State state, const std::string& action):
|
||||
std::runtime_error("An attempt to perform action \"" + action
|
||||
+ "\" on a wrong state \"" + stringStates[state].data() + '\"')
|
||||
+ "\" on a wrong component state \"" + stringStates[state].data() + '\"')
|
||||
{}
|
||||
|
||||
Component::WrongType::WrongType(Type type, const std::string& action):
|
||||
std::runtime_error("An attempt to perform an action \"" + action
|
||||
+ "\" on a wrong component type \"" + stringTypes[type].data() + '\"')
|
||||
{}
|
||||
|
@ -19,6 +19,7 @@ class Collection;
|
||||
class Component : protected Loggable {
|
||||
public:
|
||||
class WrongState;
|
||||
class WrongType;
|
||||
enum State {
|
||||
initial,
|
||||
reading,
|
||||
@ -29,10 +30,10 @@ public:
|
||||
};
|
||||
|
||||
enum Type {
|
||||
unknown,
|
||||
file,
|
||||
directory,
|
||||
mason
|
||||
mason,
|
||||
unknown
|
||||
};
|
||||
|
||||
Component(
|
||||
@ -43,6 +44,9 @@ public:
|
||||
|
||||
Type getType() const;
|
||||
State getState() const;
|
||||
std::filesystem::path getLocation() const;
|
||||
std::string getLibrariesPath() const;
|
||||
bool successfullyRead() const;
|
||||
|
||||
void read();
|
||||
void build(const std::filesystem::path& destination, const std::string& target);
|
||||
@ -52,15 +56,26 @@ private:
|
||||
bool readAsMason();
|
||||
bool errorScenario(const std::string& message);
|
||||
|
||||
void buildAsFile(const std::filesystem::path& destination);
|
||||
void buildAsDiractory(const std::filesystem::path& destination);
|
||||
void buildAsMason(const std::filesystem::path& destination, const std::string& target);
|
||||
|
||||
bool readMasonDependencies(const nlohmann::json& deps, const std::string& manifestPath);
|
||||
|
||||
private:
|
||||
State state;
|
||||
Type type;
|
||||
std::weak_ptr<Collection> collection;
|
||||
std::filesystem::path location;
|
||||
std::optional<nlohmann::json> scenario;
|
||||
std::optional<nlohmann::json> manifest;
|
||||
};
|
||||
|
||||
class Component::WrongState : public std::runtime_error {
|
||||
public:
|
||||
explicit WrongState(State state, const std::string& action);
|
||||
};
|
||||
|
||||
class Component::WrongType : public std::runtime_error {
|
||||
public:
|
||||
explicit WrongType(Type type, const std::string& action);
|
||||
};
|
||||
|
@ -71,18 +71,16 @@ void Download::proceed() {
|
||||
const FileInfo& fl = file.value();
|
||||
std::optional<std::filesystem::path> path = downloadAsFile(fl);
|
||||
if (path.has_value()) {
|
||||
bool success = true;
|
||||
info("Successfully downloaded " + url);
|
||||
if (fl.archive) {
|
||||
info(url + " appears to be an archive, extracting");
|
||||
success = extract(path.value(), destination/fl.name);
|
||||
if (success)
|
||||
bool success = extract(path.value(), destination/fl.name);
|
||||
if (success) {
|
||||
info("Successfully extracted " + url);
|
||||
}
|
||||
|
||||
if (success) {
|
||||
location = destination/fl.name;
|
||||
return;
|
||||
location = destination/fl.name;
|
||||
}
|
||||
} else {
|
||||
location = path;
|
||||
}
|
||||
}
|
||||
|
||||
|
36
src/main.cpp
36
src/main.cpp
@ -20,25 +20,31 @@ int main(int argc, char *argv[]) {
|
||||
|
||||
std::shared_ptr<Logger> logger = std::make_shared<Logger>(Logger::Severity::debug);
|
||||
std::shared_ptr<TaskManager> taskManager = std::make_shared<TaskManager>(logger);
|
||||
std::shared_ptr<Collection> collection = std::make_shared<Collection>(secondArg, "", logger, taskManager);
|
||||
std::shared_ptr<Collection> collection = std::make_shared<Collection>(
|
||||
firstArg,
|
||||
secondArg,
|
||||
"",
|
||||
logger,
|
||||
taskManager
|
||||
);
|
||||
|
||||
taskManager->start();
|
||||
collection->addSource(firstArg);
|
||||
|
||||
taskManager->wait();
|
||||
taskManager->stop();
|
||||
|
||||
// int result = -1;
|
||||
// if (success) {
|
||||
// root.info("successfully parsed project " + root.getName());
|
||||
// root.info("dependencies count is " + std::to_string(root.dependenciesCount()));
|
||||
//
|
||||
// root.discover();
|
||||
// result = 0;
|
||||
// }
|
||||
//
|
||||
// root.printLog();
|
||||
unsigned int success = collection->sourcesSuccess();
|
||||
unsigned int total = collection->sourcesTotal();
|
||||
if (total == success) {
|
||||
logger->log(Logger::Severity::info, "Successfully built " + firstArg);
|
||||
return 0;
|
||||
} else {
|
||||
logger->log(Logger::Severity::error, "Failed to build " + firstArg);
|
||||
logger->log(Logger::Severity::info, "Success: " + std::to_string(success));
|
||||
logger->log(Logger::Severity::info, "Failed: " + std::to_string(collection->sourcesError()));
|
||||
logger->log(Logger::Severity::info, "Unfinished: " + std::to_string(collection->sourcesPending()));
|
||||
logger->log(Logger::Severity::info, "Total: " + std::to_string(total));
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
||||
return 0;
|
||||
return -2;
|
||||
}
|
||||
|
@ -10,5 +10,6 @@
|
||||
},
|
||||
"https://git.macaw.me/blue/mason/archive/main.tar.gz",
|
||||
"https://git.macaw.me/blue/mason/raw/branch/main/src2/atomicmutex.h"
|
||||
]
|
||||
],
|
||||
"libraries": "lib"
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user