diff --git a/CMakeLists.txt b/CMakeLists.txt index 427475e..97869c6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,7 +42,6 @@ target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/s target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src2) add_subdirectory(src) -add_subdirectory(src2) add_subdirectory(test) target_link_libraries(${PROJECT_NAME} PRIVATE nlohmann_json::nlohmann_json) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0a04c7f..06b48a4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,14 +1,22 @@ set(SOURCES main.cpp loggger.cpp - project.cpp - dependency.cpp + component.cpp + collection.cpp + loggable.cpp + taskmanager.cpp + download.cpp + atomicmutex.cpp ) set(HEADERS logger.h - project.h - dependency.cpp + component.h + collection.h + loggable.h + taskmanager.h + download.h + atomicmutex.h ) target_sources(${PROJECT_NAME} PRIVATE ${SOURCES}) diff --git a/src2/atomicmutex.cpp b/src/atomicmutex.cpp similarity index 100% rename from src2/atomicmutex.cpp rename to src/atomicmutex.cpp diff --git a/src2/atomicmutex.h b/src/atomicmutex.h similarity index 100% rename from src2/atomicmutex.h rename to src/atomicmutex.h diff --git a/src2/collection.cpp b/src/collection.cpp similarity index 86% rename from src2/collection.cpp rename to src/collection.cpp index 1838a9c..2e339f6 100644 --- a/src2/collection.cpp +++ b/src/collection.cpp @@ -11,6 +11,8 @@ Collection::Collection( const std::shared_ptr& taskManager ): Loggable(logger), + std::enable_shared_from_this(), + mutex(), destination(destination), target(target), taskManager(taskManager), @@ -30,7 +32,8 @@ Collection::Collection( } void Collection::addSource(const std::string& source) { - if (hasSource(source)) { + std::lock_guard lock(mutex); + if (_hasSource(source)) { major("Source " + source + " is already present, skipping as duplicate"); return; } @@ -43,6 +46,8 @@ void Collection::addSource(const std::string& source) { } void Collection::sourceDownaloded(const std::string& source, TaskManager::Error err) { + std::lock_guard lock(mutex); + Downloads::const_iterator itr = downloadingSources.find(source); std::optional location = itr->second->getLocation(); downloadingSources.erase(itr); @@ -55,6 +60,9 @@ void Collection::sourceDownaloded(const std::string& source, TaskManager::Error } void Collection::sourceRead(const std::string& source, TaskManager::Error err) { + std::lock_guard lock(mutex); + + debug("Source " + source + " has been read"); Components::iterator itr = readingSources.find(source); if (err.has_value()) { error("Coundn't read " + source + ": " + err.value()->what()); @@ -62,6 +70,7 @@ void Collection::sourceRead(const std::string& source, TaskManager::Error err) { return; } + debug("Queuing build of " + source); std::pair res = buildingSources.emplace(source, std::move(itr->second)); readingSources.erase(itr); @@ -72,6 +81,8 @@ void Collection::sourceRead(const std::string& source, TaskManager::Error err) { } void Collection::sourceBuilt(const std::string& source, TaskManager::Error err) { + std::lock_guard lock(mutex); + debug("Source " + source + " has been built"); Components::iterator itr = buildingSources.find(source); buildingSources.erase(itr); if (err.has_value()) { @@ -87,6 +98,7 @@ bool Collection::isRemote(const std::string& source) { } void Collection::queueDownload(const std::string& source) { + debug("Queuing download of " + source); std::pair res = downloadingSources.emplace(std::piecewise_construct, std::forward_as_tuple(source), std::forward_as_tuple( @@ -101,6 +113,7 @@ void Collection::queueDownload(const std::string& source) { } void Collection::queueRead(const std::string& source, const std::filesystem::path& location) { + debug("Queuing read of " + source); std::pair res = readingSources.emplace(std::piecewise_construct, std::forward_as_tuple(source), std::forward_as_tuple( @@ -115,5 +128,10 @@ void Collection::queueRead(const std::string& source, const std::filesystem::pat } bool Collection::hasSource(const std::string& source) const { + std::lock_guard lock(mutex); + return _hasSource(source); +} + +bool Collection::_hasSource(const std::string& source) const { return knownSources.count(source) > 0; } diff --git a/src2/collection.h b/src/collection.h similarity index 94% rename from src2/collection.h rename to src/collection.h index 5c26fb9..7aaaa44 100644 --- a/src2/collection.h +++ b/src/collection.h @@ -7,6 +7,7 @@ #include #include #include +#include #include "loggable.h" #include "component.h" @@ -44,7 +45,10 @@ private: void queueDownload(const std::string& source); void queueRead(const std::string& source, const std::filesystem::path& location); + bool _hasSource(const std::string& source) const; + private: + mutable std::mutex mutex; std::filesystem::path destination; std::string target; std::shared_ptr taskManager; diff --git a/src2/component.cpp b/src/component.cpp similarity index 100% rename from src2/component.cpp rename to src/component.cpp diff --git a/src2/component.h b/src/component.h similarity index 100% rename from src2/component.h rename to src/component.h diff --git a/src/dependency.cpp b/src/dependency.cpp deleted file mode 100644 index b282dc2..0000000 --- a/src/dependency.cpp +++ /dev/null @@ -1,375 +0,0 @@ -#include "dependency.h" - -#include - -#include "project.h" - -constexpr std::string_view downloads("downloads"); -constexpr std::string_view sources("sources"); -constexpr std::string_view build("build"); - -constexpr std::string_view acceptJson("accept: application/json"); -const std::regex http("^https?:\\/\\/"); -const std::regex repo("(^https?):\\/\\/([\\w\\d\\.\\-\\_]+)\\/([\\w\\d\\-\\_]+)\\/([\\w\\d\\-\\_]+)"); - -Dependency::Dependency( - const std::string& path, - Type type, - const std::optional& name, - const std::optional& version -): - path(path), - type(type), - name(name), - version(version), - location() -{} - -Dependency::Type Dependency::getType() const { - return type; -} - -std::optional Dependency::getName() const { - return name; -} - -std::optional Dependency::getVersion() const { - return version; -} - -std::optional Dependency::getLocation() const { - if (location.empty()) - return std::nullopt; - else - return location; -} - -bool Dependency::prepare(const std::filesystem::path& source, const std::filesystem::path& destination) { - std::smatch results; - if (std::regex_search(path, results, repo)) { - Project::info(path + " appears to be a git repository"); - const std::string& protocol = results[1]; - const std::string& host = results[2]; - const std::string& owner = results[3]; - const std::string& repo = results[4]; - - std::filesystem::path temp; - bool success = downloadRepo(destination, protocol, host, owner, repo, temp); - if (success) { - Project::info("Successfully obtained project " + path); - - switch (type) { - case Type::automatic: - if (std::filesystem::exists(temp/"mason.json")) - type = Type::mason; - else - type = Type::simple; - - break; - case Type::mason: - if (std::filesystem::exists(temp/"mason.json")) - break; - else { - Project::warn("Project " + path + " is supposed to me a mason project, but no mason.json was found in the project directory"); - success = false; - } - - break; - case Type::simple: - break; - } - } - - return success; - } else { - Project::info("checking project at path " + path); - std::filesystem::directory_entry srcDir(source / path); - if (!srcDir.exists()) { - Project::error("Project at " + path + " doesn't exist"); - return false; - } - } - - return true; -} - -bool Dependency::downloadRepo( - const std::filesystem::path& destination, - const std::string& protocol, - const std::string& host, - const std::string& owner, - const std::string& repo, - std::filesystem::path& out -) { - nlohmann::json info; - Project::info("Trying Gitea v1 API"); - bool res = repoGiteaApi1(protocol, host, owner, repo, info); - - if (res) { - nlohmann::json::const_iterator itr = info.find("default_branch"); - if (itr != info.end() && itr->is_string()) { - std::string branchName = *itr; - Project::info("Gitea v1 API seem to have worked"); - Project::info("Default branch is " + branchName); - - std::string fileName = branchName + ".tar.gz"; - std::string url = protocol + "://" + host + "/api/v1/repos/" + owner + "/" + repo + "/archive/" + fileName; - std::filesystem::path archivePath = destination/downloads/fileName; - res = download(url, archivePath); - if (res) { - Project::info("Successfully downloaded " + archivePath.string()); - - location = destination/sources; - res = extract(archivePath, location); - if (res) { - out = destination/build/repo; - if (!std::filesystem::is_directory(out)) { - Project::error("Extracted archive " + fileName + " but the content is unexpected"); - res = false; - } - } else { - Project::error("Couldn't extract archive " + fileName); - } - - if (!res) - location.clear(); - - return res; - } - } else { - res = false; - } - } - - if (!res) { - Project::warn("Gitea v1 API didn't work"); - } - - return res; -} - -bool Dependency::repoGiteaApi1( - const std::string& protocol, - const std::string& host, - const std::string& owner, - const std::string& repo, - nlohmann::json& json -) { - bool result = false; - std::string url = protocol + "://" + host + "/api/v1/repos/" + owner + "/" + repo; - std::string data; - CURLcode code = httpGet(url, data, {acceptJson}); - if (code == CURLE_OK) { - try { - json = nlohmann::json::parse(data); - result = true; - } catch (const nlohmann::json::exception& e) { - Project::warn(e.what()); - } - } - - return result; -} - -CURLcode Dependency::httpGet(const std::string& url, std::string& result, const std::list& headers) { - CURL* curl = curl_easy_init(); - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L); - - struct curl_slist* curlHeaders = nullptr; - for (const std::string_view& header : headers) - curlHeaders = curl_slist_append(curlHeaders, header.data()); - - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curlHeaders); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeString); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &result); - - curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); - curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, trace); - curl_easy_setopt(curl, CURLOPT_DEBUGDATA, this); - - CURLcode res = curl_easy_perform(curl); - - curl_easy_cleanup(curl); - return res; -} - - -bool Dependency::download(const std::string& url, const std::filesystem::path& destination) { - std::filesystem::create_directories(destination.parent_path()); - if (std::filesystem::exists(destination)) { - Project::minor("File " + destination.string() + " already exists, will be overwritten"); - - if (std::filesystem::is_directory(destination)) - std::filesystem::remove_all(destination); - } - - bool result = false; - CURL* curl = curl_easy_init(); - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeFile); - - curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); - curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, trace); - curl_easy_setopt(curl, CURLOPT_DEBUGDATA, this); - - FILE* pagefile = fopen(destination.c_str(), "wb"); - if (pagefile) { - curl_easy_setopt(curl, CURLOPT_WRITEDATA, pagefile); - CURLcode res = curl_easy_perform(curl); - - fclose(pagefile); - if (res != CURLE_OK) { - Project::warn("Couldn't download file " + url + ": " + curl_easy_strerror(res)); - } else { - uint32_t code; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); - if (code == 200) - result = true; - else - Project::warn("Couldn't download file " + url + ": response code " + std::to_string(code)); - } - } else { - Project::error(std::string("Couldn't open output file ") + destination.c_str()); - } - - curl_easy_cleanup(curl); - - if (!result) { - Project::minor("Removing " + destination.string() + " since the donwload failed"); - std::filesystem::remove_all(destination); - } - - return result; -} - -size_t Dependency::writeString(void* data, size_t size, size_t nmemb, void* pointer) { - size_t finalSize = size * nmemb; - std::string* string = static_cast(pointer); - string->append(static_cast(data), finalSize); - - return finalSize; -} - -size_t Dependency::writeFile(void* data, size_t size, size_t nmemb, void* file) { - size_t written = fwrite(data, size, nmemb, (FILE *)file); - return written; -} - -int Dependency::trace(CURL* handle, curl_infotype type, char* data, size_t size, void* clientp) { - switch (type) { - case CURLINFO_TEXT: { - std::string message(data, size); - if (message[size - 1] == '\n') - message = message.substr(0, size - 1); - - Project::debug(message); - } break; - default: - break; - } - - return 0; -} - -bool Dependency::extract(const std::filesystem::path& source, const std::filesystem::path& destination) const { - int flags = ARCHIVE_EXTRACT_TIME; - flags |= ARCHIVE_EXTRACT_PERM; - flags |= ARCHIVE_EXTRACT_ACL; - flags |= ARCHIVE_EXTRACT_FFLAGS; - - struct archive* a = archive_read_new(); - struct archive* ext = archive_write_disk_new(); - struct archive_entry *entry; - archive_read_support_format_all(a); - archive_read_support_filter_all(a); - archive_write_disk_set_options(ext, flags); - archive_write_disk_set_standard_lookup(ext); - - bool result = true; - bool readOpen = false; - bool writeOpen = false; - int r = archive_read_open_filename(a, source.c_str(), 10240); - if (r) { - Project::major("Couldn't open file " + source.string()); - result = false; - } else { - readOpen = true; - } - - while (result) { - r = archive_read_next_header(a, &entry); - if (r == ARCHIVE_EOF) - break; - - if (r < ARCHIVE_OK) - Project::major(archive_error_string(a)); - - if (r < ARCHIVE_WARN) - break; - - std::string fileName(archive_entry_pathname(entry)); - std::filesystem::path filePath = destination/fileName; - Project::debug("Extracting " + filePath.string()); - if (std::filesystem::exists(filePath)) - Project::minor(filePath.string() + " exists, overwriting"); - - archive_entry_set_pathname_utf8(entry, filePath.c_str()); - - r = archive_write_header(ext, entry); - if (r < ARCHIVE_OK) { - Project::major(archive_error_string(ext)); - } else if (archive_entry_size(entry) > 0) { - writeOpen = true; - r = copy(a, ext); - if (r < ARCHIVE_OK) - Project::major(archive_error_string(ext)); - - if (r < ARCHIVE_WARN) - break; - } - r = archive_write_finish_entry(ext); - if (r < ARCHIVE_OK) - Project::major(archive_error_string(ext)); - - if (r < ARCHIVE_WARN) - break; - } - - if (readOpen) - archive_read_close(a); - - if (writeOpen) - archive_write_close(ext); - - archive_read_free(a); - archive_write_free(ext); - - return result; -} - -int Dependency::copy(struct archive* ar, struct archive* aw) const { - int r; - const void *buff; - size_t size; - la_int64_t offset; - - while (true) { - r = archive_read_data_block(ar, &buff, &size, &offset); - if (r == ARCHIVE_EOF) - return ARCHIVE_OK; - - if (r < ARCHIVE_OK) - return r; - - r = archive_write_data_block(aw, buff, size, offset); - if (r < ARCHIVE_OK) { - Project::major(archive_error_string(aw)); - return r; - } - } -} - - diff --git a/src/dependency.h b/src/dependency.h deleted file mode 100644 index 5e9e95d..0000000 --- a/src/dependency.h +++ /dev/null @@ -1,69 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -class Dependency { -public: - enum class Type { - automatic, - simple, - mason - }; - - Dependency( - const std::string& path, - Type type = Type::automatic, - const std::optional& name = std::nullopt, - const std::optional& version = std::nullopt - ); - - Type getType() const; - std::optional getName() const; - std::optional getVersion() const; - std::optional getLocation() const; - - bool prepare(const std::filesystem::path& source, const std::filesystem::path& destination); - - const std::string path; - -private: - static size_t writeFile(void* data, size_t size, size_t nmemb, void* file); - static size_t writeString(void* data, size_t size, size_t nmemb, void* mem); - static int trace(CURL *handle, curl_infotype type, char *data, size_t size, void *clientp); - bool download(const std::string& url, const std::filesystem::path& destination); - - bool downloadRepo( - const std::filesystem::path& destination, - const std::string& protocol, - const std::string& host, - const std::string& owner, - const std::string& repo, - std::filesystem::path& out - ); - bool repoGiteaApi1( - const std::string& protocol, - const std::string& host, - const std::string& owner, - const std::string& repo, - nlohmann::json& json - ); - CURLcode httpGet(const std::string& url, std::string& result, const std::list& headers = {}); - - bool extract(const std::filesystem::path& source, const std::filesystem::path& destination) const; - int copy(struct archive *ar, struct archive *aw) const; - -private: - Type type; - std::optional name; - std::optional version; - std::filesystem::path location; -}; diff --git a/src2/download.cpp b/src/download.cpp similarity index 80% rename from src2/download.cpp rename to src/download.cpp index a96dd00..8b61c39 100644 --- a/src2/download.cpp +++ b/src/download.cpp @@ -3,7 +3,10 @@ constexpr std::string_view archived("archived"); constexpr std::string_view headerAcceptJson("accept: application/json"); -static const std::regex repo("(^https?):\\/\\/([\\w\\d\\.\\-\\_]+)\\/([\\w\\d\\-\\_]+)\\/([\\w\\d\\-\\_]+)"); +static const std::regex file("(^https?):\\/\\/(?:[\\w\\d\\.\\-\\_]+\\/)+([\\w\\d\\-\\_]+)\\.([\\w\\d\\.]+)"); +static const std::regex repo("(^https?):\\/\\/([\\w\\d\\.\\-\\_]+)\\/([\\w\\d\\-\\_]+)\\/([\\w\\d\\-\\_]+)(?:(?:\\.git)|\\/)?$"); + +static const std::regex archive("tar\\.gz$"); unsigned int Download::instances(0); AtomicMutex Download::amx{}; @@ -50,23 +53,47 @@ std::optional Download::getLocation() const { } void Download::proceed() { - bool success = false; std::optional repo = testRepo(); - std::optional path; - if (repo.has_value()) - path = downloadAsRepo(repo.value()); - - if (path.has_value()) { + if (repo.has_value()) { + const RepoInfo& rp = repo.value(); + std::optional path = downloadAsRepo(rp); info(url + " has been successfully donwloaded as a repository, extracting"); - success = extract(path.value(), destination/repo.value().name); - if (success) - location = destination/repo.value().name; + bool success = extract(path.value(), destination); + if (success) { + location = destination/rp.name; + info("Successfully extracted " + url); + return; + } } - if (success) - info("Successfully extracted " + url); + std::optional file = testFile(); + if (file.has_value()) { + const FileInfo& fl = file.value(); + std::optional path = downloadAsFile(fl); + if (path.has_value()) { + bool success = true; + info("Successfully downloaded " + url); + if (fl.archive) { + info(url + " appears to be an archive, extracting"); + success = extract(path.value(), destination/fl.name); + if (success) + info("Successfully extracted " + url); + } + + if (success) { + location = destination/fl.name; + return; + } + } + + } + + // CURLcode code = httpDownload(url, path); + // if (code == CURLE_OK) + // return path; } + std::optional Download::repoInfoGiteaApi1(const RepoInfo& repo) { std::string url = repo.origin() + "/api/v1/repos/" + repo.project(); std::string data; @@ -99,6 +126,24 @@ std::optional Download::downloadAsRepo(const RepoInfo& re return std::nullopt; } +std::optional Download::downloadAsFile(const FileInfo& file) { + std::filesystem::path dst; + if (file.archive) + dst = destination/archived/file.fileName(); + else + dst = destination/file.fileName(); + + CURLcode code = httpDownload(url, dst); + if (code == CURLE_OK) + return dst; + + minor("Removing " + dst.string()); + if (std::filesystem::exists(dst)) + std::filesystem::remove_all(dst); + + return std::nullopt; +} + std::optional Download::downloadRepoGiteaApi1(const RepoInfo& repo, const std::string& branch) { std::string fileName = branch + ".tar.gz"; std::string url = repo.origin() + "/api/v1/repos/" + repo.project() + "/archive/" + fileName; @@ -130,7 +175,7 @@ CURLcode Download::httpGet( } CURLcode Download::httpDownload(const std::string& url, const std::filesystem::path& path, const std::vector& headers) { - info("Starting to download " + url); + info("Starting to download " + url + " to " + path.string()); std::filesystem::create_directories(path.parent_path()); if (std::filesystem::exists(path)) { minor("File " + path.string() + " already exists, will be overwritten"); @@ -185,7 +230,7 @@ void Download::createCurl() { if (!crl) throw CurlError("Error creating curl instalce"); - curl_easy_setopt(crl, CURLOPT_USERAGENT, "libcurl-agent/1.0" ); + curl_easy_setopt(crl, CURLOPT_USERAGENT, "libcurl-agent/1.0" ); curl_easy_setopt(crl, CURLOPT_VERBOSE, 1L); curl_easy_setopt(crl, CURLOPT_DEBUGFUNCTION, trace); curl_easy_setopt(crl, CURLOPT_DEBUGDATA, this); @@ -194,6 +239,21 @@ void Download::createCurl() { curl = std::unique_ptr(crl); } +std::optional Download::testFile() const { + std::smatch results; + if (std::regex_search(url, results, file)) { + info(url + " appears to be a file"); + std::string ext(results[3]); + return FileInfo { + results[1], + results[2], + ext, + std::regex_search(ext, archive) + }; + } + + return std::nullopt; +} std::optional Download::testRepo() const { std::smatch results; @@ -347,4 +407,7 @@ std::string Download::RepoInfo::project() const { return owner + "/" + name; } +std::string Download::FileInfo::fileName() const { + return name + "." + extension; +} diff --git a/src2/download.h b/src/download.h similarity index 89% rename from src2/download.h rename to src/download.h index d5deb06..1d7f794 100644 --- a/src2/download.h +++ b/src/download.h @@ -22,6 +22,7 @@ class Collection; class Download : protected Loggable { struct RepoInfo; + struct FileInfo; struct CurlDeleter { void operator() (CURL* curl) const; }; @@ -43,9 +44,11 @@ public: std::optional getLocation() const; private: + std::optional testFile() const; std::optional testRepo() const; std::optional downloadAsRepo(const RepoInfo& repo); + std::optional downloadAsFile(const FileInfo& file); std::optional repoInfoGiteaApi1(const RepoInfo& repo); std::optional downloadRepoGiteaApi1(const RepoInfo& repo, const std::string& branch); @@ -80,6 +83,15 @@ struct Download::RepoInfo { std::string project() const; }; +struct Download::FileInfo { + std::string protocol; + std::string name; + std::string extension; + bool archive; + + std::string fileName() const; +}; + class Download::CurlError : public std::runtime_error { public: explicit CurlError(const std::string& message); diff --git a/src2/loggable.cpp b/src/loggable.cpp similarity index 100% rename from src2/loggable.cpp rename to src/loggable.cpp diff --git a/src2/loggable.h b/src/loggable.h similarity index 100% rename from src2/loggable.h rename to src/loggable.h diff --git a/src/main.cpp b/src/main.cpp index 6f80f35..5a3f0e4 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -19,7 +19,7 @@ int main(int argc, char *argv[]) { std::shared_ptr logger = std::make_shared(Logger::Severity::debug); - std::shared_ptr taskManager = std::make_shared(); + std::shared_ptr taskManager = std::make_shared(logger); std::shared_ptr collection = std::make_shared(secondArg, "", logger, taskManager); taskManager->start(); diff --git a/src/project.cpp b/src/project.cpp deleted file mode 100644 index c1f0de8..0000000 --- a/src/project.cpp +++ /dev/null @@ -1,237 +0,0 @@ -#include "project.h" - -#include -#include - -using json = nlohmann::json; - -constexpr std::string_view entry("mason.json"); - -std::map types({ - {"simple", Dependency::Type::simple}, - {"none", Dependency::Type::simple}, - {"mason", Dependency::Type::mason}, - {"auto", Dependency::Type::automatic}, - {"automatic", Dependency::Type::automatic} -}); - -Logger* Project::logger = nullptr; - -Project::Project(const std::filesystem::path& location, const std::filesystem::path& destination): - parent(nullptr), - location(location), - destination(destination), - state(State::unknown), - name(), - dependencies(), - root(logger == nullptr) -{ - if (root) { - curl_global_init(CURL_GLOBAL_ALL); - logger = new Logger(Logger::Severity::debug); - } -} - -Project::Project(const std::filesystem::path& location, const std::filesystem::path& destination, Project* parent): - parent(parent), - location(location), - destination(destination), - state(State::unknown), - name(), - dependencies(), - root(false) {} - -Project::~Project() { - if (root) { - curl_global_cleanup(); - - delete logger; - logger = nullptr; - } -} - -bool Project::read() { - if (state == State::read) - return true; - - std::filesystem::path entryPointPath; - try { - location = std::filesystem::canonical(location); - entryPointPath = location / entry; - } catch (const std::exception& e) { - fatal(e.what()); - state = State::error; - return false; - } - - try { - std::filesystem::create_directories(destination); - destination = std::filesystem::canonical(destination); - } catch (const std::exception& e) { - fatal(e.what()); - state = State::error; - return false; - } - - std::ifstream file(entryPointPath); - if (!file.is_open()) { - fatal("couldn't open " + entryPointPath.string()); - state = State::error; - return false; - } - - json data; - try { - data = json::parse(file); - } catch (const json::exception& e) { - fatal(e.what()); - state = State::error; - return false; - } - - try { - parse(data); - } catch (const std::exception& e) { - fatal(e.what()); - state = State::error; - return false; - } - - return true; -} - -void Project::parse(const json& data) { - const json& nm = data.at("name"); - if (!nm.is_string()) - throw 1; - - name = nm; - - const json& dpd = data.at("dependencies"); - if (!dpd.is_array()) - throw 1; - - for (const json& dep : dpd) { - switch (dep.type()) { - case json::value_t::string: - createDepencencyFromString(dep); - break; - case json::value_t::object: - createDepencencyFromObject(dep); - break; - default: - throw 1; - } - } -} - -void Project::createDepencencyFromString(const std::string& entry) { - dependencies.emplace(entry, entry); -} - -void Project::createDepencencyFromObject(const nlohmann::json& entry) { - std::string url; - json::const_iterator itr = entry.find("url"); - if (itr == entry.end()) { - itr = entry.find("path"); - if (itr == entry.end()) - throw 1; - } - if (!itr->is_string()) - throw 1; - - url = *itr; - - std::optional name = std::nullopt; - itr = entry.find("name"); - if (itr != entry.end() && itr->is_string()) - name = *itr; - - std::optional version = std::nullopt; - itr = entry.find("version"); - if (itr != entry.end() && itr->is_string()) - version = *itr; - - Dependency::Type type = Dependency::Type::automatic; - itr = entry.find("type"); - if (itr != entry.end() && itr->is_string()) { - std::map::const_iterator titr = types.find(*itr); - if (titr != types.end()) - type = titr->second; - } - - dependencies.emplace(url, Dependency{url, type, name, version}); -} - -void Project::discover() { - int fine = 0; - for (std::pair& pair : dependencies) { - Dependency& dep = pair.second; - bool success = dep.prepare(location, destination); - if (success) { - switch (dep.getType()) { - case Dependency::Type::mason: - break; - case Dependency::Type::simple: - break; - default: - break; - } - - fine++; - } - } - - -} - - -uint32_t Project::dependenciesCount() const { - return dependencies.size(); -} - -void Project::log(Logger::Severity severity, const std::string& message) { - if (logger != nullptr) - logger->log(severity, message); -} - -void Project::info(const std::string& message) { - log(Logger::Severity::info, message); -} - -void Project::debug(const std::string& message) { - log(Logger::Severity::debug, message); -} - -void Project::minor(const std::string& message) { - log(Logger::Severity::minor, message); -} - -void Project::major(const std::string& message) { - log(Logger::Severity::major, message); -} - -void Project::error(const std::string& message) { - log(Logger::Severity::error, message); -} - -void Project::warn(const std::string& message) { - log(Logger::Severity::warning, message); -} - -void Project::fatal(const std::string& message) { - log(Logger::Severity::fatal, message); -} - -void Project::printLog() { - if (logger != nullptr) - logger->printLog(); -} - -Project::State Project::getStatus() const { - return state; -} - -std::string Project::getName() const { - return name; -} diff --git a/src/project.h b/src/project.h deleted file mode 100644 index 97da3c2..0000000 --- a/src/project.h +++ /dev/null @@ -1,64 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "dependency.h" -#include "loggger.h" - -class Project { -public: - enum class State { - unknown, - read, - discovering, - error - }; - - Project(const std::filesystem::path& location, const std::filesystem::path& destination); - ~Project(); - - bool read(); - void discover(); - State getStatus() const; - std::string getName() const; - uint32_t dependenciesCount() const; - - static void log(Logger::Severity severity, const std::string& message); - static void debug(const std::string& message); - static void info(const std::string& message); - static void minor(const std::string& message); - static void major(const std::string& message); - static void warn(const std::string& message); - static void error(const std::string& message); - static void fatal(const std::string& message); - static void printLog(); - - static void addProject(const std::filesystem::path& location, const std::filesystem::path& destination); - -private: - Project(const std::filesystem::path& location, const std::filesystem::path& destination, Project* parent); - - void parse(const nlohmann::json& json); - void createDepencencyFromString(const std::string& entry); - void createDepencencyFromObject(const nlohmann::json& entry); - -private: - Project* parent; - std::filesystem::path location; - std::filesystem::path destination; - State state; - std::string name; - std::map dependencies; - const bool root; - - static Logger* logger; - -}; diff --git a/src2/taskmanager.cpp b/src/taskmanager.cpp similarity index 81% rename from src2/taskmanager.cpp rename to src/taskmanager.cpp index 80430f7..a13c5e2 100644 --- a/src2/taskmanager.cpp +++ b/src/taskmanager.cpp @@ -1,6 +1,7 @@ #include "taskmanager.h" -TaskManager::TaskManager() : +TaskManager::TaskManager(const std::shared_ptr& logger) : + Loggable(logger), running(false), stopping(false), maxThreads(std::thread::hardware_concurrency()), @@ -38,6 +39,7 @@ void TaskManager::start() { if (running) return; + debug("Starting " + std::to_string(maxThreads) + " threads"); for (uint32_t i = 0; i < maxThreads; ++i) threads.emplace_back(&TaskManager::loop, this); @@ -49,6 +51,7 @@ void TaskManager::stop() { if (!running) return; + debug("Stopping task manager"); stopping = true; lock.unlock(); @@ -57,9 +60,12 @@ void TaskManager::stop() { thread.join(); threads.clear(); + running = false; } void TaskManager::loop() { + //debug("Thread " + std::to_string(std::this_thread::get_id()) + " entered the loop"); + debug("Thread entered the loop"); while (true) { Task task; std::unique_lock lock(mutex); @@ -67,11 +73,12 @@ void TaskManager::loop() { loopConditional.wait(lock); if (stopping) - return; + break; ++activeThreads; task = tasks.front(); tasks.pop(); + //debug("Thread took a task"); lock.unlock(); executeTask(task); @@ -81,7 +88,7 @@ void TaskManager::loop() { lock.unlock(); waitConditional.notify_all(); } - + debug("Thread left the loop"); } void TaskManager::executeTask(const Task& task) const { @@ -90,6 +97,8 @@ void TaskManager::executeTask(const Task& task) const { } catch (const std::exception& e) { if (task.second.has_value()) task.second.value()(&e); + + return; } try { @@ -106,6 +115,6 @@ bool TaskManager::busy() const { void TaskManager::wait() const { std::unique_lock lock(mutex); - while (activeThreads != 0) + while (activeThreads != 0 || !tasks.empty()) waitConditional.wait(lock); } diff --git a/src2/taskmanager.h b/src/taskmanager.h similarity index 87% rename from src2/taskmanager.h rename to src/taskmanager.h index 370ee1f..d4133ec 100644 --- a/src2/taskmanager.h +++ b/src/taskmanager.h @@ -9,14 +9,18 @@ #include #include #include +#include -class TaskManager { +#include "loggable.h" + +class TaskManager : public Loggable { public: using Error = std::optional; using Job = std::function; using Result = std::function; using Task = std::pair>; - TaskManager(); + + TaskManager(const std::shared_ptr& logger); ~TaskManager(); void start(); diff --git a/src2/CMakeLists.txt b/src2/CMakeLists.txt deleted file mode 100644 index e363156..0000000 --- a/src2/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -set(SOURCES - component.cpp - collection.cpp - loggable.cpp - taskmanager.cpp - download.cpp - atomicmutex.cpp -) - -set(HEADERS - component.h - collection.h - loggable.h - taskmanager.h - download.h - atomicmutex.h -) - -target_sources(${PROJECT_NAME} PRIVATE ${SOURCES}) diff --git a/test/mason.json b/test/mason.json index e6f5991..2340e64 100644 --- a/test/mason.json +++ b/test/mason.json @@ -6,7 +6,9 @@ "https://git.macaw.me/blue/mason.git", { "name" : "aProject", - "path" : "https://nowhere.to" - } + "path" : "https://nowhere/to/get.from" + }, + "https://git.macaw.me/blue/mason/archive/main.tar.gz", + "https://git.macaw.me/blue/mason/raw/branch/main/src2/atomicmutex.h" ] }