blue/mason
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactoring seems to be done, added ability to specify files as dependencies

Browse Source
This commit is contained in:
Blue 2023-09-28 16:43:35 -03:00
parent b07d017f86
commit 2b913d1d42
Signed by: blue
GPG Key ID: 9B203B252A63EE38
21 changed files with 148 additions and 793 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@ -42,7 +42,6 @@ target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/s
target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src2) target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src2)
add_subdirectory(src) add_subdirectory(src)
add_subdirectory(src2)
add_subdirectory(test) add_subdirectory(test)
target_link_libraries(${PROJECT_NAME} PRIVATE nlohmann_json::nlohmann_json) target_link_libraries(${PROJECT_NAME} PRIVATE nlohmann_json::nlohmann_json)

16
src/CMakeLists.txt
View File

@ -1,14 +1,22 @@
set(SOURCES set(SOURCES
main.cpp main.cpp
loggger.cpp loggger.cpp
project.cpp component.cpp
dependency.cpp collection.cpp
loggable.cpp
taskmanager.cpp
download.cpp
atomicmutex.cpp
) )
set(HEADERS set(HEADERS
logger.h logger.h
project.h component.h
dependency.cpp collection.h
loggable.h
taskmanager.h
download.h
atomicmutex.h
) )
target_sources(${PROJECT_NAME} PRIVATE ${SOURCES}) target_sources(${PROJECT_NAME} PRIVATE ${SOURCES})

0
src2/atomicmutex.cpp → src/atomicmutex.cpp
View File

0
src2/atomicmutex.h → src/atomicmutex.h
View File

20
src2/collection.cpp → src/collection.cpp
View File

@ -11,6 +11,8 @@ Collection::Collection(
const std::shared_ptr<TaskManager>& taskManager const std::shared_ptr<TaskManager>& taskManager
): ):
Loggable(logger), Loggable(logger),
std::enable_shared_from_this<Collection>(),
mutex(),
destination(destination), destination(destination),
target(target), target(target),
taskManager(taskManager), taskManager(taskManager),
@ -30,7 +32,8 @@ Collection::Collection(
} }
void Collection::addSource(const std::string& source) { void Collection::addSource(const std::string& source) {
if (hasSource(source)) { std::lock_guard lock(mutex);
if (_hasSource(source)) {
major("Source " + source + " is already present, skipping as duplicate"); major("Source " + source + " is already present, skipping as duplicate");
return; return;
} }
@ -43,6 +46,8 @@ void Collection::addSource(const std::string& source) {
} }
void Collection::sourceDownaloded(const std::string& source, TaskManager::Error err) { void Collection::sourceDownaloded(const std::string& source, TaskManager::Error err) {
std::lock_guard lock(mutex);
Downloads::const_iterator itr = downloadingSources.find(source); Downloads::const_iterator itr = downloadingSources.find(source);
std::optional<std::filesystem::path> location = itr->second->getLocation(); std::optional<std::filesystem::path> location = itr->second->getLocation();
downloadingSources.erase(itr); downloadingSources.erase(itr);
@ -55,6 +60,9 @@ void Collection::sourceDownaloded(const std::string& source, TaskManager::Error
} }
void Collection::sourceRead(const std::string& source, TaskManager::Error err) { void Collection::sourceRead(const std::string& source, TaskManager::Error err) {
std::lock_guard lock(mutex);
debug("Source " + source + " has been read");
Components::iterator itr = readingSources.find(source); Components::iterator itr = readingSources.find(source);
if (err.has_value()) { if (err.has_value()) {
error("Coundn't read " + source + ": " + err.value()->what()); error("Coundn't read " + source + ": " + err.value()->what());
@ -62,6 +70,7 @@ void Collection::sourceRead(const std::string& source, TaskManager::Error err) {
return; return;
} }
debug("Queuing build of " + source);
std::pair<Components::iterator, bool> res = buildingSources.emplace(source, std::move(itr->second)); std::pair<Components::iterator, bool> res = buildingSources.emplace(source, std::move(itr->second));
readingSources.erase(itr); readingSources.erase(itr);
@ -72,6 +81,8 @@ void Collection::sourceRead(const std::string& source, TaskManager::Error err) {
} }
void Collection::sourceBuilt(const std::string& source, TaskManager::Error err) { void Collection::sourceBuilt(const std::string& source, TaskManager::Error err) {
std::lock_guard lock(mutex);
debug("Source " + source + " has been built");
Components::iterator itr = buildingSources.find(source); Components::iterator itr = buildingSources.find(source);
buildingSources.erase(itr); buildingSources.erase(itr);
if (err.has_value()) { if (err.has_value()) {
@ -87,6 +98,7 @@ bool Collection::isRemote(const std::string& source) {
} }
void Collection::queueDownload(const std::string& source) { void Collection::queueDownload(const std::string& source) {
debug("Queuing download of " + source);
std::pair<Downloads::iterator, bool> res = downloadingSources.emplace(std::piecewise_construct, std::pair<Downloads::iterator, bool> res = downloadingSources.emplace(std::piecewise_construct,
std::forward_as_tuple(source), std::forward_as_tuple(source),
std::forward_as_tuple( std::forward_as_tuple(
@ -101,6 +113,7 @@ void Collection::queueDownload(const std::string& source) {
} }
void Collection::queueRead(const std::string& source, const std::filesystem::path& location) { void Collection::queueRead(const std::string& source, const std::filesystem::path& location) {
debug("Queuing read of " + source);
std::pair<Components::iterator, bool> res = readingSources.emplace(std::piecewise_construct, std::pair<Components::iterator, bool> res = readingSources.emplace(std::piecewise_construct,
std::forward_as_tuple(source), std::forward_as_tuple(source),
std::forward_as_tuple( std::forward_as_tuple(
@ -115,5 +128,10 @@ void Collection::queueRead(const std::string& source, const std::filesystem::pat
} }
bool Collection::hasSource(const std::string& source) const { bool Collection::hasSource(const std::string& source) const {
std::lock_guard lock(mutex);
return _hasSource(source);
}
bool Collection::_hasSource(const std::string& source) const {
return knownSources.count(source) > 0; return knownSources.count(source) > 0;
} }

4
src2/collection.h → src/collection.h
View File

@ -7,6 +7,7 @@
#include <exception> #include <exception>
#include <string_view> #include <string_view>
#include <regex> #include <regex>
#include <mutex>
#include "loggable.h" #include "loggable.h"
#include "component.h" #include "component.h"
@ -44,7 +45,10 @@ private:
void queueDownload(const std::string& source); void queueDownload(const std::string& source);
void queueRead(const std::string& source, const std::filesystem::path& location); void queueRead(const std::string& source, const std::filesystem::path& location);
bool _hasSource(const std::string& source) const;
private: private:
mutable std::mutex mutex;
std::filesystem::path destination; std::filesystem::path destination;
std::string target; std::string target;
std::shared_ptr<TaskManager> taskManager; std::shared_ptr<TaskManager> taskManager;

0
src2/component.cpp → src/component.cpp
View File

0
src2/component.h → src/component.h
View File

375
src/dependency.cpp
View File

@ -1,375 +0,0 @@
#include "dependency.h"
#include <regex>
#include "project.h"
constexpr std::string_view downloads("downloads");
constexpr std::string_view sources("sources");
constexpr std::string_view build("build");
constexpr std::string_view acceptJson("accept: application/json");
const std::regex http("^https?:\\/\\/");
const std::regex repo("(^https?):\\/\\/([\\w\\d\\.\\-\\_]+)\\/([\\w\\d\\-\\_]+)\\/([\\w\\d\\-\\_]+)");
Dependency::Dependency(
const std::string& path,
Type type,
const std::optional<std::string>& name,
const std::optional<std::string>& version
):
path(path),
type(type),
name(name),
version(version),
location()
{}
Dependency::Type Dependency::getType() const {
return type;
}
std::optional<std::string> Dependency::getName() const {
return name;
}
std::optional<std::string> Dependency::getVersion() const {
return version;
}
std::optional<std::filesystem::path> Dependency::getLocation() const {
if (location.empty())
return std::nullopt;
else
return location;
}
bool Dependency::prepare(const std::filesystem::path& source, const std::filesystem::path& destination) {
std::smatch results;
if (std::regex_search(path, results, repo)) {
Project::info(path + " appears to be a git repository");
const std::string& protocol = results[1];
const std::string& host = results[2];
const std::string& owner = results[3];
const std::string& repo = results[4];
std::filesystem::path temp;
bool success = downloadRepo(destination, protocol, host, owner, repo, temp);
if (success) {
Project::info("Successfully obtained project " + path);
switch (type) {
case Type::automatic:
if (std::filesystem::exists(temp/"mason.json"))
type = Type::mason;
else
type = Type::simple;
break;
case Type::mason:
if (std::filesystem::exists(temp/"mason.json"))
break;
else {
Project::warn("Project " + path + " is supposed to me a mason project, but no mason.json was found in the project directory");
success = false;
}
break;
case Type::simple:
break;
}
}
return success;
} else {
Project::info("checking project at path " + path);
std::filesystem::directory_entry srcDir(source / path);
if (!srcDir.exists()) {
Project::error("Project at " + path + " doesn't exist");
return false;
}
}
return true;
}
bool Dependency::downloadRepo(
const std::filesystem::path& destination,
const std::string& protocol,
const std::string& host,
const std::string& owner,
const std::string& repo,
std::filesystem::path& out
) {
nlohmann::json info;
Project::info("Trying Gitea v1 API");
bool res = repoGiteaApi1(protocol, host, owner, repo, info);
if (res) {
nlohmann::json::const_iterator itr = info.find("default_branch");
if (itr != info.end() && itr->is_string()) {
std::string branchName = *itr;
Project::info("Gitea v1 API seem to have worked");
Project::info("Default branch is " + branchName);
std::string fileName = branchName + ".tar.gz";
std::string url = protocol + "://" + host + "/api/v1/repos/" + owner + "/" + repo + "/archive/" + fileName;
std::filesystem::path archivePath = destination/downloads/fileName;
res = download(url, archivePath);
if (res) {
Project::info("Successfully downloaded " + archivePath.string());
location = destination/sources;
res = extract(archivePath, location);
if (res) {
out = destination/build/repo;
if (!std::filesystem::is_directory(out)) {
Project::error("Extracted archive " + fileName + " but the content is unexpected");
res = false;
}
} else {
Project::error("Couldn't extract archive " + fileName);
}
if (!res)
location.clear();
return res;
}
} else {
res = false;
}
}
if (!res) {
Project::warn("Gitea v1 API didn't work");
}
return res;
}
bool Dependency::repoGiteaApi1(
const std::string& protocol,
const std::string& host,
const std::string& owner,
const std::string& repo,
nlohmann::json& json
) {
bool result = false;
std::string url = protocol + "://" + host + "/api/v1/repos/" + owner + "/" + repo;
std::string data;
CURLcode code = httpGet(url, data, {acceptJson});
if (code == CURLE_OK) {
try {
json = nlohmann::json::parse(data);
result = true;
} catch (const nlohmann::json::exception& e) {
Project::warn(e.what());
}
}
return result;
}
CURLcode Dependency::httpGet(const std::string& url, std::string& result, const std::list<std::string_view>& headers) {
CURL* curl = curl_easy_init();
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L);
struct curl_slist* curlHeaders = nullptr;
for (const std::string_view& header : headers)
curlHeaders = curl_slist_append(curlHeaders, header.data());
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curlHeaders);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeString);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &result);
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, trace);
curl_easy_setopt(curl, CURLOPT_DEBUGDATA, this);
CURLcode res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
return res;
}
bool Dependency::download(const std::string& url, const std::filesystem::path& destination) {
std::filesystem::create_directories(destination.parent_path());
if (std::filesystem::exists(destination)) {
Project::minor("File " + destination.string() + " already exists, will be overwritten");
if (std::filesystem::is_directory(destination))
std::filesystem::remove_all(destination);
}
bool result = false;
CURL* curl = curl_easy_init();
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeFile);
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, trace);
curl_easy_setopt(curl, CURLOPT_DEBUGDATA, this);
FILE* pagefile = fopen(destination.c_str(), "wb");
if (pagefile) {
curl_easy_setopt(curl, CURLOPT_WRITEDATA, pagefile);
CURLcode res = curl_easy_perform(curl);
fclose(pagefile);
if (res != CURLE_OK) {
Project::warn("Couldn't download file " + url + ": " + curl_easy_strerror(res));
} else {
uint32_t code;
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code);
if (code == 200)
result = true;
else
Project::warn("Couldn't download file " + url + ": response code " + std::to_string(code));
}
} else {
Project::error(std::string("Couldn't open output file ") + destination.c_str());
}
curl_easy_cleanup(curl);
if (!result) {
Project::minor("Removing " + destination.string() + " since the donwload failed");
std::filesystem::remove_all(destination);
}
return result;
}
size_t Dependency::writeString(void* data, size_t size, size_t nmemb, void* pointer) {
size_t finalSize = size * nmemb;
std::string* string = static_cast<std::string*>(pointer);
string->append(static_cast<char*>(data), finalSize);
return finalSize;
}
size_t Dependency::writeFile(void* data, size_t size, size_t nmemb, void* file) {
size_t written = fwrite(data, size, nmemb, (FILE *)file);
return written;
}
int Dependency::trace(CURL* handle, curl_infotype type, char* data, size_t size, void* clientp) {
switch (type) {
case CURLINFO_TEXT: {
std::string message(data, size);
if (message[size - 1] == '\n')
message = message.substr(0, size - 1);
Project::debug(message);
} break;
default:
break;
}
return 0;
}
bool Dependency::extract(const std::filesystem::path& source, const std::filesystem::path& destination) const {
int flags = ARCHIVE_EXTRACT_TIME;
flags |= ARCHIVE_EXTRACT_PERM;
flags |= ARCHIVE_EXTRACT_ACL;
flags |= ARCHIVE_EXTRACT_FFLAGS;
struct archive* a = archive_read_new();
struct archive* ext = archive_write_disk_new();
struct archive_entry *entry;
archive_read_support_format_all(a);
archive_read_support_filter_all(a);
archive_write_disk_set_options(ext, flags);
archive_write_disk_set_standard_lookup(ext);
bool result = true;
bool readOpen = false;
bool writeOpen = false;
int r = archive_read_open_filename(a, source.c_str(), 10240);
if (r) {
Project::major("Couldn't open file " + source.string());
result = false;
} else {
readOpen = true;
}
while (result) {
r = archive_read_next_header(a, &entry);
if (r == ARCHIVE_EOF)
break;
if (r < ARCHIVE_OK)
Project::major(archive_error_string(a));
if (r < ARCHIVE_WARN)
break;
std::string fileName(archive_entry_pathname(entry));
std::filesystem::path filePath = destination/fileName;
Project::debug("Extracting " + filePath.string());
if (std::filesystem::exists(filePath))
Project::minor(filePath.string() + " exists, overwriting");
archive_entry_set_pathname_utf8(entry, filePath.c_str());
r = archive_write_header(ext, entry);
if (r < ARCHIVE_OK) {
Project::major(archive_error_string(ext));
} else if (archive_entry_size(entry) > 0) {
writeOpen = true;
r = copy(a, ext);
if (r < ARCHIVE_OK)
Project::major(archive_error_string(ext));
if (r < ARCHIVE_WARN)
break;
}
r = archive_write_finish_entry(ext);
if (r < ARCHIVE_OK)
Project::major(archive_error_string(ext));
if (r < ARCHIVE_WARN)
break;
}
if (readOpen)
archive_read_close(a);
if (writeOpen)
archive_write_close(ext);
archive_read_free(a);
archive_write_free(ext);
return result;
}
int Dependency::copy(struct archive* ar, struct archive* aw) const {
int r;
const void *buff;
size_t size;
la_int64_t offset;
while (true) {
r = archive_read_data_block(ar, &buff, &size, &offset);
if (r == ARCHIVE_EOF)
return ARCHIVE_OK;
if (r < ARCHIVE_OK)
return r;
r = archive_write_data_block(aw, buff, size, offset);
if (r < ARCHIVE_OK) {
Project::major(archive_error_string(aw));
return r;
}
}
}

69
src/dependency.h
View File

@ -1,69 +0,0 @@
#pragma once
#include <string>
#include <optional>
#include <filesystem>
#include <list>
#include <stdio.h>
#include <nlohmann/json.hpp>
#include <curl/curl.h>
#include <archive.h>
#include <archive_entry.h>
class Dependency {
public:
enum class Type {
automatic,
simple,
mason
};
Dependency(
const std::string& path,
Type type = Type::automatic,
const std::optional<std::string>& name = std::nullopt,
const std::optional<std::string>& version = std::nullopt
);
Type getType() const;
std::optional<std::string> getName() const;
std::optional<std::string> getVersion() const;
std::optional<std::filesystem::path> getLocation() const;
bool prepare(const std::filesystem::path& source, const std::filesystem::path& destination);
const std::string path;
private:
static size_t writeFile(void* data, size_t size, size_t nmemb, void* file);
static size_t writeString(void* data, size_t size, size_t nmemb, void* mem);
static int trace(CURL *handle, curl_infotype type, char *data, size_t size, void *clientp);
bool download(const std::string& url, const std::filesystem::path& destination);
bool downloadRepo(
const std::filesystem::path& destination,
const std::string& protocol,
const std::string& host,
const std::string& owner,
const std::string& repo,
std::filesystem::path& out
);
bool repoGiteaApi1(
const std::string& protocol,
const std::string& host,
const std::string& owner,
const std::string& repo,
nlohmann::json& json
);
CURLcode httpGet(const std::string& url, std::string& result, const std::list<std::string_view>& headers = {});
bool extract(const std::filesystem::path& source, const std::filesystem::path& destination) const;
int copy(struct archive *ar, struct archive *aw) const;
private:
Type type;
std::optional<std::string> name;
std::optional<std::string> version;
std::filesystem::path location;
};

91
src2/download.cpp → src/download.cpp
View File

@ -3,7 +3,10 @@
constexpr std::string_view archived("archived"); constexpr std::string_view archived("archived");
constexpr std::string_view headerAcceptJson("accept: application/json"); constexpr std::string_view headerAcceptJson("accept: application/json");
static const std::regex repo("(^https?):\\/\\/([\\w\\d\\.\\-\\_]+)\\/([\\w\\d\\-\\_]+)\\/([\\w\\d\\-\\_]+)"); static const std::regex file("(^https?):\\/\\/(?:[\\w\\d\\.\\-\\_]+\\/)+([\\w\\d\\-\\_]+)\\.([\\w\\d\\.]+)");
static const std::regex repo("(^https?):\\/\\/([\\w\\d\\.\\-\\_]+)\\/([\\w\\d\\-\\_]+)\\/([\\w\\d\\-\\_]+)(?:(?:\\.git)|\\/)?$");
static const std::regex archive("tar\\.gz$");
unsigned int Download::instances(0); unsigned int Download::instances(0);
AtomicMutex Download::amx{}; AtomicMutex Download::amx{};
@ -50,23 +53,47 @@ std::optional<std::filesystem::path> Download::getLocation() const {
} }
void Download::proceed() { void Download::proceed() {
bool success = false;
std::optional<RepoInfo> repo = testRepo(); std::optional<RepoInfo> repo = testRepo();
std::optional<std::filesystem::path> path; if (repo.has_value()) {
if (repo.has_value()) const RepoInfo& rp = repo.value();
path = downloadAsRepo(repo.value()); std::optional<std::filesystem::path> path = downloadAsRepo(rp);
if (path.has_value()) {
info(url + " has been successfully donwloaded as a repository, extracting"); info(url + " has been successfully donwloaded as a repository, extracting");
success = extract(path.value(), destination/repo.value().name); bool success = extract(path.value(), destination);
if (success) if (success) {
location = destination/repo.value().name; location = destination/rp.name;
info("Successfully extracted " + url);
return;
}
} }
if (success) std::optional<FileInfo> file = testFile();
info("Successfully extracted " + url); if (file.has_value()) {
const FileInfo& fl = file.value();
std::optional<std::filesystem::path> path = downloadAsFile(fl);
if (path.has_value()) {
bool success = true;
info("Successfully downloaded " + url);
if (fl.archive) {
info(url + " appears to be an archive, extracting");
success = extract(path.value(), destination/fl.name);
if (success)
info("Successfully extracted " + url);
}
if (success) {
location = destination/fl.name;
return;
}
}
}
// CURLcode code = httpDownload(url, path);
// if (code == CURLE_OK)
// return path;
} }
std::optional<nlohmann::json> Download::repoInfoGiteaApi1(const RepoInfo& repo) { std::optional<nlohmann::json> Download::repoInfoGiteaApi1(const RepoInfo& repo) {
std::string url = repo.origin() + "/api/v1/repos/" + repo.project(); std::string url = repo.origin() + "/api/v1/repos/" + repo.project();
std::string data; std::string data;
@ -99,6 +126,24 @@ std::optional<std::filesystem::path> Download::downloadAsRepo(const RepoInfo& re
return std::nullopt; return std::nullopt;
} }
std::optional<std::filesystem::path> Download::downloadAsFile(const FileInfo& file) {
std::filesystem::path dst;
if (file.archive)
dst = destination/archived/file.fileName();
else
dst = destination/file.fileName();
CURLcode code = httpDownload(url, dst);
if (code == CURLE_OK)
return dst;
minor("Removing " + dst.string());
if (std::filesystem::exists(dst))
std::filesystem::remove_all(dst);
return std::nullopt;
}
std::optional<std::filesystem::path> Download::downloadRepoGiteaApi1(const RepoInfo& repo, const std::string& branch) { std::optional<std::filesystem::path> Download::downloadRepoGiteaApi1(const RepoInfo& repo, const std::string& branch) {
std::string fileName = branch + ".tar.gz"; std::string fileName = branch + ".tar.gz";
std::string url = repo.origin() + "/api/v1/repos/" + repo.project() + "/archive/" + fileName; std::string url = repo.origin() + "/api/v1/repos/" + repo.project() + "/archive/" + fileName;
@ -130,7 +175,7 @@ CURLcode Download::httpGet(
} }
CURLcode Download::httpDownload(const std::string& url, const std::filesystem::path& path, const std::vector<std::string_view>& headers) { CURLcode Download::httpDownload(const std::string& url, const std::filesystem::path& path, const std::vector<std::string_view>& headers) {
info("Starting to download " + url); info("Starting to download " + url + " to " + path.string());
std::filesystem::create_directories(path.parent_path()); std::filesystem::create_directories(path.parent_path());
if (std::filesystem::exists(path)) { if (std::filesystem::exists(path)) {
minor("File " + path.string() + " already exists, will be overwritten"); minor("File " + path.string() + " already exists, will be overwritten");
@ -185,7 +230,7 @@ void Download::createCurl() {
if (!crl) if (!crl)
throw CurlError("Error creating curl instalce"); throw CurlError("Error creating curl instalce");
curl_easy_setopt(crl, CURLOPT_USERAGENT, "libcurl-agent/1.0" ); curl_easy_setopt(crl, CURLOPT_USERAGENT, "libcurl-agent/1.0" );
curl_easy_setopt(crl, CURLOPT_VERBOSE, 1L); curl_easy_setopt(crl, CURLOPT_VERBOSE, 1L);
curl_easy_setopt(crl, CURLOPT_DEBUGFUNCTION, trace); curl_easy_setopt(crl, CURLOPT_DEBUGFUNCTION, trace);
curl_easy_setopt(crl, CURLOPT_DEBUGDATA, this); curl_easy_setopt(crl, CURLOPT_DEBUGDATA, this);
@ -194,6 +239,21 @@ void Download::createCurl() {
curl = std::unique_ptr<CURL, CurlDeleter>(crl); curl = std::unique_ptr<CURL, CurlDeleter>(crl);
} }
std::optional<Download::FileInfo> Download::testFile() const {
std::smatch results;
if (std::regex_search(url, results, file)) {
info(url + " appears to be a file");
std::string ext(results[3]);
return FileInfo {
results[1],
results[2],
ext,
std::regex_search(ext, archive)
};
}
return std::nullopt;
}
std::optional<Download::RepoInfo> Download::testRepo() const { std::optional<Download::RepoInfo> Download::testRepo() const {
std::smatch results; std::smatch results;
@ -347,4 +407,7 @@ std::string Download::RepoInfo::project() const {
return owner + "/" + name; return owner + "/" + name;
} }
std::string Download::FileInfo::fileName() const {
return name + "." + extension;
}

12
src2/download.h → src/download.h
View File

@ -22,6 +22,7 @@ class Collection;
class Download : protected Loggable { class Download : protected Loggable {
struct RepoInfo; struct RepoInfo;
struct FileInfo;
struct CurlDeleter { struct CurlDeleter {
void operator() (CURL* curl) const; void operator() (CURL* curl) const;
}; };
@ -43,9 +44,11 @@ public:
std::optional<std::filesystem::path> getLocation() const; std::optional<std::filesystem::path> getLocation() const;
private: private:
std::optional<FileInfo> testFile() const;
std::optional<RepoInfo> testRepo() const; std::optional<RepoInfo> testRepo() const;
std::optional<std::filesystem::path> downloadAsRepo(const RepoInfo& repo); std::optional<std::filesystem::path> downloadAsRepo(const RepoInfo& repo);
std::optional<std::filesystem::path> downloadAsFile(const FileInfo& file);
std::optional<nlohmann::json> repoInfoGiteaApi1(const RepoInfo& repo); std::optional<nlohmann::json> repoInfoGiteaApi1(const RepoInfo& repo);
std::optional<std::filesystem::path> downloadRepoGiteaApi1(const RepoInfo& repo, const std::string& branch); std::optional<std::filesystem::path> downloadRepoGiteaApi1(const RepoInfo& repo, const std::string& branch);
@ -80,6 +83,15 @@ struct Download::RepoInfo {
std::string project() const; std::string project() const;
}; };
struct Download::FileInfo {
std::string protocol;
std::string name;
std::string extension;
bool archive;
std::string fileName() const;
};
class Download::CurlError : public std::runtime_error { class Download::CurlError : public std::runtime_error {
public: public:
explicit CurlError(const std::string& message); explicit CurlError(const std::string& message);

0
src2/loggable.cpp → src/loggable.cpp
View File

0
src2/loggable.h → src/loggable.h
View File

2
src/main.cpp
View File

@ -19,7 +19,7 @@ int main(int argc, char *argv[]) {
std::shared_ptr<Logger> logger = std::make_shared<Logger>(Logger::Severity::debug); std::shared_ptr<Logger> logger = std::make_shared<Logger>(Logger::Severity::debug);
std::shared_ptr<TaskManager> taskManager = std::make_shared<TaskManager>(); std::shared_ptr<TaskManager> taskManager = std::make_shared<TaskManager>(logger);
std::shared_ptr<Collection> collection = std::make_shared<Collection>(secondArg, "", logger, taskManager); std::shared_ptr<Collection> collection = std::make_shared<Collection>(secondArg, "", logger, taskManager);
taskManager->start(); taskManager->start();

237
src/project.cpp
View File

@ -1,237 +0,0 @@
#include "project.h"
#include <string_view>
#include <exception>
using json = nlohmann::json;
constexpr std::string_view entry("mason.json");
std::map<std::string, Dependency::Type> types({
{"simple", Dependency::Type::simple},
{"none", Dependency::Type::simple},
{"mason", Dependency::Type::mason},
{"auto", Dependency::Type::automatic},
{"automatic", Dependency::Type::automatic}
});
Logger* Project::logger = nullptr;
Project::Project(const std::filesystem::path& location, const std::filesystem::path& destination):
parent(nullptr),
location(location),
destination(destination),
state(State::unknown),
name(),
dependencies(),
root(logger == nullptr)
{
if (root) {
curl_global_init(CURL_GLOBAL_ALL);
logger = new Logger(Logger::Severity::debug);
}
}
Project::Project(const std::filesystem::path& location, const std::filesystem::path& destination, Project* parent):
parent(parent),
location(location),
destination(destination),
state(State::unknown),
name(),
dependencies(),
root(false) {}
Project::~Project() {
if (root) {
curl_global_cleanup();
delete logger;
logger = nullptr;
}
}
bool Project::read() {
if (state == State::read)
return true;
std::filesystem::path entryPointPath;
try {
location = std::filesystem::canonical(location);
entryPointPath = location / entry;
} catch (const std::exception& e) {
fatal(e.what());
state = State::error;
return false;
}
try {
std::filesystem::create_directories(destination);
destination = std::filesystem::canonical(destination);
} catch (const std::exception& e) {
fatal(e.what());
state = State::error;
return false;
}
std::ifstream file(entryPointPath);
if (!file.is_open()) {
fatal("couldn't open " + entryPointPath.string());
state = State::error;
return false;
}
json data;
try {
data = json::parse(file);
} catch (const json::exception& e) {
fatal(e.what());
state = State::error;
return false;
}
try {
parse(data);
} catch (const std::exception& e) {
fatal(e.what());
state = State::error;
return false;
}
return true;
}
void Project::parse(const json& data) {
const json& nm = data.at("name");
if (!nm.is_string())
throw 1;
name = nm;
const json& dpd = data.at("dependencies");
if (!dpd.is_array())
throw 1;
for (const json& dep : dpd) {
switch (dep.type()) {
case json::value_t::string:
createDepencencyFromString(dep);
break;
case json::value_t::object:
createDepencencyFromObject(dep);
break;
default:
throw 1;
}
}
}
void Project::createDepencencyFromString(const std::string& entry) {
dependencies.emplace(entry, entry);
}
void Project::createDepencencyFromObject(const nlohmann::json& entry) {
std::string url;
json::const_iterator itr = entry.find("url");
if (itr == entry.end()) {
itr = entry.find("path");
if (itr == entry.end())
throw 1;
}
if (!itr->is_string())
throw 1;
url = *itr;
std::optional<std::string> name = std::nullopt;
itr = entry.find("name");
if (itr != entry.end() && itr->is_string())
name = *itr;
std::optional<std::string> version = std::nullopt;
itr = entry.find("version");
if (itr != entry.end() && itr->is_string())
version = *itr;
Dependency::Type type = Dependency::Type::automatic;
itr = entry.find("type");
if (itr != entry.end() && itr->is_string()) {
std::map<std::string, Dependency::Type>::const_iterator titr = types.find(*itr);
if (titr != types.end())
type = titr->second;
}
dependencies.emplace(url, Dependency{url, type, name, version});
}
void Project::discover() {
int fine = 0;
for (std::pair<const std::string, Dependency>& pair : dependencies) {
Dependency& dep = pair.second;
bool success = dep.prepare(location, destination);
if (success) {
switch (dep.getType()) {
case Dependency::Type::mason:
break;
case Dependency::Type::simple:
break;
default:
break;
}
fine++;
}
}
}
uint32_t Project::dependenciesCount() const {
return dependencies.size();
}
void Project::log(Logger::Severity severity, const std::string& message) {
if (logger != nullptr)
logger->log(severity, message);
}
void Project::info(const std::string& message) {
log(Logger::Severity::info, message);
}
void Project::debug(const std::string& message) {
log(Logger::Severity::debug, message);
}
void Project::minor(const std::string& message) {
log(Logger::Severity::minor, message);
}
void Project::major(const std::string& message) {
log(Logger::Severity::major, message);
}
void Project::error(const std::string& message) {
log(Logger::Severity::error, message);
}
void Project::warn(const std::string& message) {
log(Logger::Severity::warning, message);
}
void Project::fatal(const std::string& message) {
log(Logger::Severity::fatal, message);
}
void Project::printLog() {
if (logger != nullptr)
logger->printLog();
}
Project::State Project::getStatus() const {
return state;
}
std::string Project::getName() const {
return name;
}

64
src/project.h
View File

@ -1,64 +0,0 @@
#pragma once
#include <filesystem>
#include <string>
#include <fstream>
#include <list>
#include <map>
#include <stdint.h>
#include <nlohmann/json.hpp>
#include <curl/curl.h>
#include "dependency.h"
#include "loggger.h"
class Project {
public:
enum class State {
unknown,
read,
discovering,
error
};
Project(const std::filesystem::path& location, const std::filesystem::path& destination);
~Project();
bool read();
void discover();
State getStatus() const;
std::string getName() const;
uint32_t dependenciesCount() const;
static void log(Logger::Severity severity, const std::string& message);
static void debug(const std::string& message);
static void info(const std::string& message);
static void minor(const std::string& message);
static void major(const std::string& message);
static void warn(const std::string& message);
static void error(const std::string& message);
static void fatal(const std::string& message);
static void printLog();
static void addProject(const std::filesystem::path& location, const std::filesystem::path& destination);
private:
Project(const std::filesystem::path& location, const std::filesystem::path& destination, Project* parent);
void parse(const nlohmann::json& json);
void createDepencencyFromString(const std::string& entry);
void createDepencencyFromObject(const nlohmann::json& entry);
private:
Project* parent;
std::filesystem::path location;
std::filesystem::path destination;
State state;
std::string name;
std::map<std::string, Dependency> dependencies;
const bool root;
static Logger* logger;
};

17
src2/taskmanager.cpp → src/taskmanager.cpp
View File

@ -1,6 +1,7 @@
#include "taskmanager.h" #include "taskmanager.h"
TaskManager::TaskManager() : TaskManager::TaskManager(const std::shared_ptr<Logger>& logger) :
Loggable(logger),
running(false), running(false),
stopping(false), stopping(false),
maxThreads(std::thread::hardware_concurrency()), maxThreads(std::thread::hardware_concurrency()),
@ -38,6 +39,7 @@ void TaskManager::start() {
if (running) if (running)
return; return;
debug("Starting " + std::to_string(maxThreads) + " threads");
for (uint32_t i = 0; i < maxThreads; ++i) for (uint32_t i = 0; i < maxThreads; ++i)
threads.emplace_back(&TaskManager::loop, this); threads.emplace_back(&TaskManager::loop, this);
@ -49,6 +51,7 @@ void TaskManager::stop() {
if (!running) if (!running)
return; return;
debug("Stopping task manager");
stopping = true; stopping = true;
lock.unlock(); lock.unlock();
@ -57,9 +60,12 @@ void TaskManager::stop() {
thread.join(); thread.join();
threads.clear(); threads.clear();
running = false;
} }
void TaskManager::loop() { void TaskManager::loop() {
//debug("Thread " + std::to_string(std::this_thread::get_id()) + " entered the loop");
debug("Thread entered the loop");
while (true) { while (true) {
Task task; Task task;
std::unique_lock lock(mutex); std::unique_lock lock(mutex);
@ -67,11 +73,12 @@ void TaskManager::loop() {
loopConditional.wait(lock); loopConditional.wait(lock);
if (stopping) if (stopping)
return; break;
++activeThreads; ++activeThreads;
task = tasks.front(); task = tasks.front();
tasks.pop(); tasks.pop();
//debug("Thread took a task");
lock.unlock(); lock.unlock();
executeTask(task); executeTask(task);
@ -81,7 +88,7 @@ void TaskManager::loop() {
lock.unlock(); lock.unlock();
waitConditional.notify_all(); waitConditional.notify_all();
} }
debug("Thread left the loop");
} }
void TaskManager::executeTask(const Task& task) const { void TaskManager::executeTask(const Task& task) const {
@ -90,6 +97,8 @@ void TaskManager::executeTask(const Task& task) const {
} catch (const std::exception& e) { } catch (const std::exception& e) {
if (task.second.has_value()) if (task.second.has_value())
task.second.value()(&e); task.second.value()(&e);
return;
} }
try { try {
@ -106,6 +115,6 @@ bool TaskManager::busy() const {
void TaskManager::wait() const { void TaskManager::wait() const {
std::unique_lock lock(mutex); std::unique_lock lock(mutex);
while (activeThreads != 0) while (activeThreads != 0 || !tasks.empty())
waitConditional.wait(lock); waitConditional.wait(lock);
} }

8
src2/taskmanager.h → src/taskmanager.h
View File

@ -9,14 +9,18 @@
#include <functional> #include <functional>
#include <optional> #include <optional>
#include <exception> #include <exception>
#include <memory>
class TaskManager { #include "loggable.h"
class TaskManager : public Loggable {
public: public:
using Error = std::optional<const std::exception*>; using Error = std::optional<const std::exception*>;
using Job = std::function<void ()>; using Job = std::function<void ()>;
using Result = std::function<void (Error)>; using Result = std::function<void (Error)>;
using Task = std::pair<Job, std::optional<Result>>; using Task = std::pair<Job, std::optional<Result>>;
TaskManager();
TaskManager(const std::shared_ptr<Logger>& logger);
~TaskManager(); ~TaskManager();
void start(); void start();

19
src2/CMakeLists.txt
View File

@ -1,19 +0,0 @@
set(SOURCES
component.cpp
collection.cpp
loggable.cpp
taskmanager.cpp
download.cpp
atomicmutex.cpp
)
set(HEADERS
component.h
collection.h
loggable.h
taskmanager.h
download.h
atomicmutex.h
)
target_sources(${PROJECT_NAME} PRIVATE ${SOURCES})

6
test/mason.json
View File

@ -6,7 +6,9 @@
"https://git.macaw.me/blue/mason.git", "https://git.macaw.me/blue/mason.git",
{ {
"name" : "aProject", "name" : "aProject",
"path" : "https://nowhere.to" "path" : "https://nowhere/to/get.from"
} },
"https://git.macaw.me/blue/mason/archive/main.tar.gz",
"https://git.macaw.me/blue/mason/raw/branch/main/src2/atomicmutex.h"
] ]
} }