refactoring seems to be done, added ability to specify files as dependencies
This commit is contained in:
parent
b07d017f86
commit
2b913d1d42
GPG Key ID:
9B203B252A63EE38
|
|
@ -42,7 +42,6 @@ target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/s
|
|||
target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src2)
|
||||
|
||||
add_subdirectory(src)
|
||||
add_subdirectory(src2)
|
||||
add_subdirectory(test)
|
||||
|
||||
target_link_libraries(${PROJECT_NAME} PRIVATE nlohmann_json::nlohmann_json)
|
||||
|
|
|
|||
|
|
@ -1,14 +1,22 @@
|
|||
set(SOURCES
|
||||
main.cpp
|
||||
loggger.cpp
|
||||
project.cpp
|
||||
dependency.cpp
|
||||
component.cpp
|
||||
collection.cpp
|
||||
loggable.cpp
|
||||
taskmanager.cpp
|
||||
download.cpp
|
||||
atomicmutex.cpp
|
||||
)
|
||||
|
||||
set(HEADERS
|
||||
logger.h
|
||||
project.h
|
||||
dependency.cpp
|
||||
component.h
|
||||
collection.h
|
||||
loggable.h
|
||||
taskmanager.h
|
||||
download.h
|
||||
atomicmutex.h
|
||||
)
|
||||
|
||||
target_sources(${PROJECT_NAME} PRIVATE ${SOURCES})
|
||||
|
|
|
|||
|
|
@ -11,6 +11,8 @@ Collection::Collection(
|
|||
const std::shared_ptr<TaskManager>& taskManager
|
||||
):
|
||||
Loggable(logger),
|
||||
std::enable_shared_from_this<Collection>(),
|
||||
mutex(),
|
||||
destination(destination),
|
||||
target(target),
|
||||
taskManager(taskManager),
|
||||
|
|
@ -30,7 +32,8 @@ Collection::Collection(
|
|||
}
|
||||
|
||||
void Collection::addSource(const std::string& source) {
|
||||
if (hasSource(source)) {
|
||||
std::lock_guard lock(mutex);
|
||||
if (_hasSource(source)) {
|
||||
major("Source " + source + " is already present, skipping as duplicate");
|
||||
return;
|
||||
}
|
||||
|
|
@ -43,6 +46,8 @@ void Collection::addSource(const std::string& source) {
|
|||
}
|
||||
|
||||
void Collection::sourceDownaloded(const std::string& source, TaskManager::Error err) {
|
||||
std::lock_guard lock(mutex);
|
||||
|
||||
Downloads::const_iterator itr = downloadingSources.find(source);
|
||||
std::optional<std::filesystem::path> location = itr->second->getLocation();
|
||||
downloadingSources.erase(itr);
|
||||
|
|
@ -55,6 +60,9 @@ void Collection::sourceDownaloded(const std::string& source, TaskManager::Error
|
|||
}
|
||||
|
||||
void Collection::sourceRead(const std::string& source, TaskManager::Error err) {
|
||||
std::lock_guard lock(mutex);
|
||||
|
||||
debug("Source " + source + " has been read");
|
||||
Components::iterator itr = readingSources.find(source);
|
||||
if (err.has_value()) {
|
||||
error("Coundn't read " + source + ": " + err.value()->what());
|
||||
|
|
@ -62,6 +70,7 @@ void Collection::sourceRead(const std::string& source, TaskManager::Error err) {
|
|||
return;
|
||||
}
|
||||
|
||||
debug("Queuing build of " + source);
|
||||
std::pair<Components::iterator, bool> res = buildingSources.emplace(source, std::move(itr->second));
|
||||
readingSources.erase(itr);
|
||||
|
||||
|
|
@ -72,6 +81,8 @@ void Collection::sourceRead(const std::string& source, TaskManager::Error err) {
|
|||
}
|
||||
|
||||
void Collection::sourceBuilt(const std::string& source, TaskManager::Error err) {
|
||||
std::lock_guard lock(mutex);
|
||||
debug("Source " + source + " has been built");
|
||||
Components::iterator itr = buildingSources.find(source);
|
||||
buildingSources.erase(itr);
|
||||
if (err.has_value()) {
|
||||
|
|
@ -87,6 +98,7 @@ bool Collection::isRemote(const std::string& source) {
|
|||
}
|
||||
|
||||
void Collection::queueDownload(const std::string& source) {
|
||||
debug("Queuing download of " + source);
|
||||
std::pair<Downloads::iterator, bool> res = downloadingSources.emplace(std::piecewise_construct,
|
||||
std::forward_as_tuple(source),
|
||||
std::forward_as_tuple(
|
||||
|
|
@ -101,6 +113,7 @@ void Collection::queueDownload(const std::string& source) {
|
|||
}
|
||||
|
||||
void Collection::queueRead(const std::string& source, const std::filesystem::path& location) {
|
||||
debug("Queuing read of " + source);
|
||||
std::pair<Components::iterator, bool> res = readingSources.emplace(std::piecewise_construct,
|
||||
std::forward_as_tuple(source),
|
||||
std::forward_as_tuple(
|
||||
|
|
@ -115,5 +128,10 @@ void Collection::queueRead(const std::string& source, const std::filesystem::pat
|
|||
}
|
||||
|
||||
bool Collection::hasSource(const std::string& source) const {
|
||||
std::lock_guard lock(mutex);
|
||||
return _hasSource(source);
|
||||
}
|
||||
|
||||
bool Collection::_hasSource(const std::string& source) const {
|
||||
return knownSources.count(source) > 0;
|
||||
}
|
||||
|
|
@ -7,6 +7,7 @@
|
|||
#include <exception>
|
||||
#include <string_view>
|
||||
#include <regex>
|
||||
#include <mutex>
|
||||
|
||||
#include "loggable.h"
|
||||
#include "component.h"
|
||||
|
|
@ -44,7 +45,10 @@ private:
|
|||
void queueDownload(const std::string& source);
|
||||
void queueRead(const std::string& source, const std::filesystem::path& location);
|
||||
|
||||
bool _hasSource(const std::string& source) const;
|
||||
|
||||
private:
|
||||
mutable std::mutex mutex;
|
||||
std::filesystem::path destination;
|
||||
std::string target;
|
||||
std::shared_ptr<TaskManager> taskManager;
|
||||
|
|
@ -1,375 +0,0 @@
|
|||
#include "dependency.h"
|
||||
|
||||
#include <regex>
|
||||
|
||||
#include "project.h"
|
||||
|
||||
constexpr std::string_view downloads("downloads");
|
||||
constexpr std::string_view sources("sources");
|
||||
constexpr std::string_view build("build");
|
||||
|
||||
constexpr std::string_view acceptJson("accept: application/json");
|
||||
const std::regex http("^https?:\\/\\/");
|
||||
const std::regex repo("(^https?):\\/\\/([\\w\\d\\.\\-\\_]+)\\/([\\w\\d\\-\\_]+)\\/([\\w\\d\\-\\_]+)");
|
||||
|
||||
Dependency::Dependency(
|
||||
const std::string& path,
|
||||
Type type,
|
||||
const std::optional<std::string>& name,
|
||||
const std::optional<std::string>& version
|
||||
):
|
||||
path(path),
|
||||
type(type),
|
||||
name(name),
|
||||
version(version),
|
||||
location()
|
||||
{}
|
||||
|
||||
Dependency::Type Dependency::getType() const {
|
||||
return type;
|
||||
}
|
||||
|
||||
std::optional<std::string> Dependency::getName() const {
|
||||
return name;
|
||||
}
|
||||
|
||||
std::optional<std::string> Dependency::getVersion() const {
|
||||
return version;
|
||||
}
|
||||
|
||||
std::optional<std::filesystem::path> Dependency::getLocation() const {
|
||||
if (location.empty())
|
||||
return std::nullopt;
|
||||
else
|
||||
return location;
|
||||
}
|
||||
|
||||
bool Dependency::prepare(const std::filesystem::path& source, const std::filesystem::path& destination) {
|
||||
std::smatch results;
|
||||
if (std::regex_search(path, results, repo)) {
|
||||
Project::info(path + " appears to be a git repository");
|
||||
const std::string& protocol = results[1];
|
||||
const std::string& host = results[2];
|
||||
const std::string& owner = results[3];
|
||||
const std::string& repo = results[4];
|
||||
|
||||
std::filesystem::path temp;
|
||||
bool success = downloadRepo(destination, protocol, host, owner, repo, temp);
|
||||
if (success) {
|
||||
Project::info("Successfully obtained project " + path);
|
||||
|
||||
switch (type) {
|
||||
case Type::automatic:
|
||||
if (std::filesystem::exists(temp/"mason.json"))
|
||||
type = Type::mason;
|
||||
else
|
||||
type = Type::simple;
|
||||
|
||||
break;
|
||||
case Type::mason:
|
||||
if (std::filesystem::exists(temp/"mason.json"))
|
||||
break;
|
||||
else {
|
||||
Project::warn("Project " + path + " is supposed to me a mason project, but no mason.json was found in the project directory");
|
||||
success = false;
|
||||
}
|
||||
|
||||
break;
|
||||
case Type::simple:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return success;
|
||||
} else {
|
||||
Project::info("checking project at path " + path);
|
||||
std::filesystem::directory_entry srcDir(source / path);
|
||||
if (!srcDir.exists()) {
|
||||
Project::error("Project at " + path + " doesn't exist");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Dependency::downloadRepo(
|
||||
const std::filesystem::path& destination,
|
||||
const std::string& protocol,
|
||||
const std::string& host,
|
||||
const std::string& owner,
|
||||
const std::string& repo,
|
||||
std::filesystem::path& out
|
||||
) {
|
||||
nlohmann::json info;
|
||||
Project::info("Trying Gitea v1 API");
|
||||
bool res = repoGiteaApi1(protocol, host, owner, repo, info);
|
||||
|
||||
if (res) {
|
||||
nlohmann::json::const_iterator itr = info.find("default_branch");
|
||||
if (itr != info.end() && itr->is_string()) {
|
||||
std::string branchName = *itr;
|
||||
Project::info("Gitea v1 API seem to have worked");
|
||||
Project::info("Default branch is " + branchName);
|
||||
|
||||
std::string fileName = branchName + ".tar.gz";
|
||||
std::string url = protocol + "://" + host + "/api/v1/repos/" + owner + "/" + repo + "/archive/" + fileName;
|
||||
std::filesystem::path archivePath = destination/downloads/fileName;
|
||||
res = download(url, archivePath);
|
||||
if (res) {
|
||||
Project::info("Successfully downloaded " + archivePath.string());
|
||||
|
||||
location = destination/sources;
|
||||
res = extract(archivePath, location);
|
||||
if (res) {
|
||||
out = destination/build/repo;
|
||||
if (!std::filesystem::is_directory(out)) {
|
||||
Project::error("Extracted archive " + fileName + " but the content is unexpected");
|
||||
res = false;
|
||||
}
|
||||
} else {
|
||||
Project::error("Couldn't extract archive " + fileName);
|
||||
}
|
||||
|
||||
if (!res)
|
||||
location.clear();
|
||||
|
||||
return res;
|
||||
}
|
||||
} else {
|
||||
res = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (!res) {
|
||||
Project::warn("Gitea v1 API didn't work");
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
bool Dependency::repoGiteaApi1(
|
||||
const std::string& protocol,
|
||||
const std::string& host,
|
||||
const std::string& owner,
|
||||
const std::string& repo,
|
||||
nlohmann::json& json
|
||||
) {
|
||||
bool result = false;
|
||||
std::string url = protocol + "://" + host + "/api/v1/repos/" + owner + "/" + repo;
|
||||
std::string data;
|
||||
CURLcode code = httpGet(url, data, {acceptJson});
|
||||
if (code == CURLE_OK) {
|
||||
try {
|
||||
json = nlohmann::json::parse(data);
|
||||
result = true;
|
||||
} catch (const nlohmann::json::exception& e) {
|
||||
Project::warn(e.what());
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
CURLcode Dependency::httpGet(const std::string& url, std::string& result, const std::list<std::string_view>& headers) {
|
||||
CURL* curl = curl_easy_init();
|
||||
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
|
||||
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
|
||||
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L);
|
||||
|
||||
struct curl_slist* curlHeaders = nullptr;
|
||||
for (const std::string_view& header : headers)
|
||||
curlHeaders = curl_slist_append(curlHeaders, header.data());
|
||||
|
||||
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curlHeaders);
|
||||
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeString);
|
||||
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &result);
|
||||
|
||||
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
|
||||
curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, trace);
|
||||
curl_easy_setopt(curl, CURLOPT_DEBUGDATA, this);
|
||||
|
||||
CURLcode res = curl_easy_perform(curl);
|
||||
|
||||
curl_easy_cleanup(curl);
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
bool Dependency::download(const std::string& url, const std::filesystem::path& destination) {
|
||||
std::filesystem::create_directories(destination.parent_path());
|
||||
if (std::filesystem::exists(destination)) {
|
||||
Project::minor("File " + destination.string() + " already exists, will be overwritten");
|
||||
|
||||
if (std::filesystem::is_directory(destination))
|
||||
std::filesystem::remove_all(destination);
|
||||
}
|
||||
|
||||
bool result = false;
|
||||
CURL* curl = curl_easy_init();
|
||||
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
|
||||
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L);
|
||||
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeFile);
|
||||
|
||||
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
|
||||
curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, trace);
|
||||
curl_easy_setopt(curl, CURLOPT_DEBUGDATA, this);
|
||||
|
||||
FILE* pagefile = fopen(destination.c_str(), "wb");
|
||||
if (pagefile) {
|
||||
curl_easy_setopt(curl, CURLOPT_WRITEDATA, pagefile);
|
||||
CURLcode res = curl_easy_perform(curl);
|
||||
|
||||
fclose(pagefile);
|
||||
if (res != CURLE_OK) {
|
||||
Project::warn("Couldn't download file " + url + ": " + curl_easy_strerror(res));
|
||||
} else {
|
||||
uint32_t code;
|
||||
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code);
|
||||
if (code == 200)
|
||||
result = true;
|
||||
else
|
||||
Project::warn("Couldn't download file " + url + ": response code " + std::to_string(code));
|
||||
}
|
||||
} else {
|
||||
Project::error(std::string("Couldn't open output file ") + destination.c_str());
|
||||
}
|
||||
|
||||
curl_easy_cleanup(curl);
|
||||
|
||||
if (!result) {
|
||||
Project::minor("Removing " + destination.string() + " since the donwload failed");
|
||||
std::filesystem::remove_all(destination);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
size_t Dependency::writeString(void* data, size_t size, size_t nmemb, void* pointer) {
|
||||
size_t finalSize = size * nmemb;
|
||||
std::string* string = static_cast<std::string*>(pointer);
|
||||
string->append(static_cast<char*>(data), finalSize);
|
||||
|
||||
return finalSize;
|
||||
}
|
||||
|
||||
size_t Dependency::writeFile(void* data, size_t size, size_t nmemb, void* file) {
|
||||
size_t written = fwrite(data, size, nmemb, (FILE *)file);
|
||||
return written;
|
||||
}
|
||||
|
||||
int Dependency::trace(CURL* handle, curl_infotype type, char* data, size_t size, void* clientp) {
|
||||
switch (type) {
|
||||
case CURLINFO_TEXT: {
|
||||
std::string message(data, size);
|
||||
if (message[size - 1] == '\n')
|
||||
message = message.substr(0, size - 1);
|
||||
|
||||
Project::debug(message);
|
||||
} break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool Dependency::extract(const std::filesystem::path& source, const std::filesystem::path& destination) const {
|
||||
int flags = ARCHIVE_EXTRACT_TIME;
|
||||
flags |= ARCHIVE_EXTRACT_PERM;
|
||||
flags |= ARCHIVE_EXTRACT_ACL;
|
||||
flags |= ARCHIVE_EXTRACT_FFLAGS;
|
||||
|
||||
struct archive* a = archive_read_new();
|
||||
struct archive* ext = archive_write_disk_new();
|
||||
struct archive_entry *entry;
|
||||
archive_read_support_format_all(a);
|
||||
archive_read_support_filter_all(a);
|
||||
archive_write_disk_set_options(ext, flags);
|
||||
archive_write_disk_set_standard_lookup(ext);
|
||||
|
||||
bool result = true;
|
||||
bool readOpen = false;
|
||||
bool writeOpen = false;
|
||||
int r = archive_read_open_filename(a, source.c_str(), 10240);
|
||||
if (r) {
|
||||
Project::major("Couldn't open file " + source.string());
|
||||
result = false;
|
||||
} else {
|
||||
readOpen = true;
|
||||
}
|
||||
|
||||
while (result) {
|
||||
r = archive_read_next_header(a, &entry);
|
||||
if (r == ARCHIVE_EOF)
|
||||
break;
|
||||
|
||||
if (r < ARCHIVE_OK)
|
||||
Project::major(archive_error_string(a));
|
||||
|
||||
if (r < ARCHIVE_WARN)
|
||||
break;
|
||||
|
||||
std::string fileName(archive_entry_pathname(entry));
|
||||
std::filesystem::path filePath = destination/fileName;
|
||||
Project::debug("Extracting " + filePath.string());
|
||||
if (std::filesystem::exists(filePath))
|
||||
Project::minor(filePath.string() + " exists, overwriting");
|
||||
|
||||
archive_entry_set_pathname_utf8(entry, filePath.c_str());
|
||||
|
||||
r = archive_write_header(ext, entry);
|
||||
if (r < ARCHIVE_OK) {
|
||||
Project::major(archive_error_string(ext));
|
||||
} else if (archive_entry_size(entry) > 0) {
|
||||
writeOpen = true;
|
||||
r = copy(a, ext);
|
||||
if (r < ARCHIVE_OK)
|
||||
Project::major(archive_error_string(ext));
|
||||
|
||||
if (r < ARCHIVE_WARN)
|
||||
break;
|
||||
}
|
||||
r = archive_write_finish_entry(ext);
|
||||
if (r < ARCHIVE_OK)
|
||||
Project::major(archive_error_string(ext));
|
||||
|
||||
if (r < ARCHIVE_WARN)
|
||||
break;
|
||||
}
|
||||
|
||||
if (readOpen)
|
||||
archive_read_close(a);
|
||||
|
||||
if (writeOpen)
|
||||
archive_write_close(ext);
|
||||
|
||||
archive_read_free(a);
|
||||
archive_write_free(ext);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
int Dependency::copy(struct archive* ar, struct archive* aw) const {
|
||||
int r;
|
||||
const void *buff;
|
||||
size_t size;
|
||||
la_int64_t offset;
|
||||
|
||||
while (true) {
|
||||
r = archive_read_data_block(ar, &buff, &size, &offset);
|
||||
if (r == ARCHIVE_EOF)
|
||||
return ARCHIVE_OK;
|
||||
|
||||
if (r < ARCHIVE_OK)
|
||||
return r;
|
||||
|
||||
r = archive_write_data_block(aw, buff, size, offset);
|
||||
if (r < ARCHIVE_OK) {
|
||||
Project::major(archive_error_string(aw));
|
||||
return r;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -1,69 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <optional>
|
||||
#include <filesystem>
|
||||
#include <list>
|
||||
#include <stdio.h>
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <curl/curl.h>
|
||||
#include <archive.h>
|
||||
#include <archive_entry.h>
|
||||
|
||||
class Dependency {
|
||||
public:
|
||||
enum class Type {
|
||||
automatic,
|
||||
simple,
|
||||
mason
|
||||
};
|
||||
|
||||
Dependency(
|
||||
const std::string& path,
|
||||
Type type = Type::automatic,
|
||||
const std::optional<std::string>& name = std::nullopt,
|
||||
const std::optional<std::string>& version = std::nullopt
|
||||
);
|
||||
|
||||
Type getType() const;
|
||||
std::optional<std::string> getName() const;
|
||||
std::optional<std::string> getVersion() const;
|
||||
std::optional<std::filesystem::path> getLocation() const;
|
||||
|
||||
bool prepare(const std::filesystem::path& source, const std::filesystem::path& destination);
|
||||
|
||||
const std::string path;
|
||||
|
||||
private:
|
||||
static size_t writeFile(void* data, size_t size, size_t nmemb, void* file);
|
||||
static size_t writeString(void* data, size_t size, size_t nmemb, void* mem);
|
||||
static int trace(CURL *handle, curl_infotype type, char *data, size_t size, void *clientp);
|
||||
bool download(const std::string& url, const std::filesystem::path& destination);
|
||||
|
||||
bool downloadRepo(
|
||||
const std::filesystem::path& destination,
|
||||
const std::string& protocol,
|
||||
const std::string& host,
|
||||
const std::string& owner,
|
||||
const std::string& repo,
|
||||
std::filesystem::path& out
|
||||
);
|
||||
bool repoGiteaApi1(
|
||||
const std::string& protocol,
|
||||
const std::string& host,
|
||||
const std::string& owner,
|
||||
const std::string& repo,
|
||||
nlohmann::json& json
|
||||
);
|
||||
CURLcode httpGet(const std::string& url, std::string& result, const std::list<std::string_view>& headers = {});
|
||||
|
||||
bool extract(const std::filesystem::path& source, const std::filesystem::path& destination) const;
|
||||
int copy(struct archive *ar, struct archive *aw) const;
|
||||
|
||||
private:
|
||||
Type type;
|
||||
std::optional<std::string> name;
|
||||
std::optional<std::string> version;
|
||||
std::filesystem::path location;
|
||||
};
|
||||
|
|
@ -3,7 +3,10 @@
|
|||
constexpr std::string_view archived("archived");
|
||||
constexpr std::string_view headerAcceptJson("accept: application/json");
|
||||
|
||||
static const std::regex repo("(^https?):\\/\\/([\\w\\d\\.\\-\\_]+)\\/([\\w\\d\\-\\_]+)\\/([\\w\\d\\-\\_]+)");
|
||||
static const std::regex file("(^https?):\\/\\/(?:[\\w\\d\\.\\-\\_]+\\/)+([\\w\\d\\-\\_]+)\\.([\\w\\d\\.]+)");
|
||||
static const std::regex repo("(^https?):\\/\\/([\\w\\d\\.\\-\\_]+)\\/([\\w\\d\\-\\_]+)\\/([\\w\\d\\-\\_]+)(?:(?:\\.git)|\\/)?$");
|
||||
|
||||
static const std::regex archive("tar\\.gz$");
|
||||
|
||||
unsigned int Download::instances(0);
|
||||
AtomicMutex Download::amx{};
|
||||
|
|
@ -50,23 +53,47 @@ std::optional<std::filesystem::path> Download::getLocation() const {
|
|||
}
|
||||
|
||||
void Download::proceed() {
|
||||
bool success = false;
|
||||
std::optional<RepoInfo> repo = testRepo();
|
||||
std::optional<std::filesystem::path> path;
|
||||
if (repo.has_value())
|
||||
path = downloadAsRepo(repo.value());
|
||||
|
||||
if (path.has_value()) {
|
||||
if (repo.has_value()) {
|
||||
const RepoInfo& rp = repo.value();
|
||||
std::optional<std::filesystem::path> path = downloadAsRepo(rp);
|
||||
info(url + " has been successfully donwloaded as a repository, extracting");
|
||||
success = extract(path.value(), destination/repo.value().name);
|
||||
if (success)
|
||||
location = destination/repo.value().name;
|
||||
bool success = extract(path.value(), destination);
|
||||
if (success) {
|
||||
location = destination/rp.name;
|
||||
info("Successfully extracted " + url);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (success)
|
||||
info("Successfully extracted " + url);
|
||||
std::optional<FileInfo> file = testFile();
|
||||
if (file.has_value()) {
|
||||
const FileInfo& fl = file.value();
|
||||
std::optional<std::filesystem::path> path = downloadAsFile(fl);
|
||||
if (path.has_value()) {
|
||||
bool success = true;
|
||||
info("Successfully downloaded " + url);
|
||||
if (fl.archive) {
|
||||
info(url + " appears to be an archive, extracting");
|
||||
success = extract(path.value(), destination/fl.name);
|
||||
if (success)
|
||||
info("Successfully extracted " + url);
|
||||
}
|
||||
|
||||
if (success) {
|
||||
location = destination/fl.name;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// CURLcode code = httpDownload(url, path);
|
||||
// if (code == CURLE_OK)
|
||||
// return path;
|
||||
}
|
||||
|
||||
|
||||
std::optional<nlohmann::json> Download::repoInfoGiteaApi1(const RepoInfo& repo) {
|
||||
std::string url = repo.origin() + "/api/v1/repos/" + repo.project();
|
||||
std::string data;
|
||||
|
|
@ -99,6 +126,24 @@ std::optional<std::filesystem::path> Download::downloadAsRepo(const RepoInfo& re
|
|||
return std::nullopt;
|
||||
}
|
||||
|
||||
std::optional<std::filesystem::path> Download::downloadAsFile(const FileInfo& file) {
|
||||
std::filesystem::path dst;
|
||||
if (file.archive)
|
||||
dst = destination/archived/file.fileName();
|
||||
else
|
||||
dst = destination/file.fileName();
|
||||
|
||||
CURLcode code = httpDownload(url, dst);
|
||||
if (code == CURLE_OK)
|
||||
return dst;
|
||||
|
||||
minor("Removing " + dst.string());
|
||||
if (std::filesystem::exists(dst))
|
||||
std::filesystem::remove_all(dst);
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
std::optional<std::filesystem::path> Download::downloadRepoGiteaApi1(const RepoInfo& repo, const std::string& branch) {
|
||||
std::string fileName = branch + ".tar.gz";
|
||||
std::string url = repo.origin() + "/api/v1/repos/" + repo.project() + "/archive/" + fileName;
|
||||
|
|
@ -130,7 +175,7 @@ CURLcode Download::httpGet(
|
|||
}
|
||||
|
||||
CURLcode Download::httpDownload(const std::string& url, const std::filesystem::path& path, const std::vector<std::string_view>& headers) {
|
||||
info("Starting to download " + url);
|
||||
info("Starting to download " + url + " to " + path.string());
|
||||
std::filesystem::create_directories(path.parent_path());
|
||||
if (std::filesystem::exists(path)) {
|
||||
minor("File " + path.string() + " already exists, will be overwritten");
|
||||
|
|
@ -185,7 +230,7 @@ void Download::createCurl() {
|
|||
if (!crl)
|
||||
throw CurlError("Error creating curl instalce");
|
||||
|
||||
curl_easy_setopt(crl, CURLOPT_USERAGENT, "libcurl-agent/1.0" );
|
||||
curl_easy_setopt(crl, CURLOPT_USERAGENT, "libcurl-agent/1.0" );
|
||||
curl_easy_setopt(crl, CURLOPT_VERBOSE, 1L);
|
||||
curl_easy_setopt(crl, CURLOPT_DEBUGFUNCTION, trace);
|
||||
curl_easy_setopt(crl, CURLOPT_DEBUGDATA, this);
|
||||
|
|
@ -194,6 +239,21 @@ void Download::createCurl() {
|
|||
curl = std::unique_ptr<CURL, CurlDeleter>(crl);
|
||||
}
|
||||
|
||||
std::optional<Download::FileInfo> Download::testFile() const {
|
||||
std::smatch results;
|
||||
if (std::regex_search(url, results, file)) {
|
||||
info(url + " appears to be a file");
|
||||
std::string ext(results[3]);
|
||||
return FileInfo {
|
||||
results[1],
|
||||
results[2],
|
||||
ext,
|
||||
std::regex_search(ext, archive)
|
||||
};
|
||||
}
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
std::optional<Download::RepoInfo> Download::testRepo() const {
|
||||
std::smatch results;
|
||||
|
|
@ -347,4 +407,7 @@ std::string Download::RepoInfo::project() const {
|
|||
return owner + "/" + name;
|
||||
}
|
||||
|
||||
std::string Download::FileInfo::fileName() const {
|
||||
return name + "." + extension;
|
||||
}
|
||||
|
||||
|
|
@ -22,6 +22,7 @@ class Collection;
|
|||
|
||||
class Download : protected Loggable {
|
||||
struct RepoInfo;
|
||||
struct FileInfo;
|
||||
struct CurlDeleter {
|
||||
void operator() (CURL* curl) const;
|
||||
};
|
||||
|
|
@ -43,9 +44,11 @@ public:
|
|||
std::optional<std::filesystem::path> getLocation() const;
|
||||
|
||||
private:
|
||||
std::optional<FileInfo> testFile() const;
|
||||
std::optional<RepoInfo> testRepo() const;
|
||||
|
||||
std::optional<std::filesystem::path> downloadAsRepo(const RepoInfo& repo);
|
||||
std::optional<std::filesystem::path> downloadAsFile(const FileInfo& file);
|
||||
|
||||
std::optional<nlohmann::json> repoInfoGiteaApi1(const RepoInfo& repo);
|
||||
std::optional<std::filesystem::path> downloadRepoGiteaApi1(const RepoInfo& repo, const std::string& branch);
|
||||
|
|
@ -80,6 +83,15 @@ struct Download::RepoInfo {
|
|||
std::string project() const;
|
||||
};
|
||||
|
||||
struct Download::FileInfo {
|
||||
std::string protocol;
|
||||
std::string name;
|
||||
std::string extension;
|
||||
bool archive;
|
||||
|
||||
std::string fileName() const;
|
||||
};
|
||||
|
||||
class Download::CurlError : public std::runtime_error {
|
||||
public:
|
||||
explicit CurlError(const std::string& message);
|
||||
|
|
@ -19,7 +19,7 @@ int main(int argc, char *argv[]) {
|
|||
|
||||
|
||||
std::shared_ptr<Logger> logger = std::make_shared<Logger>(Logger::Severity::debug);
|
||||
std::shared_ptr<TaskManager> taskManager = std::make_shared<TaskManager>();
|
||||
std::shared_ptr<TaskManager> taskManager = std::make_shared<TaskManager>(logger);
|
||||
std::shared_ptr<Collection> collection = std::make_shared<Collection>(secondArg, "", logger, taskManager);
|
||||
|
||||
taskManager->start();
|
||||
|
|
|
|||
237
src/project.cpp
237
src/project.cpp
|
|
@ -1,237 +0,0 @@
|
|||
#include "project.h"
|
||||
|
||||
#include <string_view>
|
||||
#include <exception>
|
||||
|
||||
using json = nlohmann::json;
|
||||
|
||||
constexpr std::string_view entry("mason.json");
|
||||
|
||||
std::map<std::string, Dependency::Type> types({
|
||||
{"simple", Dependency::Type::simple},
|
||||
{"none", Dependency::Type::simple},
|
||||
{"mason", Dependency::Type::mason},
|
||||
{"auto", Dependency::Type::automatic},
|
||||
{"automatic", Dependency::Type::automatic}
|
||||
});
|
||||
|
||||
Logger* Project::logger = nullptr;
|
||||
|
||||
Project::Project(const std::filesystem::path& location, const std::filesystem::path& destination):
|
||||
parent(nullptr),
|
||||
location(location),
|
||||
destination(destination),
|
||||
state(State::unknown),
|
||||
name(),
|
||||
dependencies(),
|
||||
root(logger == nullptr)
|
||||
{
|
||||
if (root) {
|
||||
curl_global_init(CURL_GLOBAL_ALL);
|
||||
logger = new Logger(Logger::Severity::debug);
|
||||
}
|
||||
}
|
||||
|
||||
Project::Project(const std::filesystem::path& location, const std::filesystem::path& destination, Project* parent):
|
||||
parent(parent),
|
||||
location(location),
|
||||
destination(destination),
|
||||
state(State::unknown),
|
||||
name(),
|
||||
dependencies(),
|
||||
root(false) {}
|
||||
|
||||
Project::~Project() {
|
||||
if (root) {
|
||||
curl_global_cleanup();
|
||||
|
||||
delete logger;
|
||||
logger = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
bool Project::read() {
|
||||
if (state == State::read)
|
||||
return true;
|
||||
|
||||
std::filesystem::path entryPointPath;
|
||||
try {
|
||||
location = std::filesystem::canonical(location);
|
||||
entryPointPath = location / entry;
|
||||
} catch (const std::exception& e) {
|
||||
fatal(e.what());
|
||||
state = State::error;
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
std::filesystem::create_directories(destination);
|
||||
destination = std::filesystem::canonical(destination);
|
||||
} catch (const std::exception& e) {
|
||||
fatal(e.what());
|
||||
state = State::error;
|
||||
return false;
|
||||
}
|
||||
|
||||
std::ifstream file(entryPointPath);
|
||||
if (!file.is_open()) {
|
||||
fatal("couldn't open " + entryPointPath.string());
|
||||
state = State::error;
|
||||
return false;
|
||||
}
|
||||
|
||||
json data;
|
||||
try {
|
||||
data = json::parse(file);
|
||||
} catch (const json::exception& e) {
|
||||
fatal(e.what());
|
||||
state = State::error;
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
parse(data);
|
||||
} catch (const std::exception& e) {
|
||||
fatal(e.what());
|
||||
state = State::error;
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void Project::parse(const json& data) {
|
||||
const json& nm = data.at("name");
|
||||
if (!nm.is_string())
|
||||
throw 1;
|
||||
|
||||
name = nm;
|
||||
|
||||
const json& dpd = data.at("dependencies");
|
||||
if (!dpd.is_array())
|
||||
throw 1;
|
||||
|
||||
for (const json& dep : dpd) {
|
||||
switch (dep.type()) {
|
||||
case json::value_t::string:
|
||||
createDepencencyFromString(dep);
|
||||
break;
|
||||
case json::value_t::object:
|
||||
createDepencencyFromObject(dep);
|
||||
break;
|
||||
default:
|
||||
throw 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void Project::createDepencencyFromString(const std::string& entry) {
|
||||
dependencies.emplace(entry, entry);
|
||||
}
|
||||
|
||||
void Project::createDepencencyFromObject(const nlohmann::json& entry) {
|
||||
std::string url;
|
||||
json::const_iterator itr = entry.find("url");
|
||||
if (itr == entry.end()) {
|
||||
itr = entry.find("path");
|
||||
if (itr == entry.end())
|
||||
throw 1;
|
||||
}
|
||||
if (!itr->is_string())
|
||||
throw 1;
|
||||
|
||||
url = *itr;
|
||||
|
||||
std::optional<std::string> name = std::nullopt;
|
||||
itr = entry.find("name");
|
||||
if (itr != entry.end() && itr->is_string())
|
||||
name = *itr;
|
||||
|
||||
std::optional<std::string> version = std::nullopt;
|
||||
itr = entry.find("version");
|
||||
if (itr != entry.end() && itr->is_string())
|
||||
version = *itr;
|
||||
|
||||
Dependency::Type type = Dependency::Type::automatic;
|
||||
itr = entry.find("type");
|
||||
if (itr != entry.end() && itr->is_string()) {
|
||||
std::map<std::string, Dependency::Type>::const_iterator titr = types.find(*itr);
|
||||
if (titr != types.end())
|
||||
type = titr->second;
|
||||
}
|
||||
|
||||
dependencies.emplace(url, Dependency{url, type, name, version});
|
||||
}
|
||||
|
||||
void Project::discover() {
|
||||
int fine = 0;
|
||||
for (std::pair<const std::string, Dependency>& pair : dependencies) {
|
||||
Dependency& dep = pair.second;
|
||||
bool success = dep.prepare(location, destination);
|
||||
if (success) {
|
||||
switch (dep.getType()) {
|
||||
case Dependency::Type::mason:
|
||||
break;
|
||||
case Dependency::Type::simple:
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
fine++;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
uint32_t Project::dependenciesCount() const {
|
||||
return dependencies.size();
|
||||
}
|
||||
|
||||
void Project::log(Logger::Severity severity, const std::string& message) {
|
||||
if (logger != nullptr)
|
||||
logger->log(severity, message);
|
||||
}
|
||||
|
||||
void Project::info(const std::string& message) {
|
||||
log(Logger::Severity::info, message);
|
||||
}
|
||||
|
||||
void Project::debug(const std::string& message) {
|
||||
log(Logger::Severity::debug, message);
|
||||
}
|
||||
|
||||
void Project::minor(const std::string& message) {
|
||||
log(Logger::Severity::minor, message);
|
||||
}
|
||||
|
||||
void Project::major(const std::string& message) {
|
||||
log(Logger::Severity::major, message);
|
||||
}
|
||||
|
||||
void Project::error(const std::string& message) {
|
||||
log(Logger::Severity::error, message);
|
||||
}
|
||||
|
||||
void Project::warn(const std::string& message) {
|
||||
log(Logger::Severity::warning, message);
|
||||
}
|
||||
|
||||
void Project::fatal(const std::string& message) {
|
||||
log(Logger::Severity::fatal, message);
|
||||
}
|
||||
|
||||
void Project::printLog() {
|
||||
if (logger != nullptr)
|
||||
logger->printLog();
|
||||
}
|
||||
|
||||
Project::State Project::getStatus() const {
|
||||
return state;
|
||||
}
|
||||
|
||||
std::string Project::getName() const {
|
||||
return name;
|
||||
}
|
||||
|
|
@ -1,64 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
#include <filesystem>
|
||||
#include <string>
|
||||
#include <fstream>
|
||||
#include <list>
|
||||
#include <map>
|
||||
#include <stdint.h>
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <curl/curl.h>
|
||||
|
||||
#include "dependency.h"
|
||||
#include "loggger.h"
|
||||
|
||||
class Project {
|
||||
public:
|
||||
enum class State {
|
||||
unknown,
|
||||
read,
|
||||
discovering,
|
||||
error
|
||||
};
|
||||
|
||||
Project(const std::filesystem::path& location, const std::filesystem::path& destination);
|
||||
~Project();
|
||||
|
||||
bool read();
|
||||
void discover();
|
||||
State getStatus() const;
|
||||
std::string getName() const;
|
||||
uint32_t dependenciesCount() const;
|
||||
|
||||
static void log(Logger::Severity severity, const std::string& message);
|
||||
static void debug(const std::string& message);
|
||||
static void info(const std::string& message);
|
||||
static void minor(const std::string& message);
|
||||
static void major(const std::string& message);
|
||||
static void warn(const std::string& message);
|
||||
static void error(const std::string& message);
|
||||
static void fatal(const std::string& message);
|
||||
static void printLog();
|
||||
|
||||
static void addProject(const std::filesystem::path& location, const std::filesystem::path& destination);
|
||||
|
||||
private:
|
||||
Project(const std::filesystem::path& location, const std::filesystem::path& destination, Project* parent);
|
||||
|
||||
void parse(const nlohmann::json& json);
|
||||
void createDepencencyFromString(const std::string& entry);
|
||||
void createDepencencyFromObject(const nlohmann::json& entry);
|
||||
|
||||
private:
|
||||
Project* parent;
|
||||
std::filesystem::path location;
|
||||
std::filesystem::path destination;
|
||||
State state;
|
||||
std::string name;
|
||||
std::map<std::string, Dependency> dependencies;
|
||||
const bool root;
|
||||
|
||||
static Logger* logger;
|
||||
|
||||
};
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
#include "taskmanager.h"
|
||||
|
||||
TaskManager::TaskManager() :
|
||||
TaskManager::TaskManager(const std::shared_ptr<Logger>& logger) :
|
||||
Loggable(logger),
|
||||
running(false),
|
||||
stopping(false),
|
||||
maxThreads(std::thread::hardware_concurrency()),
|
||||
|
|
@ -38,6 +39,7 @@ void TaskManager::start() {
|
|||
if (running)
|
||||
return;
|
||||
|
||||
debug("Starting " + std::to_string(maxThreads) + " threads");
|
||||
for (uint32_t i = 0; i < maxThreads; ++i)
|
||||
threads.emplace_back(&TaskManager::loop, this);
|
||||
|
||||
|
|
@ -49,6 +51,7 @@ void TaskManager::stop() {
|
|||
if (!running)
|
||||
return;
|
||||
|
||||
debug("Stopping task manager");
|
||||
stopping = true;
|
||||
lock.unlock();
|
||||
|
||||
|
|
@ -57,9 +60,12 @@ void TaskManager::stop() {
|
|||
thread.join();
|
||||
|
||||
threads.clear();
|
||||
running = false;
|
||||
}
|
||||
|
||||
void TaskManager::loop() {
|
||||
//debug("Thread " + std::to_string(std::this_thread::get_id()) + " entered the loop");
|
||||
debug("Thread entered the loop");
|
||||
while (true) {
|
||||
Task task;
|
||||
std::unique_lock lock(mutex);
|
||||
|
|
@ -67,11 +73,12 @@ void TaskManager::loop() {
|
|||
loopConditional.wait(lock);
|
||||
|
||||
if (stopping)
|
||||
return;
|
||||
break;
|
||||
|
||||
++activeThreads;
|
||||
task = tasks.front();
|
||||
tasks.pop();
|
||||
//debug("Thread took a task");
|
||||
lock.unlock();
|
||||
|
||||
executeTask(task);
|
||||
|
|
@ -81,7 +88,7 @@ void TaskManager::loop() {
|
|||
lock.unlock();
|
||||
waitConditional.notify_all();
|
||||
}
|
||||
|
||||
debug("Thread left the loop");
|
||||
}
|
||||
|
||||
void TaskManager::executeTask(const Task& task) const {
|
||||
|
|
@ -90,6 +97,8 @@ void TaskManager::executeTask(const Task& task) const {
|
|||
} catch (const std::exception& e) {
|
||||
if (task.second.has_value())
|
||||
task.second.value()(&e);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
|
|
@ -106,6 +115,6 @@ bool TaskManager::busy() const {
|
|||
|
||||
void TaskManager::wait() const {
|
||||
std::unique_lock lock(mutex);
|
||||
while (activeThreads != 0)
|
||||
while (activeThreads != 0 || !tasks.empty())
|
||||
waitConditional.wait(lock);
|
||||
}
|
||||
|
|
@ -9,14 +9,18 @@
|
|||
#include <functional>
|
||||
#include <optional>
|
||||
#include <exception>
|
||||
#include <memory>
|
||||
|
||||
class TaskManager {
|
||||
#include "loggable.h"
|
||||
|
||||
class TaskManager : public Loggable {
|
||||
public:
|
||||
using Error = std::optional<const std::exception*>;
|
||||
using Job = std::function<void ()>;
|
||||
using Result = std::function<void (Error)>;
|
||||
using Task = std::pair<Job, std::optional<Result>>;
|
||||
TaskManager();
|
||||
|
||||
TaskManager(const std::shared_ptr<Logger>& logger);
|
||||
~TaskManager();
|
||||
|
||||
void start();
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
set(SOURCES
|
||||
component.cpp
|
||||
collection.cpp
|
||||
loggable.cpp
|
||||
taskmanager.cpp
|
||||
download.cpp
|
||||
atomicmutex.cpp
|
||||
)
|
||||
|
||||
set(HEADERS
|
||||
component.h
|
||||
collection.h
|
||||
loggable.h
|
||||
taskmanager.h
|
||||
download.h
|
||||
atomicmutex.h
|
||||
)
|
||||
|
||||
target_sources(${PROJECT_NAME} PRIVATE ${SOURCES})
|
||||
|
|
@ -6,7 +6,9 @@
|
|||
"https://git.macaw.me/blue/mason.git",
|
||||
{
|
||||
"name" : "aProject",
|
||||
"path" : "https://nowhere.to"
|
||||
}
|
||||
"path" : "https://nowhere/to/get.from"
|
||||
},
|
||||
"https://git.macaw.me/blue/mason/archive/main.tar.gz",
|
||||
"https://git.macaw.me/blue/mason/raw/branch/main/src2/atomicmutex.h"
|
||||
]
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue