refactoring seems to be done, added ability to specify files as dependencies
This commit is contained in:
parent
b07d017f86
commit
2b913d1d42
GPG key ID:
9B203B252A63EE38
21 changed files with 148 additions and 793 deletions
|
|
@ -1,14 +1,22 @@
|
|||
set(SOURCES
|
||||
main.cpp
|
||||
loggger.cpp
|
||||
project.cpp
|
||||
dependency.cpp
|
||||
component.cpp
|
||||
collection.cpp
|
||||
loggable.cpp
|
||||
taskmanager.cpp
|
||||
download.cpp
|
||||
atomicmutex.cpp
|
||||
)
|
||||
|
||||
set(HEADERS
|
||||
logger.h
|
||||
project.h
|
||||
dependency.cpp
|
||||
component.h
|
||||
collection.h
|
||||
loggable.h
|
||||
taskmanager.h
|
||||
download.h
|
||||
atomicmutex.h
|
||||
)
|
||||
|
||||
target_sources(${PROJECT_NAME} PRIVATE ${SOURCES})
|
||||
|
|
|
|||
13
src/atomicmutex.cpp
Normal file
13
src/atomicmutex.cpp
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
#include "atomicmutex.h"
|
||||
|
||||
AtomicMutex::AtomicMutex():
|
||||
flag()
|
||||
{}
|
||||
|
||||
void AtomicMutex::lock() noexcept {
|
||||
while (flag.test_and_set());
|
||||
}
|
||||
|
||||
void AtomicMutex::unlock() noexcept {
|
||||
flag.clear();
|
||||
}
|
||||
13
src/atomicmutex.h
Normal file
13
src/atomicmutex.h
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
#pragma once
|
||||
|
||||
#include <atomic>
|
||||
|
||||
class AtomicMutex {
|
||||
public:
|
||||
explicit AtomicMutex();
|
||||
void lock() noexcept;
|
||||
void unlock() noexcept;
|
||||
|
||||
private:
|
||||
std::atomic_flag flag;
|
||||
};
|
||||
137
src/collection.cpp
Normal file
137
src/collection.cpp
Normal file
|
|
@ -0,0 +1,137 @@
|
|||
#include "collection.h"
|
||||
|
||||
constexpr std::string_view downloads("downloads");
|
||||
|
||||
static const std::regex remote("(^https?):\\/\\/");
|
||||
|
||||
Collection::Collection(
|
||||
const std::filesystem::path& destination,
|
||||
const std::string& target,
|
||||
const std::shared_ptr<Logger>& logger,
|
||||
const std::shared_ptr<TaskManager>& taskManager
|
||||
):
|
||||
Loggable(logger),
|
||||
std::enable_shared_from_this<Collection>(),
|
||||
mutex(),
|
||||
destination(destination),
|
||||
target(target),
|
||||
taskManager(taskManager),
|
||||
knownSources(),
|
||||
successSources(),
|
||||
downloadingSources(),
|
||||
readingSources(),
|
||||
buildingSources()
|
||||
{
|
||||
try {
|
||||
std::filesystem::create_directories(destination);
|
||||
} catch (const std::exception& e) {
|
||||
fatal(e.what());
|
||||
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
void Collection::addSource(const std::string& source) {
|
||||
std::lock_guard lock(mutex);
|
||||
if (_hasSource(source)) {
|
||||
major("Source " + source + " is already present, skipping as duplicate");
|
||||
return;
|
||||
}
|
||||
|
||||
debug("Adding source " + source);
|
||||
if (isRemote(source))
|
||||
queueDownload(source);
|
||||
else
|
||||
queueRead(source, source);
|
||||
}
|
||||
|
||||
void Collection::sourceDownaloded(const std::string& source, TaskManager::Error err) {
|
||||
std::lock_guard lock(mutex);
|
||||
|
||||
Downloads::const_iterator itr = downloadingSources.find(source);
|
||||
std::optional<std::filesystem::path> location = itr->second->getLocation();
|
||||
downloadingSources.erase(itr);
|
||||
if (err.has_value() || !location.has_value()) {
|
||||
error("Coundn't download " + source + ": " + err.value()->what());
|
||||
return;
|
||||
}
|
||||
|
||||
queueRead(source, location.value());
|
||||
}
|
||||
|
||||
void Collection::sourceRead(const std::string& source, TaskManager::Error err) {
|
||||
std::lock_guard lock(mutex);
|
||||
|
||||
debug("Source " + source + " has been read");
|
||||
Components::iterator itr = readingSources.find(source);
|
||||
if (err.has_value()) {
|
||||
error("Coundn't read " + source + ": " + err.value()->what());
|
||||
readingSources.erase(itr);
|
||||
return;
|
||||
}
|
||||
|
||||
debug("Queuing build of " + source);
|
||||
std::pair<Components::iterator, bool> res = buildingSources.emplace(source, std::move(itr->second));
|
||||
readingSources.erase(itr);
|
||||
|
||||
taskManager->queue(
|
||||
std::bind(&Component::build, res.first->second.get(), destination, target),
|
||||
std::bind(&Collection::sourceBuilt, this, source, std::placeholders::_1)
|
||||
);
|
||||
}
|
||||
|
||||
void Collection::sourceBuilt(const std::string& source, TaskManager::Error err) {
|
||||
std::lock_guard lock(mutex);
|
||||
debug("Source " + source + " has been built");
|
||||
Components::iterator itr = buildingSources.find(source);
|
||||
buildingSources.erase(itr);
|
||||
if (err.has_value()) {
|
||||
error("Coundn't read " + source + ": " + err.value()->what());
|
||||
return;
|
||||
}
|
||||
|
||||
successSources.emplace(source);
|
||||
}
|
||||
|
||||
bool Collection::isRemote(const std::string& source) {
|
||||
return std::regex_search(source, remote);
|
||||
}
|
||||
|
||||
void Collection::queueDownload(const std::string& source) {
|
||||
debug("Queuing download of " + source);
|
||||
std::pair<Downloads::iterator, bool> res = downloadingSources.emplace(std::piecewise_construct,
|
||||
std::forward_as_tuple(source),
|
||||
std::forward_as_tuple(
|
||||
std::make_unique<Download>(source, destination/downloads, logger)
|
||||
)
|
||||
);
|
||||
|
||||
taskManager->queue(
|
||||
std::bind(&Download::proceed, res.first->second.get()),
|
||||
std::bind(&Collection::sourceDownaloded, this, source, std::placeholders::_1)
|
||||
);
|
||||
}
|
||||
|
||||
void Collection::queueRead(const std::string& source, const std::filesystem::path& location) {
|
||||
debug("Queuing read of " + source);
|
||||
std::pair<Components::iterator, bool> res = readingSources.emplace(std::piecewise_construct,
|
||||
std::forward_as_tuple(source),
|
||||
std::forward_as_tuple(
|
||||
std::make_unique<Component>(location, shared_from_this(), logger)
|
||||
)
|
||||
);
|
||||
|
||||
taskManager->queue(
|
||||
std::bind(&Component::read, res.first->second.get()),
|
||||
std::bind(&Collection::sourceRead, this, source, std::placeholders::_1)
|
||||
);
|
||||
}
|
||||
|
||||
bool Collection::hasSource(const std::string& source) const {
|
||||
std::lock_guard lock(mutex);
|
||||
return _hasSource(source);
|
||||
}
|
||||
|
||||
bool Collection::_hasSource(const std::string& source) const {
|
||||
return knownSources.count(source) > 0;
|
||||
}
|
||||
60
src/collection.h
Normal file
60
src/collection.h
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <filesystem>
|
||||
#include <map>
|
||||
#include <set>
|
||||
#include <exception>
|
||||
#include <string_view>
|
||||
#include <regex>
|
||||
#include <mutex>
|
||||
|
||||
#include "loggable.h"
|
||||
#include "component.h"
|
||||
#include "download.h"
|
||||
#include "taskmanager.h"
|
||||
|
||||
class Collection : protected Loggable, public std::enable_shared_from_this<Collection> {
|
||||
using Sources = std::set<std::string>;
|
||||
using Downloads = std::map<std::string, std::unique_ptr<Download>>;
|
||||
using Components = std::map<std::string, std::unique_ptr<Component>>;
|
||||
|
||||
public:
|
||||
Collection(
|
||||
const std::filesystem::path& destination,
|
||||
const std::string& target,
|
||||
const std::shared_ptr<Logger>& logger,
|
||||
const std::shared_ptr<TaskManager>& taskManager
|
||||
);
|
||||
|
||||
unsigned int sourcesTotal() const;
|
||||
unsigned int sourcesPending() const;
|
||||
unsigned int sourcesError() const;
|
||||
unsigned int sourcesSuccess() const;
|
||||
|
||||
bool hasSource(const std::string& source) const;
|
||||
void addSource(const std::string& source);
|
||||
|
||||
private:
|
||||
void sourceDownaloded(const std::string& source, TaskManager::Error err);
|
||||
void sourceRead(const std::string& source, TaskManager::Error err);
|
||||
void sourceBuilt(const std::string& source, TaskManager::Error err);
|
||||
|
||||
static bool isRemote(const std::string& source);
|
||||
|
||||
void queueDownload(const std::string& source);
|
||||
void queueRead(const std::string& source, const std::filesystem::path& location);
|
||||
|
||||
bool _hasSource(const std::string& source) const;
|
||||
|
||||
private:
|
||||
mutable std::mutex mutex;
|
||||
std::filesystem::path destination;
|
||||
std::string target;
|
||||
std::shared_ptr<TaskManager> taskManager;
|
||||
Sources knownSources;
|
||||
Sources successSources;
|
||||
Downloads downloadingSources;
|
||||
Components readingSources;
|
||||
Components buildingSources;
|
||||
};
|
||||
143
src/component.cpp
Normal file
143
src/component.cpp
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
#include "component.h"
|
||||
|
||||
#include "collection.h"
|
||||
|
||||
constexpr std::string_view masonJSON("mason.json");
|
||||
constexpr std::string_view dependencies("dependencies");
|
||||
constexpr std::array<std::string_view, Component::done + 1> stringStates {
|
||||
"initial",
|
||||
"reading",
|
||||
"ready",
|
||||
"building",
|
||||
"error",
|
||||
"done"
|
||||
};
|
||||
|
||||
Component::Component(
|
||||
const std::filesystem::path& path,
|
||||
const std::weak_ptr<Collection>& collection,
|
||||
const std::shared_ptr<Logger>& logger
|
||||
):
|
||||
Loggable(logger),
|
||||
state(initial),
|
||||
type(unknown),
|
||||
collection(collection),
|
||||
location(path),
|
||||
scenario(std::nullopt)
|
||||
{}
|
||||
|
||||
void Component::read() {
|
||||
if (state != initial)
|
||||
throw WrongState(state, "read");
|
||||
|
||||
state = reading;
|
||||
|
||||
std::filesystem::file_status status = std::filesystem::status(location);
|
||||
switch (status.type()) {
|
||||
case std::filesystem::file_type::directory:
|
||||
type = directory;
|
||||
state = ready;
|
||||
tryReadingBuildScenarios();
|
||||
break;
|
||||
case std::filesystem::file_type::regular:
|
||||
type = file;
|
||||
state = ready;
|
||||
break;
|
||||
default:
|
||||
warn(location.string() + " doesn't appear to be a file nor a directory");
|
||||
state = error;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
void Component::tryReadingBuildScenarios() {
|
||||
if (readAsMason())
|
||||
return;
|
||||
}
|
||||
|
||||
bool Component::readAsMason() {
|
||||
std::filesystem::path masonScenarion = location/masonJSON;
|
||||
try {
|
||||
std::ifstream file(masonScenarion);
|
||||
if (file.is_open())
|
||||
scenario = nlohmann::json::parse(file);
|
||||
else
|
||||
minor("Couldn't open " + masonScenarion.string());
|
||||
} catch (const nlohmann::json::exception& e) {
|
||||
major("Couldn't parse " + masonScenarion.string());
|
||||
}
|
||||
|
||||
if (scenario.has_value()) {
|
||||
const nlohmann::json& sc = scenario.value();
|
||||
if (!sc.is_object())
|
||||
return errorScenario(masonScenarion.string() + " is unexpected root type");
|
||||
|
||||
nlohmann::json::const_iterator itr = sc.find(dependencies);
|
||||
if (itr == sc.end()) {
|
||||
info(masonScenarion.string() + " doesn't have dependencies");
|
||||
} else {
|
||||
const nlohmann::json& deps = *itr;
|
||||
if (!deps.is_array())
|
||||
return errorScenario(masonScenarion.string() + " dependencies are not organized as an array");
|
||||
|
||||
for (const nlohmann::json& dep : deps) {
|
||||
std::shared_ptr<Collection> col = collection.lock();
|
||||
switch (dep.type()) {
|
||||
case nlohmann::json::value_t::string:
|
||||
col->addSource(dep);
|
||||
break;
|
||||
case nlohmann::json::value_t::object: {
|
||||
nlohmann::json::const_iterator ditr = dep.find("path");
|
||||
if (ditr == dep.end())
|
||||
return errorScenario(masonScenarion.string() + " object of unexpected format in dependecies");
|
||||
|
||||
nlohmann::json path = *ditr;
|
||||
if (!path.is_string())
|
||||
return errorScenario(masonScenarion.string() + " object of unexpected format in dependecies");
|
||||
|
||||
col->addSource(path);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
return errorScenario(masonScenarion.string() + " has unexpected entries in dependecies");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type = mason;
|
||||
info(location.string() + " seems to be a mason project");
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
bool Component::errorScenario(const std::string& message) {
|
||||
major(message);
|
||||
scenario = std::nullopt;
|
||||
return false;
|
||||
return false;
|
||||
}
|
||||
|
||||
void Component::build(const std::filesystem::path& destination, const std::string& target) {
|
||||
if (state != ready)
|
||||
throw WrongState(state, "build");
|
||||
|
||||
state = building;
|
||||
|
||||
info("Building " + location.string() + " to " + destination.string());
|
||||
state = done;
|
||||
}
|
||||
|
||||
Component::State Component::getState() const {
|
||||
return state;
|
||||
}
|
||||
|
||||
Component::Type Component::getType() const {
|
||||
return type;
|
||||
}
|
||||
|
||||
Component::WrongState::WrongState(State state, const std::string& action):
|
||||
std::runtime_error("An attempt to perform action \"" + action
|
||||
+ "\" on a wrong state \"" + stringStates[state].data() + '\"')
|
||||
{}
|
||||
66
src/component.h
Normal file
66
src/component.h
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <memory>
|
||||
#include <exception>
|
||||
#include <optional>
|
||||
#include <functional>
|
||||
#include <array>
|
||||
#include <string_view>
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
#include "loggable.h"
|
||||
#include "loggger.h"
|
||||
|
||||
class Collection;
|
||||
|
||||
class Component : protected Loggable {
|
||||
public:
|
||||
class WrongState;
|
||||
enum State {
|
||||
initial,
|
||||
reading,
|
||||
ready,
|
||||
building,
|
||||
error,
|
||||
done
|
||||
};
|
||||
|
||||
enum Type {
|
||||
unknown,
|
||||
file,
|
||||
directory,
|
||||
mason
|
||||
};
|
||||
|
||||
Component(
|
||||
const std::filesystem::path& path,
|
||||
const std::weak_ptr<Collection>& collection,
|
||||
const std::shared_ptr<Logger>& logger
|
||||
);
|
||||
|
||||
Type getType() const;
|
||||
State getState() const;
|
||||
|
||||
void read();
|
||||
void build(const std::filesystem::path& destination, const std::string& target);
|
||||
|
||||
private:
|
||||
void tryReadingBuildScenarios();
|
||||
bool readAsMason();
|
||||
bool errorScenario(const std::string& message);
|
||||
|
||||
private:
|
||||
State state;
|
||||
Type type;
|
||||
std::weak_ptr<Collection> collection;
|
||||
std::filesystem::path location;
|
||||
std::optional<nlohmann::json> scenario;
|
||||
};
|
||||
|
||||
class Component::WrongState : public std::runtime_error {
|
||||
public:
|
||||
explicit WrongState(State state, const std::string& action);
|
||||
};
|
||||
|
|
@ -1,375 +0,0 @@
|
|||
#include "dependency.h"
|
||||
|
||||
#include <regex>
|
||||
|
||||
#include "project.h"
|
||||
|
||||
constexpr std::string_view downloads("downloads");
|
||||
constexpr std::string_view sources("sources");
|
||||
constexpr std::string_view build("build");
|
||||
|
||||
constexpr std::string_view acceptJson("accept: application/json");
|
||||
const std::regex http("^https?:\\/\\/");
|
||||
const std::regex repo("(^https?):\\/\\/([\\w\\d\\.\\-\\_]+)\\/([\\w\\d\\-\\_]+)\\/([\\w\\d\\-\\_]+)");
|
||||
|
||||
Dependency::Dependency(
|
||||
const std::string& path,
|
||||
Type type,
|
||||
const std::optional<std::string>& name,
|
||||
const std::optional<std::string>& version
|
||||
):
|
||||
path(path),
|
||||
type(type),
|
||||
name(name),
|
||||
version(version),
|
||||
location()
|
||||
{}
|
||||
|
||||
Dependency::Type Dependency::getType() const {
|
||||
return type;
|
||||
}
|
||||
|
||||
std::optional<std::string> Dependency::getName() const {
|
||||
return name;
|
||||
}
|
||||
|
||||
std::optional<std::string> Dependency::getVersion() const {
|
||||
return version;
|
||||
}
|
||||
|
||||
std::optional<std::filesystem::path> Dependency::getLocation() const {
|
||||
if (location.empty())
|
||||
return std::nullopt;
|
||||
else
|
||||
return location;
|
||||
}
|
||||
|
||||
bool Dependency::prepare(const std::filesystem::path& source, const std::filesystem::path& destination) {
|
||||
std::smatch results;
|
||||
if (std::regex_search(path, results, repo)) {
|
||||
Project::info(path + " appears to be a git repository");
|
||||
const std::string& protocol = results[1];
|
||||
const std::string& host = results[2];
|
||||
const std::string& owner = results[3];
|
||||
const std::string& repo = results[4];
|
||||
|
||||
std::filesystem::path temp;
|
||||
bool success = downloadRepo(destination, protocol, host, owner, repo, temp);
|
||||
if (success) {
|
||||
Project::info("Successfully obtained project " + path);
|
||||
|
||||
switch (type) {
|
||||
case Type::automatic:
|
||||
if (std::filesystem::exists(temp/"mason.json"))
|
||||
type = Type::mason;
|
||||
else
|
||||
type = Type::simple;
|
||||
|
||||
break;
|
||||
case Type::mason:
|
||||
if (std::filesystem::exists(temp/"mason.json"))
|
||||
break;
|
||||
else {
|
||||
Project::warn("Project " + path + " is supposed to me a mason project, but no mason.json was found in the project directory");
|
||||
success = false;
|
||||
}
|
||||
|
||||
break;
|
||||
case Type::simple:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return success;
|
||||
} else {
|
||||
Project::info("checking project at path " + path);
|
||||
std::filesystem::directory_entry srcDir(source / path);
|
||||
if (!srcDir.exists()) {
|
||||
Project::error("Project at " + path + " doesn't exist");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Dependency::downloadRepo(
|
||||
const std::filesystem::path& destination,
|
||||
const std::string& protocol,
|
||||
const std::string& host,
|
||||
const std::string& owner,
|
||||
const std::string& repo,
|
||||
std::filesystem::path& out
|
||||
) {
|
||||
nlohmann::json info;
|
||||
Project::info("Trying Gitea v1 API");
|
||||
bool res = repoGiteaApi1(protocol, host, owner, repo, info);
|
||||
|
||||
if (res) {
|
||||
nlohmann::json::const_iterator itr = info.find("default_branch");
|
||||
if (itr != info.end() && itr->is_string()) {
|
||||
std::string branchName = *itr;
|
||||
Project::info("Gitea v1 API seem to have worked");
|
||||
Project::info("Default branch is " + branchName);
|
||||
|
||||
std::string fileName = branchName + ".tar.gz";
|
||||
std::string url = protocol + "://" + host + "/api/v1/repos/" + owner + "/" + repo + "/archive/" + fileName;
|
||||
std::filesystem::path archivePath = destination/downloads/fileName;
|
||||
res = download(url, archivePath);
|
||||
if (res) {
|
||||
Project::info("Successfully downloaded " + archivePath.string());
|
||||
|
||||
location = destination/sources;
|
||||
res = extract(archivePath, location);
|
||||
if (res) {
|
||||
out = destination/build/repo;
|
||||
if (!std::filesystem::is_directory(out)) {
|
||||
Project::error("Extracted archive " + fileName + " but the content is unexpected");
|
||||
res = false;
|
||||
}
|
||||
} else {
|
||||
Project::error("Couldn't extract archive " + fileName);
|
||||
}
|
||||
|
||||
if (!res)
|
||||
location.clear();
|
||||
|
||||
return res;
|
||||
}
|
||||
} else {
|
||||
res = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (!res) {
|
||||
Project::warn("Gitea v1 API didn't work");
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
bool Dependency::repoGiteaApi1(
|
||||
const std::string& protocol,
|
||||
const std::string& host,
|
||||
const std::string& owner,
|
||||
const std::string& repo,
|
||||
nlohmann::json& json
|
||||
) {
|
||||
bool result = false;
|
||||
std::string url = protocol + "://" + host + "/api/v1/repos/" + owner + "/" + repo;
|
||||
std::string data;
|
||||
CURLcode code = httpGet(url, data, {acceptJson});
|
||||
if (code == CURLE_OK) {
|
||||
try {
|
||||
json = nlohmann::json::parse(data);
|
||||
result = true;
|
||||
} catch (const nlohmann::json::exception& e) {
|
||||
Project::warn(e.what());
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
CURLcode Dependency::httpGet(const std::string& url, std::string& result, const std::list<std::string_view>& headers) {
|
||||
CURL* curl = curl_easy_init();
|
||||
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
|
||||
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
|
||||
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L);
|
||||
|
||||
struct curl_slist* curlHeaders = nullptr;
|
||||
for (const std::string_view& header : headers)
|
||||
curlHeaders = curl_slist_append(curlHeaders, header.data());
|
||||
|
||||
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curlHeaders);
|
||||
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeString);
|
||||
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &result);
|
||||
|
||||
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
|
||||
curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, trace);
|
||||
curl_easy_setopt(curl, CURLOPT_DEBUGDATA, this);
|
||||
|
||||
CURLcode res = curl_easy_perform(curl);
|
||||
|
||||
curl_easy_cleanup(curl);
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
bool Dependency::download(const std::string& url, const std::filesystem::path& destination) {
|
||||
std::filesystem::create_directories(destination.parent_path());
|
||||
if (std::filesystem::exists(destination)) {
|
||||
Project::minor("File " + destination.string() + " already exists, will be overwritten");
|
||||
|
||||
if (std::filesystem::is_directory(destination))
|
||||
std::filesystem::remove_all(destination);
|
||||
}
|
||||
|
||||
bool result = false;
|
||||
CURL* curl = curl_easy_init();
|
||||
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
|
||||
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L);
|
||||
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeFile);
|
||||
|
||||
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
|
||||
curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, trace);
|
||||
curl_easy_setopt(curl, CURLOPT_DEBUGDATA, this);
|
||||
|
||||
FILE* pagefile = fopen(destination.c_str(), "wb");
|
||||
if (pagefile) {
|
||||
curl_easy_setopt(curl, CURLOPT_WRITEDATA, pagefile);
|
||||
CURLcode res = curl_easy_perform(curl);
|
||||
|
||||
fclose(pagefile);
|
||||
if (res != CURLE_OK) {
|
||||
Project::warn("Couldn't download file " + url + ": " + curl_easy_strerror(res));
|
||||
} else {
|
||||
uint32_t code;
|
||||
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code);
|
||||
if (code == 200)
|
||||
result = true;
|
||||
else
|
||||
Project::warn("Couldn't download file " + url + ": response code " + std::to_string(code));
|
||||
}
|
||||
} else {
|
||||
Project::error(std::string("Couldn't open output file ") + destination.c_str());
|
||||
}
|
||||
|
||||
curl_easy_cleanup(curl);
|
||||
|
||||
if (!result) {
|
||||
Project::minor("Removing " + destination.string() + " since the donwload failed");
|
||||
std::filesystem::remove_all(destination);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
size_t Dependency::writeString(void* data, size_t size, size_t nmemb, void* pointer) {
|
||||
size_t finalSize = size * nmemb;
|
||||
std::string* string = static_cast<std::string*>(pointer);
|
||||
string->append(static_cast<char*>(data), finalSize);
|
||||
|
||||
return finalSize;
|
||||
}
|
||||
|
||||
size_t Dependency::writeFile(void* data, size_t size, size_t nmemb, void* file) {
|
||||
size_t written = fwrite(data, size, nmemb, (FILE *)file);
|
||||
return written;
|
||||
}
|
||||
|
||||
int Dependency::trace(CURL* handle, curl_infotype type, char* data, size_t size, void* clientp) {
|
||||
switch (type) {
|
||||
case CURLINFO_TEXT: {
|
||||
std::string message(data, size);
|
||||
if (message[size - 1] == '\n')
|
||||
message = message.substr(0, size - 1);
|
||||
|
||||
Project::debug(message);
|
||||
} break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool Dependency::extract(const std::filesystem::path& source, const std::filesystem::path& destination) const {
|
||||
int flags = ARCHIVE_EXTRACT_TIME;
|
||||
flags |= ARCHIVE_EXTRACT_PERM;
|
||||
flags |= ARCHIVE_EXTRACT_ACL;
|
||||
flags |= ARCHIVE_EXTRACT_FFLAGS;
|
||||
|
||||
struct archive* a = archive_read_new();
|
||||
struct archive* ext = archive_write_disk_new();
|
||||
struct archive_entry *entry;
|
||||
archive_read_support_format_all(a);
|
||||
archive_read_support_filter_all(a);
|
||||
archive_write_disk_set_options(ext, flags);
|
||||
archive_write_disk_set_standard_lookup(ext);
|
||||
|
||||
bool result = true;
|
||||
bool readOpen = false;
|
||||
bool writeOpen = false;
|
||||
int r = archive_read_open_filename(a, source.c_str(), 10240);
|
||||
if (r) {
|
||||
Project::major("Couldn't open file " + source.string());
|
||||
result = false;
|
||||
} else {
|
||||
readOpen = true;
|
||||
}
|
||||
|
||||
while (result) {
|
||||
r = archive_read_next_header(a, &entry);
|
||||
if (r == ARCHIVE_EOF)
|
||||
break;
|
||||
|
||||
if (r < ARCHIVE_OK)
|
||||
Project::major(archive_error_string(a));
|
||||
|
||||
if (r < ARCHIVE_WARN)
|
||||
break;
|
||||
|
||||
std::string fileName(archive_entry_pathname(entry));
|
||||
std::filesystem::path filePath = destination/fileName;
|
||||
Project::debug("Extracting " + filePath.string());
|
||||
if (std::filesystem::exists(filePath))
|
||||
Project::minor(filePath.string() + " exists, overwriting");
|
||||
|
||||
archive_entry_set_pathname_utf8(entry, filePath.c_str());
|
||||
|
||||
r = archive_write_header(ext, entry);
|
||||
if (r < ARCHIVE_OK) {
|
||||
Project::major(archive_error_string(ext));
|
||||
} else if (archive_entry_size(entry) > 0) {
|
||||
writeOpen = true;
|
||||
r = copy(a, ext);
|
||||
if (r < ARCHIVE_OK)
|
||||
Project::major(archive_error_string(ext));
|
||||
|
||||
if (r < ARCHIVE_WARN)
|
||||
break;
|
||||
}
|
||||
r = archive_write_finish_entry(ext);
|
||||
if (r < ARCHIVE_OK)
|
||||
Project::major(archive_error_string(ext));
|
||||
|
||||
if (r < ARCHIVE_WARN)
|
||||
break;
|
||||
}
|
||||
|
||||
if (readOpen)
|
||||
archive_read_close(a);
|
||||
|
||||
if (writeOpen)
|
||||
archive_write_close(ext);
|
||||
|
||||
archive_read_free(a);
|
||||
archive_write_free(ext);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
int Dependency::copy(struct archive* ar, struct archive* aw) const {
|
||||
int r;
|
||||
const void *buff;
|
||||
size_t size;
|
||||
la_int64_t offset;
|
||||
|
||||
while (true) {
|
||||
r = archive_read_data_block(ar, &buff, &size, &offset);
|
||||
if (r == ARCHIVE_EOF)
|
||||
return ARCHIVE_OK;
|
||||
|
||||
if (r < ARCHIVE_OK)
|
||||
return r;
|
||||
|
||||
r = archive_write_data_block(aw, buff, size, offset);
|
||||
if (r < ARCHIVE_OK) {
|
||||
Project::major(archive_error_string(aw));
|
||||
return r;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -1,69 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <optional>
|
||||
#include <filesystem>
|
||||
#include <list>
|
||||
#include <stdio.h>
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <curl/curl.h>
|
||||
#include <archive.h>
|
||||
#include <archive_entry.h>
|
||||
|
||||
class Dependency {
|
||||
public:
|
||||
enum class Type {
|
||||
automatic,
|
||||
simple,
|
||||
mason
|
||||
};
|
||||
|
||||
Dependency(
|
||||
const std::string& path,
|
||||
Type type = Type::automatic,
|
||||
const std::optional<std::string>& name = std::nullopt,
|
||||
const std::optional<std::string>& version = std::nullopt
|
||||
);
|
||||
|
||||
Type getType() const;
|
||||
std::optional<std::string> getName() const;
|
||||
std::optional<std::string> getVersion() const;
|
||||
std::optional<std::filesystem::path> getLocation() const;
|
||||
|
||||
bool prepare(const std::filesystem::path& source, const std::filesystem::path& destination);
|
||||
|
||||
const std::string path;
|
||||
|
||||
private:
|
||||
static size_t writeFile(void* data, size_t size, size_t nmemb, void* file);
|
||||
static size_t writeString(void* data, size_t size, size_t nmemb, void* mem);
|
||||
static int trace(CURL *handle, curl_infotype type, char *data, size_t size, void *clientp);
|
||||
bool download(const std::string& url, const std::filesystem::path& destination);
|
||||
|
||||
bool downloadRepo(
|
||||
const std::filesystem::path& destination,
|
||||
const std::string& protocol,
|
||||
const std::string& host,
|
||||
const std::string& owner,
|
||||
const std::string& repo,
|
||||
std::filesystem::path& out
|
||||
);
|
||||
bool repoGiteaApi1(
|
||||
const std::string& protocol,
|
||||
const std::string& host,
|
||||
const std::string& owner,
|
||||
const std::string& repo,
|
||||
nlohmann::json& json
|
||||
);
|
||||
CURLcode httpGet(const std::string& url, std::string& result, const std::list<std::string_view>& headers = {});
|
||||
|
||||
bool extract(const std::filesystem::path& source, const std::filesystem::path& destination) const;
|
||||
int copy(struct archive *ar, struct archive *aw) const;
|
||||
|
||||
private:
|
||||
Type type;
|
||||
std::optional<std::string> name;
|
||||
std::optional<std::string> version;
|
||||
std::filesystem::path location;
|
||||
};
|
||||
413
src/download.cpp
Normal file
413
src/download.cpp
Normal file
|
|
@ -0,0 +1,413 @@
|
|||
#include "download.h"
|
||||
|
||||
constexpr std::string_view archived("archived");
|
||||
constexpr std::string_view headerAcceptJson("accept: application/json");
|
||||
|
||||
static const std::regex file("(^https?):\\/\\/(?:[\\w\\d\\.\\-\\_]+\\/)+([\\w\\d\\-\\_]+)\\.([\\w\\d\\.]+)");
|
||||
static const std::regex repo("(^https?):\\/\\/([\\w\\d\\.\\-\\_]+)\\/([\\w\\d\\-\\_]+)\\/([\\w\\d\\-\\_]+)(?:(?:\\.git)|\\/)?$");
|
||||
|
||||
static const std::regex archive("tar\\.gz$");
|
||||
|
||||
unsigned int Download::instances(0);
|
||||
AtomicMutex Download::amx{};
|
||||
|
||||
void Download::CurlDeleter::operator()(CURL* curl) const {
|
||||
curl_easy_cleanup(curl);
|
||||
}
|
||||
|
||||
void Download::FileDeleter::operator()(FILE* file) const {
|
||||
fclose(file);
|
||||
}
|
||||
|
||||
Download::Download(
|
||||
const std::string& url,
|
||||
const std::filesystem::path& destination,
|
||||
const std::shared_ptr<Logger>& logger
|
||||
) :
|
||||
Loggable(logger),
|
||||
curl(),
|
||||
url(url),
|
||||
destination(destination),
|
||||
location(std::nullopt)
|
||||
{
|
||||
std::lock_guard lock(amx);
|
||||
if (instances == 0) {
|
||||
CURLcode res = curl_global_init(CURL_GLOBAL_ALL);
|
||||
if (res != CURLE_OK)
|
||||
throw CurlError(std::string("Error initializing curl global ") + curl_easy_strerror(res));
|
||||
}
|
||||
++instances;
|
||||
|
||||
createCurl();
|
||||
}
|
||||
|
||||
Download::~Download() {
|
||||
std::lock_guard lock(amx);
|
||||
--instances;
|
||||
if (instances == 0)
|
||||
curl_global_cleanup();
|
||||
}
|
||||
|
||||
std::optional<std::filesystem::path> Download::getLocation() const {
|
||||
return location;
|
||||
}
|
||||
|
||||
void Download::proceed() {
|
||||
std::optional<RepoInfo> repo = testRepo();
|
||||
if (repo.has_value()) {
|
||||
const RepoInfo& rp = repo.value();
|
||||
std::optional<std::filesystem::path> path = downloadAsRepo(rp);
|
||||
info(url + " has been successfully donwloaded as a repository, extracting");
|
||||
bool success = extract(path.value(), destination);
|
||||
if (success) {
|
||||
location = destination/rp.name;
|
||||
info("Successfully extracted " + url);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
std::optional<FileInfo> file = testFile();
|
||||
if (file.has_value()) {
|
||||
const FileInfo& fl = file.value();
|
||||
std::optional<std::filesystem::path> path = downloadAsFile(fl);
|
||||
if (path.has_value()) {
|
||||
bool success = true;
|
||||
info("Successfully downloaded " + url);
|
||||
if (fl.archive) {
|
||||
info(url + " appears to be an archive, extracting");
|
||||
success = extract(path.value(), destination/fl.name);
|
||||
if (success)
|
||||
info("Successfully extracted " + url);
|
||||
}
|
||||
|
||||
if (success) {
|
||||
location = destination/fl.name;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// CURLcode code = httpDownload(url, path);
|
||||
// if (code == CURLE_OK)
|
||||
// return path;
|
||||
}
|
||||
|
||||
|
||||
std::optional<nlohmann::json> Download::repoInfoGiteaApi1(const RepoInfo& repo) {
|
||||
std::string url = repo.origin() + "/api/v1/repos/" + repo.project();
|
||||
std::string data;
|
||||
CURLcode code = httpGet(url, data, {headerAcceptJson});
|
||||
if (code == CURLE_OK) {
|
||||
try {
|
||||
return nlohmann::json::parse(data);
|
||||
} catch (const nlohmann::json::exception& e) {
|
||||
warn(e.what());
|
||||
}
|
||||
}
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
std::optional<std::filesystem::path> Download::downloadAsRepo(const RepoInfo& repo) {
|
||||
info("Trying Gitea v1 API");
|
||||
std::optional<nlohmann::json> repoInfo = repoInfoGiteaApi1(repo);
|
||||
if (repoInfo.has_value()) {
|
||||
nlohmann::json::const_iterator itr = repoInfo.value().find("default_branch");
|
||||
if (itr != repoInfo.value().end() && itr->is_string()) {
|
||||
std::string branchName = *itr;
|
||||
info("Gitea v1 API seem to have worked");
|
||||
info("Default branch is " + branchName);
|
||||
|
||||
return downloadRepoGiteaApi1(repo, branchName);
|
||||
}
|
||||
}
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
std::optional<std::filesystem::path> Download::downloadAsFile(const FileInfo& file) {
|
||||
std::filesystem::path dst;
|
||||
if (file.archive)
|
||||
dst = destination/archived/file.fileName();
|
||||
else
|
||||
dst = destination/file.fileName();
|
||||
|
||||
CURLcode code = httpDownload(url, dst);
|
||||
if (code == CURLE_OK)
|
||||
return dst;
|
||||
|
||||
minor("Removing " + dst.string());
|
||||
if (std::filesystem::exists(dst))
|
||||
std::filesystem::remove_all(dst);
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
std::optional<std::filesystem::path> Download::downloadRepoGiteaApi1(const RepoInfo& repo, const std::string& branch) {
|
||||
std::string fileName = branch + ".tar.gz";
|
||||
std::string url = repo.origin() + "/api/v1/repos/" + repo.project() + "/archive/" + fileName;
|
||||
std::filesystem::path path = destination/archived/fileName;
|
||||
CURLcode code = httpDownload(url, path);
|
||||
if (code == CURLE_OK)
|
||||
return path;
|
||||
|
||||
minor("Removing " + path.string());
|
||||
if (std::filesystem::exists(path))
|
||||
std::filesystem::remove_all(path);
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
|
||||
CURLcode Download::httpGet(
|
||||
const std::string& url,
|
||||
std::string& result,
|
||||
const std::vector<std::string_view>& headers)
|
||||
{
|
||||
setHeaders(headers);
|
||||
|
||||
curl_easy_setopt(curl.get(), CURLOPT_URL, url.c_str());
|
||||
curl_easy_setopt(curl.get(), CURLOPT_WRITEFUNCTION, writeString);
|
||||
curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, &result);
|
||||
|
||||
return curl_easy_perform(curl.get());
|
||||
}
|
||||
|
||||
CURLcode Download::httpDownload(const std::string& url, const std::filesystem::path& path, const std::vector<std::string_view>& headers) {
|
||||
info("Starting to download " + url + " to " + path.string());
|
||||
std::filesystem::create_directories(path.parent_path());
|
||||
if (std::filesystem::exists(path)) {
|
||||
minor("File " + path.string() + " already exists, will be overwritten");
|
||||
|
||||
if (std::filesystem::is_directory(path))
|
||||
std::filesystem::remove_all(path);
|
||||
}
|
||||
|
||||
std::unique_ptr<FILE, FileDeleter> filePtr(fopen(path.c_str(), "wb"));
|
||||
if (!filePtr)
|
||||
throw FileError("Could not open file " + path.string() + " to write");
|
||||
|
||||
setHeaders(headers);
|
||||
curl_easy_setopt(curl.get(), CURLOPT_URL, url.c_str());
|
||||
curl_easy_setopt(curl.get(), CURLOPT_WRITEFUNCTION, fwrite);
|
||||
curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, filePtr.get());
|
||||
|
||||
CURLcode code = curl_easy_perform(curl.get());
|
||||
if (code == CURLE_OK)
|
||||
info("Successfully downloaded " + url);
|
||||
else
|
||||
warn("Couldn't download file " + url + ": " + curl_easy_strerror(code));
|
||||
|
||||
return code;
|
||||
|
||||
// return res;
|
||||
// if (res != CURLE_OK) {
|
||||
// Project::warn("Couldn't download file " + url + ": " + curl_easy_strerror(res));
|
||||
// } else {
|
||||
// uint32_t code;
|
||||
// curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code);
|
||||
// if (code == 200)
|
||||
// result = true;
|
||||
// else
|
||||
// Project::warn("Couldn't download file " + url + ": response code " + std::to_string(code));
|
||||
// }
|
||||
// } else {
|
||||
// Project::error(std::string("Couldn't open output file ") + destination.c_str());
|
||||
// }
|
||||
}
|
||||
|
||||
void Download::setHeaders(const std::vector<std::string_view>& headers) {
|
||||
struct curl_slist* curlHeaders = nullptr;
|
||||
for (const std::string_view& header : headers)
|
||||
curlHeaders = curl_slist_append(curlHeaders, header.data());
|
||||
|
||||
curl_easy_setopt(curl.get(), CURLOPT_HTTPHEADER, curlHeaders);
|
||||
}
|
||||
|
||||
void Download::createCurl() {
|
||||
CURL* crl = curl_easy_init();
|
||||
if (!crl)
|
||||
throw CurlError("Error creating curl instalce");
|
||||
|
||||
curl_easy_setopt(crl, CURLOPT_USERAGENT, "libcurl-agent/1.0" );
|
||||
curl_easy_setopt(crl, CURLOPT_VERBOSE, 1L);
|
||||
curl_easy_setopt(crl, CURLOPT_DEBUGFUNCTION, trace);
|
||||
curl_easy_setopt(crl, CURLOPT_DEBUGDATA, this);
|
||||
curl_easy_setopt(crl, CURLOPT_FOLLOWLOCATION, 1L);
|
||||
curl_easy_setopt(crl, CURLOPT_NOPROGRESS, 1L);
|
||||
curl = std::unique_ptr<CURL, CurlDeleter>(crl);
|
||||
}
|
||||
|
||||
std::optional<Download::FileInfo> Download::testFile() const {
|
||||
std::smatch results;
|
||||
if (std::regex_search(url, results, file)) {
|
||||
info(url + " appears to be a file");
|
||||
std::string ext(results[3]);
|
||||
return FileInfo {
|
||||
results[1],
|
||||
results[2],
|
||||
ext,
|
||||
std::regex_search(ext, archive)
|
||||
};
|
||||
}
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
std::optional<Download::RepoInfo> Download::testRepo() const {
|
||||
std::smatch results;
|
||||
if (std::regex_search(url, results, repo)) {
|
||||
info(url + " appears to be a git repository");
|
||||
return RepoInfo {
|
||||
results[1],
|
||||
results[2],
|
||||
results[3],
|
||||
results[4]
|
||||
};
|
||||
}
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
bool Download::extract(const std::filesystem::path& source, const std::filesystem::path& destination) const {
|
||||
int flags = ARCHIVE_EXTRACT_TIME;
|
||||
flags |= ARCHIVE_EXTRACT_PERM;
|
||||
flags |= ARCHIVE_EXTRACT_ACL;
|
||||
flags |= ARCHIVE_EXTRACT_FFLAGS;
|
||||
|
||||
struct archive* a = archive_read_new();
|
||||
struct archive* ext = archive_write_disk_new();
|
||||
struct archive_entry *entry;
|
||||
archive_read_support_format_all(a);
|
||||
archive_read_support_filter_all(a);
|
||||
archive_write_disk_set_options(ext, flags);
|
||||
archive_write_disk_set_standard_lookup(ext);
|
||||
|
||||
bool result = true;
|
||||
bool readOpen = false;
|
||||
bool writeOpen = false;
|
||||
int r = archive_read_open_filename(a, source.c_str(), 10240);
|
||||
if (r) {
|
||||
major("Couldn't open file " + source.string());
|
||||
result = false;
|
||||
} else {
|
||||
readOpen = true;
|
||||
}
|
||||
|
||||
while (result) {
|
||||
r = archive_read_next_header(a, &entry);
|
||||
if (r == ARCHIVE_EOF)
|
||||
break;
|
||||
|
||||
if (r < ARCHIVE_OK)
|
||||
major(archive_error_string(a));
|
||||
|
||||
if (r < ARCHIVE_WARN)
|
||||
break;
|
||||
|
||||
std::string fileName(archive_entry_pathname(entry));
|
||||
std::filesystem::path filePath = destination/fileName;
|
||||
debug("Extracting " + filePath.string());
|
||||
if (std::filesystem::exists(filePath))
|
||||
minor(filePath.string() + " exists, overwriting");
|
||||
|
||||
archive_entry_set_pathname_utf8(entry, filePath.c_str());
|
||||
|
||||
r = archive_write_header(ext, entry);
|
||||
if (r < ARCHIVE_OK) {
|
||||
major(archive_error_string(ext));
|
||||
} else if (archive_entry_size(entry) > 0) {
|
||||
writeOpen = true;
|
||||
r = copy(a, ext);
|
||||
if (r < ARCHIVE_OK)
|
||||
major(archive_error_string(ext));
|
||||
|
||||
if (r < ARCHIVE_WARN)
|
||||
break;
|
||||
}
|
||||
r = archive_write_finish_entry(ext);
|
||||
if (r < ARCHIVE_OK)
|
||||
major(archive_error_string(ext));
|
||||
|
||||
if (r < ARCHIVE_WARN)
|
||||
break;
|
||||
}
|
||||
|
||||
if (readOpen)
|
||||
archive_read_close(a);
|
||||
|
||||
if (writeOpen)
|
||||
archive_write_close(ext);
|
||||
|
||||
archive_read_free(a);
|
||||
archive_write_free(ext);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
int Download::copy(struct archive* ar, struct archive* aw) const {
|
||||
int r;
|
||||
const void *buff;
|
||||
size_t size;
|
||||
la_int64_t offset;
|
||||
|
||||
while (true) {
|
||||
r = archive_read_data_block(ar, &buff, &size, &offset);
|
||||
if (r == ARCHIVE_EOF)
|
||||
return ARCHIVE_OK;
|
||||
|
||||
if (r < ARCHIVE_OK)
|
||||
return r;
|
||||
|
||||
r = archive_write_data_block(aw, buff, size, offset);
|
||||
if (r < ARCHIVE_OK) {
|
||||
major(archive_error_string(aw));
|
||||
return r;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int Download::trace(CURL* handle, curl_infotype type, char* data, size_t size, void* clientp) {
|
||||
(void)(handle);
|
||||
switch (type) {
|
||||
case CURLINFO_TEXT: {
|
||||
std::string message(data, size);
|
||||
if (message[size - 1] == '\n')
|
||||
message = message.substr(0, size - 1);
|
||||
|
||||
static_cast<Download*>(clientp)->debug(message);
|
||||
} break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
size_t Download::writeString(void* data, size_t size, size_t nmemb, void* mem) {
|
||||
size_t finalSize = size * nmemb;
|
||||
std::string* string = static_cast<std::string*>(mem);
|
||||
string->append(static_cast<char*>(data), finalSize);
|
||||
|
||||
return finalSize;
|
||||
}
|
||||
|
||||
Download::CurlError::CurlError(const std::string& message) :
|
||||
std::runtime_error(message) {}
|
||||
|
||||
Download::FileError::FileError(const std::string& message) :
|
||||
std::runtime_error(message) {}
|
||||
|
||||
std::string Download::RepoInfo::origin() const {
|
||||
return protocol + "://" + host;
|
||||
}
|
||||
|
||||
std::string Download::RepoInfo::project() const {
|
||||
return owner + "/" + name;
|
||||
}
|
||||
|
||||
std::string Download::FileInfo::fileName() const {
|
||||
return name + "." + extension;
|
||||
}
|
||||
|
||||
103
src/download.h
Normal file
103
src/download.h
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
#pragma once
|
||||
|
||||
#include <mutex>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <string_view>
|
||||
#include <filesystem>
|
||||
#include <optional>
|
||||
#include <exception>
|
||||
#include <regex>
|
||||
|
||||
#include <curl/curl.h>
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <archive.h>
|
||||
#include <archive_entry.h>
|
||||
|
||||
#include "atomicmutex.h"
|
||||
#include "loggable.h"
|
||||
|
||||
class Collection;
|
||||
|
||||
class Download : protected Loggable {
|
||||
struct RepoInfo;
|
||||
struct FileInfo;
|
||||
struct CurlDeleter {
|
||||
void operator() (CURL* curl) const;
|
||||
};
|
||||
struct FileDeleter {
|
||||
void operator() (FILE* file) const;
|
||||
};
|
||||
class CurlError;
|
||||
class FileError;
|
||||
|
||||
public:
|
||||
explicit Download(
|
||||
const std::string& url,
|
||||
const std::filesystem::path& destination,
|
||||
const std::shared_ptr<Logger>& logger
|
||||
);
|
||||
~Download();
|
||||
|
||||
void proceed();
|
||||
std::optional<std::filesystem::path> getLocation() const;
|
||||
|
||||
private:
|
||||
std::optional<FileInfo> testFile() const;
|
||||
std::optional<RepoInfo> testRepo() const;
|
||||
|
||||
std::optional<std::filesystem::path> downloadAsRepo(const RepoInfo& repo);
|
||||
std::optional<std::filesystem::path> downloadAsFile(const FileInfo& file);
|
||||
|
||||
std::optional<nlohmann::json> repoInfoGiteaApi1(const RepoInfo& repo);
|
||||
std::optional<std::filesystem::path> downloadRepoGiteaApi1(const RepoInfo& repo, const std::string& branch);
|
||||
|
||||
bool extract(const std::filesystem::path& source, const std::filesystem::path& destination) const;
|
||||
int copy(struct archive *ar, struct archive *aw) const;
|
||||
|
||||
CURLcode httpGet(const std::string& url, std::string& result, const std::vector<std::string_view>& headers = {});
|
||||
CURLcode httpDownload(const std::string& url, const std::filesystem::path& path, const std::vector<std::string_view>& headers = {});
|
||||
void setHeaders (const std::vector<std::string_view>& headers);
|
||||
|
||||
static size_t writeString(void* data, size_t size, size_t nmemb, void* mem);
|
||||
static int trace(CURL *handle, curl_infotype type, char *data, size_t size, void *clientp);
|
||||
void createCurl();
|
||||
|
||||
private:
|
||||
static AtomicMutex amx;
|
||||
static unsigned int instances;
|
||||
std::unique_ptr<CURL, CurlDeleter> curl;
|
||||
std::string url;
|
||||
std::filesystem::path destination;
|
||||
std::optional<std::filesystem::path> location;
|
||||
};
|
||||
|
||||
struct Download::RepoInfo {
|
||||
std::string protocol;
|
||||
std::string host;
|
||||
std::string owner;
|
||||
std::string name;
|
||||
|
||||
std::string origin() const;
|
||||
std::string project() const;
|
||||
};
|
||||
|
||||
struct Download::FileInfo {
|
||||
std::string protocol;
|
||||
std::string name;
|
||||
std::string extension;
|
||||
bool archive;
|
||||
|
||||
std::string fileName() const;
|
||||
};
|
||||
|
||||
class Download::CurlError : public std::runtime_error {
|
||||
public:
|
||||
explicit CurlError(const std::string& message);
|
||||
};
|
||||
|
||||
class Download::FileError : public std::runtime_error {
|
||||
public:
|
||||
explicit FileError(const std::string& message);
|
||||
};
|
||||
37
src/loggable.cpp
Normal file
37
src/loggable.cpp
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
#include "loggable.h"
|
||||
|
||||
Loggable::Loggable(const std::shared_ptr<Logger>& logger):
|
||||
logger(logger)
|
||||
{}
|
||||
|
||||
void Loggable::log(Logger::Severity severity, const std::string& message) const {
|
||||
logger->log(severity, message);
|
||||
}
|
||||
|
||||
void Loggable::debug(const std::string& message) const {
|
||||
log(Logger::Severity::debug, message);
|
||||
}
|
||||
|
||||
void Loggable::info(const std::string& message) const {
|
||||
log(Logger::Severity::info, message);
|
||||
}
|
||||
|
||||
void Loggable::minor(const std::string& message) const {
|
||||
log(Logger::Severity::minor, message);
|
||||
}
|
||||
|
||||
void Loggable::major(const std::string& message) const {
|
||||
log(Logger::Severity::major, message);
|
||||
}
|
||||
|
||||
void Loggable::warn(const std::string& message) const {
|
||||
log(Logger::Severity::warning, message);
|
||||
}
|
||||
|
||||
void Loggable::error(const std::string& message) const {
|
||||
log(Logger::Severity::error, message);
|
||||
}
|
||||
|
||||
void Loggable::fatal(const std::string& message) const {
|
||||
log(Logger::Severity::fatal, message);
|
||||
}
|
||||
25
src/loggable.h
Normal file
25
src/loggable.h
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "loggger.h"
|
||||
|
||||
class Loggable {
|
||||
public:
|
||||
explicit Loggable(const std::shared_ptr<Logger>& logger);
|
||||
virtual ~Loggable() = default;
|
||||
|
||||
void debug(const std::string& message) const;
|
||||
void info(const std::string& message) const;
|
||||
void minor(const std::string& message) const;
|
||||
void major(const std::string& message) const;
|
||||
void warn(const std::string& message) const;
|
||||
void error(const std::string& message) const;
|
||||
void fatal(const std::string& message) const;
|
||||
|
||||
private:
|
||||
void log(Logger::Severity severity, const std::string& message) const;
|
||||
|
||||
protected:
|
||||
std::shared_ptr<Logger> logger;
|
||||
};
|
||||
|
|
@ -19,7 +19,7 @@ int main(int argc, char *argv[]) {
|
|||
|
||||
|
||||
std::shared_ptr<Logger> logger = std::make_shared<Logger>(Logger::Severity::debug);
|
||||
std::shared_ptr<TaskManager> taskManager = std::make_shared<TaskManager>();
|
||||
std::shared_ptr<TaskManager> taskManager = std::make_shared<TaskManager>(logger);
|
||||
std::shared_ptr<Collection> collection = std::make_shared<Collection>(secondArg, "", logger, taskManager);
|
||||
|
||||
taskManager->start();
|
||||
|
|
|
|||
237
src/project.cpp
237
src/project.cpp
|
|
@ -1,237 +0,0 @@
|
|||
#include "project.h"
|
||||
|
||||
#include <string_view>
|
||||
#include <exception>
|
||||
|
||||
using json = nlohmann::json;
|
||||
|
||||
constexpr std::string_view entry("mason.json");
|
||||
|
||||
std::map<std::string, Dependency::Type> types({
|
||||
{"simple", Dependency::Type::simple},
|
||||
{"none", Dependency::Type::simple},
|
||||
{"mason", Dependency::Type::mason},
|
||||
{"auto", Dependency::Type::automatic},
|
||||
{"automatic", Dependency::Type::automatic}
|
||||
});
|
||||
|
||||
Logger* Project::logger = nullptr;
|
||||
|
||||
Project::Project(const std::filesystem::path& location, const std::filesystem::path& destination):
|
||||
parent(nullptr),
|
||||
location(location),
|
||||
destination(destination),
|
||||
state(State::unknown),
|
||||
name(),
|
||||
dependencies(),
|
||||
root(logger == nullptr)
|
||||
{
|
||||
if (root) {
|
||||
curl_global_init(CURL_GLOBAL_ALL);
|
||||
logger = new Logger(Logger::Severity::debug);
|
||||
}
|
||||
}
|
||||
|
||||
Project::Project(const std::filesystem::path& location, const std::filesystem::path& destination, Project* parent):
|
||||
parent(parent),
|
||||
location(location),
|
||||
destination(destination),
|
||||
state(State::unknown),
|
||||
name(),
|
||||
dependencies(),
|
||||
root(false) {}
|
||||
|
||||
Project::~Project() {
|
||||
if (root) {
|
||||
curl_global_cleanup();
|
||||
|
||||
delete logger;
|
||||
logger = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
bool Project::read() {
|
||||
if (state == State::read)
|
||||
return true;
|
||||
|
||||
std::filesystem::path entryPointPath;
|
||||
try {
|
||||
location = std::filesystem::canonical(location);
|
||||
entryPointPath = location / entry;
|
||||
} catch (const std::exception& e) {
|
||||
fatal(e.what());
|
||||
state = State::error;
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
std::filesystem::create_directories(destination);
|
||||
destination = std::filesystem::canonical(destination);
|
||||
} catch (const std::exception& e) {
|
||||
fatal(e.what());
|
||||
state = State::error;
|
||||
return false;
|
||||
}
|
||||
|
||||
std::ifstream file(entryPointPath);
|
||||
if (!file.is_open()) {
|
||||
fatal("couldn't open " + entryPointPath.string());
|
||||
state = State::error;
|
||||
return false;
|
||||
}
|
||||
|
||||
json data;
|
||||
try {
|
||||
data = json::parse(file);
|
||||
} catch (const json::exception& e) {
|
||||
fatal(e.what());
|
||||
state = State::error;
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
parse(data);
|
||||
} catch (const std::exception& e) {
|
||||
fatal(e.what());
|
||||
state = State::error;
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void Project::parse(const json& data) {
|
||||
const json& nm = data.at("name");
|
||||
if (!nm.is_string())
|
||||
throw 1;
|
||||
|
||||
name = nm;
|
||||
|
||||
const json& dpd = data.at("dependencies");
|
||||
if (!dpd.is_array())
|
||||
throw 1;
|
||||
|
||||
for (const json& dep : dpd) {
|
||||
switch (dep.type()) {
|
||||
case json::value_t::string:
|
||||
createDepencencyFromString(dep);
|
||||
break;
|
||||
case json::value_t::object:
|
||||
createDepencencyFromObject(dep);
|
||||
break;
|
||||
default:
|
||||
throw 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void Project::createDepencencyFromString(const std::string& entry) {
|
||||
dependencies.emplace(entry, entry);
|
||||
}
|
||||
|
||||
void Project::createDepencencyFromObject(const nlohmann::json& entry) {
|
||||
std::string url;
|
||||
json::const_iterator itr = entry.find("url");
|
||||
if (itr == entry.end()) {
|
||||
itr = entry.find("path");
|
||||
if (itr == entry.end())
|
||||
throw 1;
|
||||
}
|
||||
if (!itr->is_string())
|
||||
throw 1;
|
||||
|
||||
url = *itr;
|
||||
|
||||
std::optional<std::string> name = std::nullopt;
|
||||
itr = entry.find("name");
|
||||
if (itr != entry.end() && itr->is_string())
|
||||
name = *itr;
|
||||
|
||||
std::optional<std::string> version = std::nullopt;
|
||||
itr = entry.find("version");
|
||||
if (itr != entry.end() && itr->is_string())
|
||||
version = *itr;
|
||||
|
||||
Dependency::Type type = Dependency::Type::automatic;
|
||||
itr = entry.find("type");
|
||||
if (itr != entry.end() && itr->is_string()) {
|
||||
std::map<std::string, Dependency::Type>::const_iterator titr = types.find(*itr);
|
||||
if (titr != types.end())
|
||||
type = titr->second;
|
||||
}
|
||||
|
||||
dependencies.emplace(url, Dependency{url, type, name, version});
|
||||
}
|
||||
|
||||
void Project::discover() {
|
||||
int fine = 0;
|
||||
for (std::pair<const std::string, Dependency>& pair : dependencies) {
|
||||
Dependency& dep = pair.second;
|
||||
bool success = dep.prepare(location, destination);
|
||||
if (success) {
|
||||
switch (dep.getType()) {
|
||||
case Dependency::Type::mason:
|
||||
break;
|
||||
case Dependency::Type::simple:
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
fine++;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
uint32_t Project::dependenciesCount() const {
|
||||
return dependencies.size();
|
||||
}
|
||||
|
||||
void Project::log(Logger::Severity severity, const std::string& message) {
|
||||
if (logger != nullptr)
|
||||
logger->log(severity, message);
|
||||
}
|
||||
|
||||
void Project::info(const std::string& message) {
|
||||
log(Logger::Severity::info, message);
|
||||
}
|
||||
|
||||
void Project::debug(const std::string& message) {
|
||||
log(Logger::Severity::debug, message);
|
||||
}
|
||||
|
||||
void Project::minor(const std::string& message) {
|
||||
log(Logger::Severity::minor, message);
|
||||
}
|
||||
|
||||
void Project::major(const std::string& message) {
|
||||
log(Logger::Severity::major, message);
|
||||
}
|
||||
|
||||
void Project::error(const std::string& message) {
|
||||
log(Logger::Severity::error, message);
|
||||
}
|
||||
|
||||
void Project::warn(const std::string& message) {
|
||||
log(Logger::Severity::warning, message);
|
||||
}
|
||||
|
||||
void Project::fatal(const std::string& message) {
|
||||
log(Logger::Severity::fatal, message);
|
||||
}
|
||||
|
||||
void Project::printLog() {
|
||||
if (logger != nullptr)
|
||||
logger->printLog();
|
||||
}
|
||||
|
||||
Project::State Project::getStatus() const {
|
||||
return state;
|
||||
}
|
||||
|
||||
std::string Project::getName() const {
|
||||
return name;
|
||||
}
|
||||
|
|
@ -1,64 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
#include <filesystem>
|
||||
#include <string>
|
||||
#include <fstream>
|
||||
#include <list>
|
||||
#include <map>
|
||||
#include <stdint.h>
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <curl/curl.h>
|
||||
|
||||
#include "dependency.h"
|
||||
#include "loggger.h"
|
||||
|
||||
class Project {
|
||||
public:
|
||||
enum class State {
|
||||
unknown,
|
||||
read,
|
||||
discovering,
|
||||
error
|
||||
};
|
||||
|
||||
Project(const std::filesystem::path& location, const std::filesystem::path& destination);
|
||||
~Project();
|
||||
|
||||
bool read();
|
||||
void discover();
|
||||
State getStatus() const;
|
||||
std::string getName() const;
|
||||
uint32_t dependenciesCount() const;
|
||||
|
||||
static void log(Logger::Severity severity, const std::string& message);
|
||||
static void debug(const std::string& message);
|
||||
static void info(const std::string& message);
|
||||
static void minor(const std::string& message);
|
||||
static void major(const std::string& message);
|
||||
static void warn(const std::string& message);
|
||||
static void error(const std::string& message);
|
||||
static void fatal(const std::string& message);
|
||||
static void printLog();
|
||||
|
||||
static void addProject(const std::filesystem::path& location, const std::filesystem::path& destination);
|
||||
|
||||
private:
|
||||
Project(const std::filesystem::path& location, const std::filesystem::path& destination, Project* parent);
|
||||
|
||||
void parse(const nlohmann::json& json);
|
||||
void createDepencencyFromString(const std::string& entry);
|
||||
void createDepencencyFromObject(const nlohmann::json& entry);
|
||||
|
||||
private:
|
||||
Project* parent;
|
||||
std::filesystem::path location;
|
||||
std::filesystem::path destination;
|
||||
State state;
|
||||
std::string name;
|
||||
std::map<std::string, Dependency> dependencies;
|
||||
const bool root;
|
||||
|
||||
static Logger* logger;
|
||||
|
||||
};
|
||||
120
src/taskmanager.cpp
Normal file
120
src/taskmanager.cpp
Normal file
|
|
@ -0,0 +1,120 @@
|
|||
#include "taskmanager.h"
|
||||
|
||||
TaskManager::TaskManager(const std::shared_ptr<Logger>& logger) :
|
||||
Loggable(logger),
|
||||
running(false),
|
||||
stopping(false),
|
||||
maxThreads(std::thread::hardware_concurrency()),
|
||||
activeThreads(0),
|
||||
tasks(),
|
||||
mutex(),
|
||||
loopConditional(),
|
||||
waitConditional(),
|
||||
threads()
|
||||
{
|
||||
threads.reserve(maxThreads);
|
||||
}
|
||||
|
||||
TaskManager::~TaskManager() {
|
||||
stop();
|
||||
}
|
||||
|
||||
void TaskManager::queue(const Job& job) {
|
||||
std::unique_lock lock(mutex);
|
||||
tasks.emplace(job, std::nullopt);
|
||||
lock.unlock();
|
||||
loopConditional.notify_one();
|
||||
}
|
||||
|
||||
void TaskManager::queue(const Job& job, const Result& result) {
|
||||
std::unique_lock lock(mutex);
|
||||
tasks.emplace(job, result);
|
||||
lock.unlock();
|
||||
loopConditional.notify_one();
|
||||
}
|
||||
|
||||
|
||||
void TaskManager::start() {
|
||||
std::lock_guard lock(mutex);
|
||||
if (running)
|
||||
return;
|
||||
|
||||
debug("Starting " + std::to_string(maxThreads) + " threads");
|
||||
for (uint32_t i = 0; i < maxThreads; ++i)
|
||||
threads.emplace_back(&TaskManager::loop, this);
|
||||
|
||||
running = true;
|
||||
}
|
||||
|
||||
void TaskManager::stop() {
|
||||
std::unique_lock lock(mutex);
|
||||
if (!running)
|
||||
return;
|
||||
|
||||
debug("Stopping task manager");
|
||||
stopping = true;
|
||||
lock.unlock();
|
||||
|
||||
loopConditional.notify_all();
|
||||
for (std::thread& thread : threads)
|
||||
thread.join();
|
||||
|
||||
threads.clear();
|
||||
running = false;
|
||||
}
|
||||
|
||||
void TaskManager::loop() {
|
||||
//debug("Thread " + std::to_string(std::this_thread::get_id()) + " entered the loop");
|
||||
debug("Thread entered the loop");
|
||||
while (true) {
|
||||
Task task;
|
||||
std::unique_lock lock(mutex);
|
||||
while (!stopping && tasks.empty())
|
||||
loopConditional.wait(lock);
|
||||
|
||||
if (stopping)
|
||||
break;
|
||||
|
||||
++activeThreads;
|
||||
task = tasks.front();
|
||||
tasks.pop();
|
||||
//debug("Thread took a task");
|
||||
lock.unlock();
|
||||
|
||||
executeTask(task);
|
||||
|
||||
lock.lock();
|
||||
--activeThreads;
|
||||
lock.unlock();
|
||||
waitConditional.notify_all();
|
||||
}
|
||||
debug("Thread left the loop");
|
||||
}
|
||||
|
||||
void TaskManager::executeTask(const Task& task) const {
|
||||
try {
|
||||
task.first();
|
||||
} catch (const std::exception& e) {
|
||||
if (task.second.has_value())
|
||||
task.second.value()(&e);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
if (task.second.has_value())
|
||||
task.second.value()(std::nullopt);
|
||||
} catch (...) {}
|
||||
}
|
||||
|
||||
|
||||
bool TaskManager::busy() const {
|
||||
std::lock_guard lock(mutex);
|
||||
return activeThreads == 0;
|
||||
}
|
||||
|
||||
void TaskManager::wait() const {
|
||||
std::unique_lock lock(mutex);
|
||||
while (activeThreads != 0 || !tasks.empty())
|
||||
waitConditional.wait(lock);
|
||||
}
|
||||
47
src/taskmanager.h
Normal file
47
src/taskmanager.h
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <queue>
|
||||
#include <vector>
|
||||
#include <thread>
|
||||
#include <mutex>
|
||||
#include <condition_variable>
|
||||
#include <functional>
|
||||
#include <optional>
|
||||
#include <exception>
|
||||
#include <memory>
|
||||
|
||||
#include "loggable.h"
|
||||
|
||||
class TaskManager : public Loggable {
|
||||
public:
|
||||
using Error = std::optional<const std::exception*>;
|
||||
using Job = std::function<void ()>;
|
||||
using Result = std::function<void (Error)>;
|
||||
using Task = std::pair<Job, std::optional<Result>>;
|
||||
|
||||
TaskManager(const std::shared_ptr<Logger>& logger);
|
||||
~TaskManager();
|
||||
|
||||
void start();
|
||||
void stop();
|
||||
void wait() const;
|
||||
void queue(const Job& job);
|
||||
void queue(const Job& job, const Result& result);
|
||||
bool busy() const;
|
||||
|
||||
private:
|
||||
void loop();
|
||||
void executeTask(const Task& task) const;
|
||||
|
||||
private:
|
||||
bool running;
|
||||
bool stopping;
|
||||
uint32_t maxThreads;
|
||||
uint32_t activeThreads;
|
||||
std::queue<Task> tasks;
|
||||
mutable std::mutex mutex;
|
||||
mutable std::condition_variable loopConditional;
|
||||
mutable std::condition_variable waitConditional;
|
||||
std::vector<std::thread> threads;
|
||||
};
|
||||
Loading…
Add table
Add a link
Reference in a new issue