some more thoughts of how to organize it better
This commit is contained in:
parent
02df7de0c3
commit
7b9112c0dd
GPG Key ID:
9B203B252A63EE38
|
|
@ -3,6 +3,8 @@ set(SOURCES
|
|||
collection.cpp
|
||||
loggable.cpp
|
||||
taskmanager.cpp
|
||||
download.cpp
|
||||
atomicmutex.cpp
|
||||
)
|
||||
|
||||
set(HEADERS
|
||||
|
|
@ -10,6 +12,8 @@ set(HEADERS
|
|||
collection.h
|
||||
loggable.h
|
||||
taskmanager.h
|
||||
download.h
|
||||
atomicmutex.h
|
||||
)
|
||||
|
||||
target_sources(${PROJECT_NAME} PRIVATE ${SOURCES})
|
||||
|
|
|
|||
|
|
@ -0,0 +1,13 @@
|
|||
#include "atomicmutex.h"
|
||||
|
||||
AtomicMutex::AtomicMutex():
|
||||
flag()
|
||||
{}
|
||||
|
||||
void AtomicMutex::lock() noexcept {
|
||||
while (flag.test_and_set());
|
||||
}
|
||||
|
||||
void AtomicMutex::unlock() noexcept {
|
||||
flag.clear();
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
#pragma once
|
||||
|
||||
#include <atomic>
|
||||
|
||||
class AtomicMutex {
|
||||
public:
|
||||
explicit AtomicMutex();
|
||||
void lock() noexcept;
|
||||
void unlock() noexcept;
|
||||
|
||||
private:
|
||||
std::atomic_flag flag;
|
||||
};
|
||||
|
|
@ -6,13 +6,20 @@
|
|||
#include <set>
|
||||
#include <exception>
|
||||
|
||||
class Collection {
|
||||
#include "loggable.h"
|
||||
#include "component.h"
|
||||
#include "taskmanager.h"
|
||||
|
||||
class Collection : protected Loggable {
|
||||
public:
|
||||
class UnknownSource;
|
||||
class DuplicateSource;
|
||||
class DuplicatePath;
|
||||
|
||||
Collection();
|
||||
Collection(
|
||||
const std::shared_ptr<Logger>& logger,
|
||||
const std::shared_ptr<TaskManager>& taskManager
|
||||
);
|
||||
|
||||
unsigned int sourcesTotal() const;
|
||||
unsigned int sourcesPending() const;
|
||||
|
|
@ -21,13 +28,21 @@ public:
|
|||
|
||||
bool hasSource(const std::string& source) const;
|
||||
void addSource(const std::string& source);
|
||||
void sourceError(const std::string& source);
|
||||
void sourceSuccess(const std::string& source, const std::filesystem::path& path);
|
||||
|
||||
private:
|
||||
void sourceDownloadError(const std::string& source);
|
||||
void sourceBuildError(const std::string& source);
|
||||
void sourceDownaloded(const std::string& source);
|
||||
void sourceBuilt(const std::string& source);
|
||||
|
||||
private:
|
||||
std::shared_ptr<TaskManager> taskManager;
|
||||
std::set<std::string> errorSources;
|
||||
std::set<std::string> pendingSources;
|
||||
std::map<std::string, std::filesystem::path> successSources; //would be nice to have a bimap here
|
||||
std::map<std::string, std::unique_ptr<Component>> failedSources;
|
||||
std::map<std::string, std::unique_ptr<Component>> downloadedSources;
|
||||
std::map<std::string, std::unique_ptr<Component>> buildingSources;
|
||||
std::map<std::string, std::unique_ptr<Component>> successSources;
|
||||
};
|
||||
|
||||
class Collection::UnknownSource : public std::runtime_error {
|
||||
|
|
|
|||
|
|
@ -3,23 +3,24 @@
|
|||
#include <array>
|
||||
#include <string_view>
|
||||
|
||||
#include "collection.h"
|
||||
|
||||
constexpr std::array<std::string_view, Component::done + 1> stringStates {
|
||||
"initial",
|
||||
"reading",
|
||||
"ready",
|
||||
"building",
|
||||
"error",
|
||||
"done"
|
||||
};
|
||||
|
||||
Component::Component(
|
||||
const std::filesystem::path& path,
|
||||
const std::shared_ptr<Collection>& collection,
|
||||
const std::shared_ptr<TaskManager>& taskManager,
|
||||
const std::shared_ptr<Logger>& logger
|
||||
):
|
||||
Loggable(logger),
|
||||
collection(collection),
|
||||
taskManager(taskManager),
|
||||
location(path)
|
||||
{}
|
||||
|
||||
|
|
@ -27,14 +28,13 @@ void Component::read() {
|
|||
if (state != initial)
|
||||
throw WrongState(state, "read");
|
||||
|
||||
try {
|
||||
collection->addSource(location);
|
||||
} catch (const Collection::DuplicateSource e) {
|
||||
state = error;
|
||||
throw e;
|
||||
}
|
||||
// try {
|
||||
// collection->addSource(location);
|
||||
// } catch (const Collection::DuplicateSource e) {
|
||||
// state = error;
|
||||
// throw e;
|
||||
// }
|
||||
state = reading;
|
||||
taskManager->queue(std::bind(&Component::performRead, this));
|
||||
}
|
||||
|
||||
void Component::build(const std::filesystem::path& destination, const std::string& target) {
|
||||
|
|
@ -42,13 +42,6 @@ void Component::build(const std::filesystem::path& destination, const std::strin
|
|||
throw WrongState(state, "build");
|
||||
|
||||
state = building;
|
||||
taskManager->queue(std::bind(&Component::performBuild, this, destination, target));
|
||||
}
|
||||
|
||||
void Component::performRead() {
|
||||
}
|
||||
|
||||
void Component::performBuild(std::filesystem::path destination, std::string target) {
|
||||
}
|
||||
|
||||
Component::State Component::getState() const {
|
||||
|
|
|
|||
|
|
@ -8,8 +8,8 @@
|
|||
|
||||
#include "loggable.h"
|
||||
#include "loggger.h"
|
||||
#include "collection.h"
|
||||
#include "taskmanager.h"
|
||||
|
||||
class Collection;
|
||||
|
||||
class Component : protected Loggable {
|
||||
public:
|
||||
|
|
@ -33,7 +33,6 @@ public:
|
|||
Component(
|
||||
const std::filesystem::path& path,
|
||||
const std::shared_ptr<Collection>& collection,
|
||||
const std::shared_ptr<TaskManager>& taskManager,
|
||||
const std::shared_ptr<Logger>& logger
|
||||
);
|
||||
|
||||
|
|
@ -43,15 +42,10 @@ public:
|
|||
void read();
|
||||
void build(const std::filesystem::path& destination, const std::string& target);
|
||||
|
||||
private:
|
||||
void performRead();
|
||||
void performBuild(std::filesystem::path destination, std::string target);
|
||||
|
||||
private:
|
||||
State state;
|
||||
Type type;
|
||||
std::shared_ptr<Collection> collection;
|
||||
std::shared_ptr<TaskManager> taskManager;
|
||||
std::filesystem::path location;
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,216 @@
|
|||
#include "download.h"
|
||||
|
||||
#include <regex>
|
||||
|
||||
#include "collection.h"
|
||||
|
||||
constexpr std::string_view downloads("downloads");
|
||||
constexpr std::string_view headerAcceptJson("accept: application/json");
|
||||
|
||||
static const std::regex repo("(^https?):\\/\\/([\\w\\d\\.\\-\\_]+)\\/([\\w\\d\\-\\_]+)\\/([\\w\\d\\-\\_]+)");
|
||||
|
||||
unsigned int Download::instances(0);
|
||||
AtomicMutex Download::amx{};
|
||||
|
||||
void Download::CurlDeleter::operator()(CURL* curl) const {
|
||||
curl_easy_cleanup(curl);
|
||||
}
|
||||
|
||||
void Download::FileDeleter::operator()(FILE* file) const {
|
||||
fclose(file);
|
||||
}
|
||||
|
||||
|
||||
Download::Download(
|
||||
const std::string& url,
|
||||
const std::filesystem::path& destination,
|
||||
const std::shared_ptr<Collection>& collection,
|
||||
const std::shared_ptr<Logger>& logger
|
||||
) :
|
||||
Loggable(logger),
|
||||
curl(),
|
||||
collection(collection),
|
||||
url(url),
|
||||
destination(destination)
|
||||
{
|
||||
std::lock_guard lock(amx);
|
||||
if (instances == 0) {
|
||||
CURLcode res = curl_global_init(CURL_GLOBAL_ALL);
|
||||
if (res != CURLE_OK)
|
||||
throw CurlError(std::string("Error initializing curl global ") + curl_easy_strerror(res));
|
||||
}
|
||||
|
||||
++instances;
|
||||
|
||||
createCurl();
|
||||
}
|
||||
|
||||
Download::~Download() {
|
||||
std::lock_guard lock(amx);
|
||||
--instances;
|
||||
if (instances == 0)
|
||||
curl_global_cleanup();
|
||||
}
|
||||
|
||||
|
||||
void Download::proceed() {
|
||||
bool success = false;
|
||||
std::optional<RepoInfo> repo = testRepo();
|
||||
if (repo.has_value()) {
|
||||
std::optional<std::filesystem::path> path = downloadAsRepo(repo.value());
|
||||
}
|
||||
}
|
||||
|
||||
std::optional<nlohmann::json> Download::repoInfoGiteaApi1(const RepoInfo& repo) {
|
||||
std::string url = repo.origin() + "/api/v1/repos/" + repo.project();
|
||||
std::string data;
|
||||
CURLcode code = httpGet(url, data, {headerAcceptJson});
|
||||
if (code == CURLE_OK) {
|
||||
try {
|
||||
return nlohmann::json::parse(data);
|
||||
} catch (const nlohmann::json::exception& e) {
|
||||
warn(e.what());
|
||||
}
|
||||
}
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
std::optional<std::filesystem::path> Download::downloadAsRepo(const RepoInfo& repo) {
|
||||
info("Trying Gitea v1 API");
|
||||
std::optional<nlohmann::json> repoInfo = repoInfoGiteaApi1(repo);
|
||||
if (repoInfo.has_value()) {
|
||||
nlohmann::json::const_iterator itr = repoInfo.value().find("default_branch");
|
||||
if (itr != repoInfo.value().end() && itr->is_string()) {
|
||||
std::string branchName = *itr;
|
||||
info("Gitea v1 API seem to have worked");
|
||||
info("Default branch is " + branchName);
|
||||
|
||||
std::optional<std::filesystem::path> path = downloadRepoGiteaApi1(repo, branchName);
|
||||
}
|
||||
}
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
std::optional<std::filesystem::path> Download::downloadRepoGiteaApi1(const RepoInfo& repo, const std::string& branch) {
|
||||
|
||||
std::string fileName = branch + ".tar.gz";
|
||||
std::string url = repo.origin() + "/api/v1/repos/" + repo.project() + "/archive/" + fileName;
|
||||
std::filesystem::path path = destination/downloads/fileName;
|
||||
CURLcode code = httpDownload(url, path);
|
||||
if (code == CURLE_OK)
|
||||
return path;
|
||||
|
||||
warn("Couldn't download file " + url + ": " + curl_easy_strerror(code));
|
||||
minor("Removing " + path.string());
|
||||
if (std::filesystem::exists(path))
|
||||
std::filesystem::remove_all(path);
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
|
||||
CURLcode Download::httpGet(
|
||||
const std::string& url,
|
||||
std::string& result,
|
||||
const std::vector<std::string_view>& headers)
|
||||
{
|
||||
setHeaders(headers);
|
||||
|
||||
curl_easy_setopt(curl.get(), CURLOPT_URL, url.c_str());
|
||||
curl_easy_setopt(curl.get(), CURLOPT_WRITEFUNCTION, writeString);
|
||||
curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, &result);
|
||||
|
||||
return curl_easy_perform(curl.get());
|
||||
}
|
||||
|
||||
CURLcode Download::httpDownload(const std::string& url, const std::filesystem::path& path, const std::vector<std::string_view>& headers) {
|
||||
std::filesystem::create_directories(path.parent_path());
|
||||
if (std::filesystem::exists(path)) {
|
||||
minor("File " + path.string() + " already exists, will be overwritten");
|
||||
|
||||
if (std::filesystem::is_directory(path))
|
||||
std::filesystem::remove_all(path);
|
||||
}
|
||||
|
||||
std::unique_ptr<FILE, FileDeleter> filePtr(fopen(path.c_str(), "wb"));
|
||||
if (!filePtr)
|
||||
throw FileError("Could not open file " + path.string() + " to write");
|
||||
|
||||
setHeaders(headers);
|
||||
curl_easy_setopt(curl.get(), CURLOPT_URL, url.c_str());
|
||||
curl_easy_setopt(curl.get(), CURLOPT_WRITEFUNCTION, writeFile);
|
||||
curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, filePtr.get());
|
||||
|
||||
return curl_easy_perform(curl.get());
|
||||
|
||||
// return res;
|
||||
// if (res != CURLE_OK) {
|
||||
// Project::warn("Couldn't download file " + url + ": " + curl_easy_strerror(res));
|
||||
// } else {
|
||||
// uint32_t code;
|
||||
// curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code);
|
||||
// if (code == 200)
|
||||
// result = true;
|
||||
// else
|
||||
// Project::warn("Couldn't download file " + url + ": response code " + std::to_string(code));
|
||||
// }
|
||||
// } else {
|
||||
// Project::error(std::string("Couldn't open output file ") + destination.c_str());
|
||||
// }
|
||||
}
|
||||
|
||||
void Download::setHeaders(const std::vector<std::string_view>& headers) {
|
||||
struct curl_slist* curlHeaders = nullptr;
|
||||
for (const std::string_view& header : headers)
|
||||
curlHeaders = curl_slist_append(curlHeaders, header.data());
|
||||
|
||||
curl_easy_setopt(curl.get(), CURLOPT_HTTPHEADER, curlHeaders);
|
||||
}
|
||||
|
||||
void Download::createCurl() {
|
||||
CURL* crl = curl_easy_init();
|
||||
if (!crl)
|
||||
throw CurlError("Error creating curl instalce");
|
||||
|
||||
curl_easy_setopt(crl, CURLOPT_USERAGENT, "libcurl-agent/1.0" );
|
||||
curl_easy_setopt(crl, CURLOPT_VERBOSE, 1L);
|
||||
curl_easy_setopt(crl, CURLOPT_DEBUGFUNCTION, trace);
|
||||
curl_easy_setopt(crl, CURLOPT_DEBUGDATA, this);
|
||||
curl_easy_setopt(crl, CURLOPT_FOLLOWLOCATION, 1L);
|
||||
curl_easy_setopt(crl, CURLOPT_NOPROGRESS, 1L);
|
||||
curl = std::unique_ptr<CURL, CurlDeleter>(crl);
|
||||
}
|
||||
|
||||
|
||||
std::optional<Download::RepoInfo> Download::testRepo() const {
|
||||
std::smatch results;
|
||||
if (std::regex_search(url, results, repo)) {
|
||||
info(url + " appears to be a git repository");
|
||||
return RepoInfo {
|
||||
results[1],
|
||||
results[2],
|
||||
results[3],
|
||||
results[4]
|
||||
};
|
||||
}
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
Download::CurlError::CurlError(const std::string& message) :
|
||||
std::runtime_error(message) {}
|
||||
|
||||
Download::FileError::FileError(const std::string& message) :
|
||||
std::runtime_error(message) {}
|
||||
|
||||
std::string Download::RepoInfo::origin() const {
|
||||
return protocol + "://" + host;
|
||||
}
|
||||
|
||||
std::string Download::RepoInfo::project() const {
|
||||
return owner + "/" + name;
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,86 @@
|
|||
#pragma once
|
||||
|
||||
#include <mutex>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <string_view>
|
||||
#include <filesystem>
|
||||
#include <optional>
|
||||
#include <exception>
|
||||
|
||||
#include <curl/curl.h>
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
#include "atomicmutex.h"
|
||||
#include "loggable.h"
|
||||
|
||||
class Collection;
|
||||
|
||||
class Download : protected Loggable {
|
||||
struct RepoInfo;
|
||||
struct CurlDeleter {
|
||||
void operator() (CURL* curl) const;
|
||||
};
|
||||
struct FileDeleter {
|
||||
void operator() (FILE* file) const;
|
||||
};
|
||||
class CurlError;
|
||||
class FileError;
|
||||
|
||||
public:
|
||||
explicit Download(
|
||||
const std::string& url,
|
||||
const std::filesystem::path& destination,
|
||||
const std::shared_ptr<Collection>& collection,
|
||||
const std::shared_ptr<Logger>& logger
|
||||
);
|
||||
~Download();
|
||||
|
||||
void proceed();
|
||||
|
||||
private:
|
||||
std::optional<RepoInfo> testRepo() const;
|
||||
|
||||
std::optional<std::filesystem::path> downloadAsRepo(const RepoInfo& repo);
|
||||
|
||||
std::optional<nlohmann::json> repoInfoGiteaApi1(const RepoInfo& repo);
|
||||
std::optional<std::filesystem::path> downloadRepoGiteaApi1(const RepoInfo& repo, const std::string& branch);
|
||||
|
||||
CURLcode httpGet(const std::string& url, std::string& result, const std::vector<std::string_view>& headers = {});
|
||||
CURLcode httpDownload(const std::string& url, const std::filesystem::path& path, const std::vector<std::string_view>& headers = {});
|
||||
void setHeaders (const std::vector<std::string_view>& headers);
|
||||
|
||||
static size_t writeString(void* data, size_t size, size_t nmemb, void* mem);
|
||||
static size_t writeFile(void* data, size_t size, size_t nmemb, void* file);
|
||||
static int trace(CURL *handle, curl_infotype type, char *data, size_t size, void *clientp);
|
||||
void createCurl();
|
||||
|
||||
private:
|
||||
static AtomicMutex amx;
|
||||
static unsigned int instances;
|
||||
std::unique_ptr<CURL, CurlDeleter> curl;
|
||||
std::shared_ptr<Collection> collection;
|
||||
std::string url;
|
||||
std::filesystem::path destination;
|
||||
};
|
||||
|
||||
struct Download::RepoInfo {
|
||||
std::string protocol;
|
||||
std::string host;
|
||||
std::string owner;
|
||||
std::string name;
|
||||
|
||||
std::string origin() const;
|
||||
std::string project() const;
|
||||
};
|
||||
|
||||
class Download::CurlError : public std::runtime_error {
|
||||
public:
|
||||
explicit CurlError(const std::string& message);
|
||||
};
|
||||
|
||||
class Download::FileError : public std::runtime_error {
|
||||
public:
|
||||
explicit FileError(const std::string& message);
|
||||
};
|
||||
Loading…
Reference in New Issue