From 7b9112c0dd4e5e5857704bfd5578c1a931cf3f80 Mon Sep 17 00:00:00 2001 From: blue Date: Sat, 23 Sep 2023 13:10:24 -0300 Subject: [PATCH] some more thoughts of how to organize it better --- src2/CMakeLists.txt | 4 + src2/atomicmutex.cpp | 13 +++ src2/atomicmutex.h | 13 +++ src2/collection.h | 25 ++++- src2/component.cpp | 25 ++--- src2/component.h | 10 +- src2/download.cpp | 216 +++++++++++++++++++++++++++++++++++++++++++ src2/download.h | 86 +++++++++++++++++ 8 files changed, 363 insertions(+), 29 deletions(-) create mode 100644 src2/atomicmutex.cpp create mode 100644 src2/atomicmutex.h create mode 100644 src2/download.cpp create mode 100644 src2/download.h diff --git a/src2/CMakeLists.txt b/src2/CMakeLists.txt index ce42eff..e363156 100644 --- a/src2/CMakeLists.txt +++ b/src2/CMakeLists.txt @@ -3,6 +3,8 @@ set(SOURCES collection.cpp loggable.cpp taskmanager.cpp + download.cpp + atomicmutex.cpp ) set(HEADERS @@ -10,6 +12,8 @@ set(HEADERS collection.h loggable.h taskmanager.h + download.h + atomicmutex.h ) target_sources(${PROJECT_NAME} PRIVATE ${SOURCES}) diff --git a/src2/atomicmutex.cpp b/src2/atomicmutex.cpp new file mode 100644 index 0000000..34e9feb --- /dev/null +++ b/src2/atomicmutex.cpp @@ -0,0 +1,13 @@ +#include "atomicmutex.h" + +AtomicMutex::AtomicMutex(): + flag() +{} + +void AtomicMutex::lock() noexcept { + while (flag.test_and_set()); +} + +void AtomicMutex::unlock() noexcept { + flag.clear(); +} diff --git a/src2/atomicmutex.h b/src2/atomicmutex.h new file mode 100644 index 0000000..93b133b --- /dev/null +++ b/src2/atomicmutex.h @@ -0,0 +1,13 @@ +#pragma once + +#include + +class AtomicMutex { +public: + explicit AtomicMutex(); + void lock() noexcept; + void unlock() noexcept; + +private: + std::atomic_flag flag; +}; diff --git a/src2/collection.h b/src2/collection.h index c35663b..8655074 100644 --- a/src2/collection.h +++ b/src2/collection.h @@ -6,13 +6,20 @@ #include #include -class Collection { +#include "loggable.h" +#include "component.h" +#include "taskmanager.h" + +class Collection : protected Loggable { public: class UnknownSource; class DuplicateSource; class DuplicatePath; - Collection(); + Collection( + const std::shared_ptr& logger, + const std::shared_ptr& taskManager + ); unsigned int sourcesTotal() const; unsigned int sourcesPending() const; @@ -21,13 +28,21 @@ public: bool hasSource(const std::string& source) const; void addSource(const std::string& source); - void sourceError(const std::string& source); - void sourceSuccess(const std::string& source, const std::filesystem::path& path); private: + void sourceDownloadError(const std::string& source); + void sourceBuildError(const std::string& source); + void sourceDownaloded(const std::string& source); + void sourceBuilt(const std::string& source); + +private: + std::shared_ptr taskManager; std::set errorSources; std::set pendingSources; - std::map successSources; //would be nice to have a bimap here + std::map> failedSources; + std::map> downloadedSources; + std::map> buildingSources; + std::map> successSources; }; class Collection::UnknownSource : public std::runtime_error { diff --git a/src2/component.cpp b/src2/component.cpp index edecb21..a4c7c6b 100644 --- a/src2/component.cpp +++ b/src2/component.cpp @@ -3,23 +3,24 @@ #include #include +#include "collection.h" + constexpr std::array stringStates { "initial", "reading", "ready", "building", + "error", "done" }; Component::Component( const std::filesystem::path& path, const std::shared_ptr& collection, - const std::shared_ptr& taskManager, const std::shared_ptr& logger ): Loggable(logger), collection(collection), - taskManager(taskManager), location(path) {} @@ -27,14 +28,13 @@ void Component::read() { if (state != initial) throw WrongState(state, "read"); - try { - collection->addSource(location); - } catch (const Collection::DuplicateSource e) { - state = error; - throw e; - } + // try { + // collection->addSource(location); + // } catch (const Collection::DuplicateSource e) { + // state = error; + // throw e; + // } state = reading; - taskManager->queue(std::bind(&Component::performRead, this)); } void Component::build(const std::filesystem::path& destination, const std::string& target) { @@ -42,13 +42,6 @@ void Component::build(const std::filesystem::path& destination, const std::strin throw WrongState(state, "build"); state = building; - taskManager->queue(std::bind(&Component::performBuild, this, destination, target)); -} - -void Component::performRead() { -} - -void Component::performBuild(std::filesystem::path destination, std::string target) { } Component::State Component::getState() const { diff --git a/src2/component.h b/src2/component.h index 9fa12de..0b7daf6 100644 --- a/src2/component.h +++ b/src2/component.h @@ -8,8 +8,8 @@ #include "loggable.h" #include "loggger.h" -#include "collection.h" -#include "taskmanager.h" + +class Collection; class Component : protected Loggable { public: @@ -33,7 +33,6 @@ public: Component( const std::filesystem::path& path, const std::shared_ptr& collection, - const std::shared_ptr& taskManager, const std::shared_ptr& logger ); @@ -43,15 +42,10 @@ public: void read(); void build(const std::filesystem::path& destination, const std::string& target); -private: - void performRead(); - void performBuild(std::filesystem::path destination, std::string target); - private: State state; Type type; std::shared_ptr collection; - std::shared_ptr taskManager; std::filesystem::path location; }; diff --git a/src2/download.cpp b/src2/download.cpp new file mode 100644 index 0000000..bdf3d4a --- /dev/null +++ b/src2/download.cpp @@ -0,0 +1,216 @@ +#include "download.h" + +#include + +#include "collection.h" + +constexpr std::string_view downloads("downloads"); +constexpr std::string_view headerAcceptJson("accept: application/json"); + +static const std::regex repo("(^https?):\\/\\/([\\w\\d\\.\\-\\_]+)\\/([\\w\\d\\-\\_]+)\\/([\\w\\d\\-\\_]+)"); + +unsigned int Download::instances(0); +AtomicMutex Download::amx{}; + +void Download::CurlDeleter::operator()(CURL* curl) const { + curl_easy_cleanup(curl); +} + +void Download::FileDeleter::operator()(FILE* file) const { + fclose(file); +} + + +Download::Download( + const std::string& url, + const std::filesystem::path& destination, + const std::shared_ptr& collection, + const std::shared_ptr& logger +) : + Loggable(logger), + curl(), + collection(collection), + url(url), + destination(destination) +{ + std::lock_guard lock(amx); + if (instances == 0) { + CURLcode res = curl_global_init(CURL_GLOBAL_ALL); + if (res != CURLE_OK) + throw CurlError(std::string("Error initializing curl global ") + curl_easy_strerror(res)); + } + + ++instances; + + createCurl(); +} + +Download::~Download() { + std::lock_guard lock(amx); + --instances; + if (instances == 0) + curl_global_cleanup(); +} + + +void Download::proceed() { + bool success = false; + std::optional repo = testRepo(); + if (repo.has_value()) { + std::optional path = downloadAsRepo(repo.value()); + } +} + +std::optional Download::repoInfoGiteaApi1(const RepoInfo& repo) { + std::string url = repo.origin() + "/api/v1/repos/" + repo.project(); + std::string data; + CURLcode code = httpGet(url, data, {headerAcceptJson}); + if (code == CURLE_OK) { + try { + return nlohmann::json::parse(data); + } catch (const nlohmann::json::exception& e) { + warn(e.what()); + } + } + + return std::nullopt; +} + +std::optional Download::downloadAsRepo(const RepoInfo& repo) { + info("Trying Gitea v1 API"); + std::optional repoInfo = repoInfoGiteaApi1(repo); + if (repoInfo.has_value()) { + nlohmann::json::const_iterator itr = repoInfo.value().find("default_branch"); + if (itr != repoInfo.value().end() && itr->is_string()) { + std::string branchName = *itr; + info("Gitea v1 API seem to have worked"); + info("Default branch is " + branchName); + + std::optional path = downloadRepoGiteaApi1(repo, branchName); + } + } + + return std::nullopt; +} + +std::optional Download::downloadRepoGiteaApi1(const RepoInfo& repo, const std::string& branch) { + + std::string fileName = branch + ".tar.gz"; + std::string url = repo.origin() + "/api/v1/repos/" + repo.project() + "/archive/" + fileName; + std::filesystem::path path = destination/downloads/fileName; + CURLcode code = httpDownload(url, path); + if (code == CURLE_OK) + return path; + + warn("Couldn't download file " + url + ": " + curl_easy_strerror(code)); + minor("Removing " + path.string()); + if (std::filesystem::exists(path)) + std::filesystem::remove_all(path); + + return std::nullopt; +} + + +CURLcode Download::httpGet( + const std::string& url, + std::string& result, + const std::vector& headers) +{ + setHeaders(headers); + + curl_easy_setopt(curl.get(), CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl.get(), CURLOPT_WRITEFUNCTION, writeString); + curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, &result); + + return curl_easy_perform(curl.get()); +} + +CURLcode Download::httpDownload(const std::string& url, const std::filesystem::path& path, const std::vector& headers) { + std::filesystem::create_directories(path.parent_path()); + if (std::filesystem::exists(path)) { + minor("File " + path.string() + " already exists, will be overwritten"); + + if (std::filesystem::is_directory(path)) + std::filesystem::remove_all(path); + } + + std::unique_ptr filePtr(fopen(path.c_str(), "wb")); + if (!filePtr) + throw FileError("Could not open file " + path.string() + " to write"); + + setHeaders(headers); + curl_easy_setopt(curl.get(), CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl.get(), CURLOPT_WRITEFUNCTION, writeFile); + curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, filePtr.get()); + + return curl_easy_perform(curl.get()); + + // return res; + // if (res != CURLE_OK) { + // Project::warn("Couldn't download file " + url + ": " + curl_easy_strerror(res)); + // } else { + // uint32_t code; + // curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); + // if (code == 200) + // result = true; + // else + // Project::warn("Couldn't download file " + url + ": response code " + std::to_string(code)); + // } + // } else { + // Project::error(std::string("Couldn't open output file ") + destination.c_str()); + // } +} + +void Download::setHeaders(const std::vector& headers) { + struct curl_slist* curlHeaders = nullptr; + for (const std::string_view& header : headers) + curlHeaders = curl_slist_append(curlHeaders, header.data()); + + curl_easy_setopt(curl.get(), CURLOPT_HTTPHEADER, curlHeaders); +} + +void Download::createCurl() { + CURL* crl = curl_easy_init(); + if (!crl) + throw CurlError("Error creating curl instalce"); + + curl_easy_setopt(crl, CURLOPT_USERAGENT, "libcurl-agent/1.0" ); + curl_easy_setopt(crl, CURLOPT_VERBOSE, 1L); + curl_easy_setopt(crl, CURLOPT_DEBUGFUNCTION, trace); + curl_easy_setopt(crl, CURLOPT_DEBUGDATA, this); + curl_easy_setopt(crl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(crl, CURLOPT_NOPROGRESS, 1L); + curl = std::unique_ptr(crl); +} + + +std::optional Download::testRepo() const { + std::smatch results; + if (std::regex_search(url, results, repo)) { + info(url + " appears to be a git repository"); + return RepoInfo { + results[1], + results[2], + results[3], + results[4] + }; + } + + return std::nullopt; +} + +Download::CurlError::CurlError(const std::string& message) : + std::runtime_error(message) {} + +Download::FileError::FileError(const std::string& message) : + std::runtime_error(message) {} + +std::string Download::RepoInfo::origin() const { + return protocol + "://" + host; +} + +std::string Download::RepoInfo::project() const { + return owner + "/" + name; +} + + diff --git a/src2/download.h b/src2/download.h new file mode 100644 index 0000000..0fd0d0f --- /dev/null +++ b/src2/download.h @@ -0,0 +1,86 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "atomicmutex.h" +#include "loggable.h" + +class Collection; + +class Download : protected Loggable { + struct RepoInfo; + struct CurlDeleter { + void operator() (CURL* curl) const; + }; + struct FileDeleter { + void operator() (FILE* file) const; + }; + class CurlError; + class FileError; + +public: + explicit Download( + const std::string& url, + const std::filesystem::path& destination, + const std::shared_ptr& collection, + const std::shared_ptr& logger + ); + ~Download(); + + void proceed(); + +private: + std::optional testRepo() const; + + std::optional downloadAsRepo(const RepoInfo& repo); + + std::optional repoInfoGiteaApi1(const RepoInfo& repo); + std::optional downloadRepoGiteaApi1(const RepoInfo& repo, const std::string& branch); + + CURLcode httpGet(const std::string& url, std::string& result, const std::vector& headers = {}); + CURLcode httpDownload(const std::string& url, const std::filesystem::path& path, const std::vector& headers = {}); + void setHeaders (const std::vector& headers); + + static size_t writeString(void* data, size_t size, size_t nmemb, void* mem); + static size_t writeFile(void* data, size_t size, size_t nmemb, void* file); + static int trace(CURL *handle, curl_infotype type, char *data, size_t size, void *clientp); + void createCurl(); + +private: + static AtomicMutex amx; + static unsigned int instances; + std::unique_ptr curl; + std::shared_ptr collection; + std::string url; + std::filesystem::path destination; +}; + +struct Download::RepoInfo { + std::string protocol; + std::string host; + std::string owner; + std::string name; + + std::string origin() const; + std::string project() const; +}; + +class Download::CurlError : public std::runtime_error { +public: + explicit CurlError(const std::string& message); +}; + +class Download::FileError : public std::runtime_error { +public: + explicit FileError(const std::string& message); +};