some more thoughts of how to organize it better

src2/CMakeLists.txt

@@ -3,6 +3,8 @@ set(SOURCES
     collection.cpp
     loggable.cpp
     taskmanager.cpp
+    download.cpp
+    atomicmutex.cpp
 )
 
 set(HEADERS
@@ -10,6 +12,8 @@ set(HEADERS
     collection.h
     loggable.h
     taskmanager.h
+    download.h
+    atomicmutex.h
 )
 
 target_sources(${PROJECT_NAME} PRIVATE ${SOURCES})

src2/atomicmutex.cpp

@@ -0,0 +1,13 @@
+#include "atomicmutex.h"
+
+AtomicMutex::AtomicMutex():
+    flag()
+{}
+
+void AtomicMutex::lock() noexcept {
+    while (flag.test_and_set());
+}
+
+void AtomicMutex::unlock() noexcept {
+    flag.clear();
+}

src2/atomicmutex.h

@@ -0,0 +1,13 @@
+#pragma once
+
+#include <atomic>
+
+class AtomicMutex {
+public:
+    explicit AtomicMutex();
+    void lock() noexcept;
+    void unlock() noexcept;
+
+private:
+    std::atomic_flag flag;
+};

src2/collection.h

@@ -6,13 +6,20 @@
 #include <set>
 #include <exception>
 
-class Collection {
+#include "loggable.h"
+#include "component.h"
+#include "taskmanager.h"
+
+class Collection : protected Loggable {
 public:
     class UnknownSource;
     class DuplicateSource;
     class DuplicatePath;
 
-    Collection();
+    Collection(
+        const std::shared_ptr<Logger>& logger,
+        const std::shared_ptr<TaskManager>& taskManager
+    );
 
     unsigned int sourcesTotal() const;
     unsigned int sourcesPending() const;
@@ -21,13 +28,21 @@ public:
 
     bool hasSource(const std::string& source) const;
     void addSource(const std::string& source);
-    void sourceError(const std::string& source);
-    void sourceSuccess(const std::string& source, const std::filesystem::path& path);
 
 private:
+    void sourceDownloadError(const std::string& source);
+    void sourceBuildError(const std::string& source);
+    void sourceDownaloded(const std::string& source);
+    void sourceBuilt(const std::string& source);
+
+private:
+    std::shared_ptr<TaskManager> taskManager;
     std::set<std::string> errorSources;
     std::set<std::string> pendingSources;
-    std::map<std::string, std::filesystem::path> successSources;    //would be nice to have a bimap here
+    std::map<std::string, std::unique_ptr<Component>> failedSources;
+    std::map<std::string, std::unique_ptr<Component>> downloadedSources;
+    std::map<std::string, std::unique_ptr<Component>> buildingSources;
+    std::map<std::string, std::unique_ptr<Component>> successSources;
 };
 
 class Collection::UnknownSource : public std::runtime_error {

src2/component.cpp

@@ -3,23 +3,24 @@
 #include <array>
 #include <string_view>
 
+#include "collection.h"
+
 constexpr std::array<std::string_view, Component::done + 1> stringStates {
     "initial",
     "reading",
     "ready",
     "building",
+    "error",
     "done"
 };
 
 Component::Component(
     const std::filesystem::path& path,
     const std::shared_ptr<Collection>& collection,
-    const std::shared_ptr<TaskManager>& taskManager,
     const std::shared_ptr<Logger>& logger
 ):
     Loggable(logger),
     collection(collection),
-    taskManager(taskManager),
     location(path)
 {}
 
@@ -27,14 +28,13 @@ void Component::read() {
     if (state != initial)
         throw WrongState(state, "read");
 
-    try {
-        collection->addSource(location);
-    } catch (const Collection::DuplicateSource e) {
-        state = error;
-        throw e;
-    }
+    // try {
+    //     collection->addSource(location);
+    // } catch (const Collection::DuplicateSource e) {
+    //     state = error;
+    //     throw e;
+    // }
     state = reading;
-    taskManager->queue(std::bind(&Component::performRead, this));
 }
 
 void Component::build(const std::filesystem::path& destination, const std::string& target) {
@@ -42,13 +42,6 @@ void Component::build(const std::filesystem::path& destination, const std::strin
         throw WrongState(state, "build");
 
     state = building;
-    taskManager->queue(std::bind(&Component::performBuild, this, destination, target));
-}
-
-void Component::performRead() {
-}
-
-void Component::performBuild(std::filesystem::path destination, std::string target) {
 }
 
 Component::State Component::getState() const {

src2/component.h

@@ -8,8 +8,8 @@
 
 #include "loggable.h"
 #include "loggger.h"
-#include "collection.h"
-#include "taskmanager.h"
+
+class Collection;
 
 class Component : protected Loggable {
 public:
@@ -33,7 +33,6 @@ public:
     Component(
         const std::filesystem::path& path,
         const std::shared_ptr<Collection>& collection,
-        const std::shared_ptr<TaskManager>& taskManager,
         const std::shared_ptr<Logger>& logger
     );
 
@@ -43,15 +42,10 @@ public:
     void read();
     void build(const std::filesystem::path& destination, const std::string& target);
 
-private:
-    void performRead();
-    void performBuild(std::filesystem::path destination, std::string target);
-
 private:
     State state;
     Type type;
     std::shared_ptr<Collection> collection;
-    std::shared_ptr<TaskManager> taskManager;
     std::filesystem::path location;
 };
 

src2/download.cpp

@@ -0,0 +1,216 @@
+#include "download.h"
+
+#include <regex>
+
+#include "collection.h"
+
+constexpr std::string_view downloads("downloads");
+constexpr std::string_view headerAcceptJson("accept: application/json");
+
+static const std::regex repo("(^https?):\\/\\/([\\w\\d\\.\\-\\_]+)\\/([\\w\\d\\-\\_]+)\\/([\\w\\d\\-\\_]+)");
+
+unsigned int Download::instances(0);
+AtomicMutex Download::amx{};
+
+void Download::CurlDeleter::operator()(CURL* curl) const {
+    curl_easy_cleanup(curl);
+}
+
+void Download::FileDeleter::operator()(FILE* file) const {
+    fclose(file);
+}
+
+
+Download::Download(
+    const std::string& url,
+    const std::filesystem::path& destination,
+    const std::shared_ptr<Collection>& collection,
+    const std::shared_ptr<Logger>& logger
+) :
+    Loggable(logger),
+    curl(),
+    collection(collection),
+    url(url),
+    destination(destination)
+{
+    std::lock_guard lock(amx);
+    if (instances == 0) {
+        CURLcode res = curl_global_init(CURL_GLOBAL_ALL);
+        if (res != CURLE_OK)
+            throw CurlError(std::string("Error initializing curl global ") + curl_easy_strerror(res));
+    }
+
+    ++instances;
+
+    createCurl();
+}
+
+Download::~Download() {
+    std::lock_guard lock(amx);
+    --instances;
+    if (instances == 0)
+        curl_global_cleanup();
+}
+
+
+void Download::proceed() {
+    bool success = false;
+    std::optional<RepoInfo> repo = testRepo();
+    if (repo.has_value()) {
+        std::optional<std::filesystem::path> path = downloadAsRepo(repo.value());
+    }
+}
+
+std::optional<nlohmann::json> Download::repoInfoGiteaApi1(const RepoInfo& repo)  {
+    std::string url = repo.origin() + "/api/v1/repos/" + repo.project();
+    std::string data;
+    CURLcode code = httpGet(url, data, {headerAcceptJson});
+    if (code == CURLE_OK) {
+        try {
+            return nlohmann::json::parse(data);
+        } catch (const nlohmann::json::exception& e) {
+            warn(e.what());
+        }
+    }
+
+    return std::nullopt;
+}
+
+std::optional<std::filesystem::path> Download::downloadAsRepo(const RepoInfo& repo) {
+    info("Trying Gitea v1 API");
+    std::optional<nlohmann::json> repoInfo = repoInfoGiteaApi1(repo);
+    if (repoInfo.has_value()) {
+        nlohmann::json::const_iterator itr = repoInfo.value().find("default_branch");
+        if (itr != repoInfo.value().end() && itr->is_string()) {
+            std::string branchName = *itr;
+            info("Gitea v1 API seem to have worked");
+            info("Default branch is " + branchName);
+
+            std::optional<std::filesystem::path> path = downloadRepoGiteaApi1(repo, branchName);
+        }
+    }
+
+    return std::nullopt;
+}
+
+std::optional<std::filesystem::path> Download::downloadRepoGiteaApi1(const RepoInfo& repo, const std::string& branch) {
+
+    std::string fileName = branch + ".tar.gz";
+    std::string url = repo.origin() + "/api/v1/repos/" + repo.project() + "/archive/" + fileName;
+    std::filesystem::path path = destination/downloads/fileName;
+    CURLcode code = httpDownload(url, path);
+    if (code == CURLE_OK)
+        return path;
+
+    warn("Couldn't download file " + url + ": " + curl_easy_strerror(code));
+    minor("Removing " + path.string());
+    if (std::filesystem::exists(path))
+        std::filesystem::remove_all(path);
+
+    return std::nullopt;
+}
+
+
+CURLcode Download::httpGet(
+    const std::string& url,
+    std::string& result,
+    const std::vector<std::string_view>& headers)
+{
+    setHeaders(headers);
+
+    curl_easy_setopt(curl.get(), CURLOPT_URL, url.c_str());
+    curl_easy_setopt(curl.get(), CURLOPT_WRITEFUNCTION, writeString);
+    curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, &result);
+
+    return curl_easy_perform(curl.get());
+}
+
+CURLcode Download::httpDownload(const std::string& url, const std::filesystem::path& path, const std::vector<std::string_view>& headers) {
+    std::filesystem::create_directories(path.parent_path());
+    if (std::filesystem::exists(path)) {
+        minor("File " + path.string() + " already exists, will be overwritten");
+
+        if (std::filesystem::is_directory(path))
+            std::filesystem::remove_all(path);
+    }
+
+    std::unique_ptr<FILE, FileDeleter> filePtr(fopen(path.c_str(), "wb"));
+    if (!filePtr)
+        throw FileError("Could not open file " + path.string() + " to write");
+
+    setHeaders(headers);
+    curl_easy_setopt(curl.get(), CURLOPT_URL, url.c_str());
+    curl_easy_setopt(curl.get(), CURLOPT_WRITEFUNCTION, writeFile);
+    curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, filePtr.get());
+
+    return curl_easy_perform(curl.get());
+
+    //     return res;
+    //     if (res != CURLE_OK) {
+    //         Project::warn("Couldn't download file " + url + ": " + curl_easy_strerror(res));
+    //     } else {
+    //         uint32_t code;
+    //         curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code);
+    //         if (code == 200)
+    //             result = true;
+    //         else
+    //             Project::warn("Couldn't download file " + url + ": response code " + std::to_string(code));
+    //     }
+    // } else {
+    //     Project::error(std::string("Couldn't open output file ") + destination.c_str());
+    // }
+}
+
+void Download::setHeaders(const std::vector<std::string_view>& headers) {
+    struct curl_slist* curlHeaders = nullptr;
+    for (const std::string_view& header : headers)
+        curlHeaders = curl_slist_append(curlHeaders, header.data());
+
+    curl_easy_setopt(curl.get(), CURLOPT_HTTPHEADER, curlHeaders);
+}
+
+void Download::createCurl() {
+    CURL* crl = curl_easy_init();
+    if (!crl)
+        throw CurlError("Error creating curl instalce");
+
+	curl_easy_setopt(crl, CURLOPT_USERAGENT, "libcurl-agent/1.0" );
+    curl_easy_setopt(crl, CURLOPT_VERBOSE, 1L);
+    curl_easy_setopt(crl, CURLOPT_DEBUGFUNCTION, trace);
+    curl_easy_setopt(crl, CURLOPT_DEBUGDATA, this);
+    curl_easy_setopt(crl, CURLOPT_FOLLOWLOCATION, 1L);
+    curl_easy_setopt(crl, CURLOPT_NOPROGRESS, 1L);
+    curl = std::unique_ptr<CURL, CurlDeleter>(crl);
+}
+
+
+std::optional<Download::RepoInfo> Download::testRepo() const {
+    std::smatch results;
+    if (std::regex_search(url, results, repo)) {
+        info(url + " appears to be a git repository");
+        return RepoInfo {
+            results[1],
+            results[2],
+            results[3],
+            results[4]
+        };
+    }
+
+    return std::nullopt;
+}
+
+Download::CurlError::CurlError(const std::string& message) :
+    std::runtime_error(message) {}
+
+Download::FileError::FileError(const std::string& message) :
+    std::runtime_error(message) {}
+
+std::string Download::RepoInfo::origin() const {
+    return protocol + "://" + host;
+}
+
+std::string Download::RepoInfo::project() const {
+    return owner + "/" + name;
+}
+
+

src2/download.h

@@ -0,0 +1,86 @@
+#pragma once
+
+#include <mutex>
+#include <memory>
+#include <string>
+#include <vector>
+#include <string_view>
+#include <filesystem>
+#include <optional>
+#include <exception>
+
+#include <curl/curl.h>
+#include <nlohmann/json.hpp>
+
+#include "atomicmutex.h"
+#include "loggable.h"
+
+class Collection;
+
+class Download : protected Loggable {
+    struct RepoInfo;
+    struct CurlDeleter {
+        void operator() (CURL* curl) const;
+    };
+    struct FileDeleter {
+        void operator() (FILE* file) const;
+    };
+    class CurlError;
+    class FileError;
+
+public:
+    explicit Download(
+        const std::string& url,
+        const std::filesystem::path& destination,
+        const std::shared_ptr<Collection>& collection,
+        const std::shared_ptr<Logger>& logger
+    );
+    ~Download();
+
+    void proceed();
+
+private:
+    std::optional<RepoInfo> testRepo() const;
+
+    std::optional<std::filesystem::path> downloadAsRepo(const RepoInfo& repo);
+
+    std::optional<nlohmann::json> repoInfoGiteaApi1(const RepoInfo& repo);
+    std::optional<std::filesystem::path> downloadRepoGiteaApi1(const RepoInfo& repo, const std::string& branch);
+
+    CURLcode httpGet(const std::string& url, std::string& result, const std::vector<std::string_view>& headers = {});
+    CURLcode httpDownload(const std::string& url, const std::filesystem::path& path, const std::vector<std::string_view>& headers = {});
+    void setHeaders (const std::vector<std::string_view>& headers);
+
+    static size_t writeString(void* data, size_t size, size_t nmemb, void* mem);
+    static size_t writeFile(void* data, size_t size, size_t nmemb, void* file);
+    static int trace(CURL *handle, curl_infotype type, char *data, size_t size, void *clientp);
+    void createCurl();
+
+private:
+    static AtomicMutex amx;
+    static unsigned int instances;
+    std::unique_ptr<CURL, CurlDeleter> curl;
+    std::shared_ptr<Collection> collection;
+    std::string url;
+    std::filesystem::path destination;
+};
+
+struct Download::RepoInfo {
+    std::string protocol;
+    std::string host;
+    std::string owner;
+    std::string name;
+
+    std::string origin() const;
+    std::string project() const;
+};
+
+class Download::CurlError : public std::runtime_error {
+public:
+    explicit CurlError(const std::string& message);
+};
+
+class Download::FileError : public std::runtime_error {
+public:
+    explicit FileError(const std::string& message);
+};