managed to download a tarball

This commit is contained in:
Blue 2023-09-02 21:53:36 -03:00
parent 7ac697f111
commit 76bd9a0497
Signed by: blue
GPG Key ID: 9B203B252A63EE38
6 changed files with 234 additions and 63 deletions

View File

@ -4,7 +4,9 @@
#include "project.h" #include "project.h"
const std::regex http("^https?://"); constexpr std::string_view acceptJson("accept: application/json");
const std::regex http("^https?:\\/\\/");
const std::regex repo("(^https?):\\/\\/([\\w\\d\\.\\-\\_]+)\\/([\\w\\d\\-\\_]+)\\/([\\w\\d\\-\\_]+)");
Dependency::Dependency( Dependency::Dependency(
const std::string& path, const std::string& path,
@ -31,11 +33,15 @@ std::optional<std::string> Dependency::getVersion() const {
} }
bool Dependency::prepare(const std::filesystem::path& source, const std::filesystem::path& destination) { bool Dependency::prepare(const std::filesystem::path& source, const std::filesystem::path& destination) {
if (std::regex_search(path, http)) { std::smatch results;
Project::info("downloading project from " + path); if (std::regex_search(path, results, repo)) {
std::filesystem::create_directories(destination / "downloads"); Project::info(path + " appears to be a git repository");
const std::string& protocol = results[1];
const std::string& host = results[2];
const std::string& owner = results[3];
const std::string& repo = results[4];
return download(destination); return downloadRepo(destination, protocol, host, owner, repo);
} else { } else {
Project::info("checking project at path " + path); Project::info("checking project at path " + path);
std::filesystem::directory_entry srcDir(source / path); std::filesystem::directory_entry srcDir(source / path);
@ -48,50 +54,160 @@ bool Dependency::prepare(const std::filesystem::path& source, const std::filesys
return true; return true;
} }
bool Dependency::download(const std::filesystem::path& destination) { bool Dependency::downloadRepo(
std::filesystem::path dwn = destination / "downloads"; const std::filesystem::path& destination,
bool result = std::filesystem::create_directories(dwn); const std::string& protocol,
// if (!result) { const std::string& host,
// Project::error("couldn't create " + dwn.string()); const std::string& owner,
// return result; const std::string& repo
// } ) {
nlohmann::json info;
Project::info("Trying Gitea v1 API");
bool res = repoGiteaApi1(protocol, host, owner, repo, info);
CURL *curl_handle; if (res) {
std::filesystem::path out = dwn / ("out" + std::to_string(std::rand())); nlohmann::json::const_iterator itr = info.find("default_branch");
FILE *pagefile; if (itr != info.end() && itr->is_string()) {
std::string branchName = *itr;
Project::info("Gitea v1 API seem to have worked");
Project::info("Default branch is " + branchName);
curl_global_init(CURL_GLOBAL_ALL); std::string fileName = branchName + ".tar.gz";
/* init the curl session */ std::string url = protocol + "://" + host + "/api/v1/repos/" + owner + "/" + repo + "/archive/" + fileName;
curl_handle = curl_easy_init(); res = download(url, destination / "downloads" / fileName);
/* set URL to get here */ if (res)
curl_easy_setopt(curl_handle, CURLOPT_URL, path.c_str()); Project::info("Successfully downloaded " + fileName);
/* disable progress meter, set to 0L to enable it */
curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, 1L);
/* send all data to this function */
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, write);
/* open the file */
pagefile = fopen(out.c_str(), "wb");
if (pagefile) {
/* write the page body to this file handle */
curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, pagefile);
/* get it! */
curl_easy_perform(curl_handle);
/* close the header file */
fclose(pagefile);
} else { } else {
Project::error(std::string("couldn't open output file ") + out.c_str()); res = false;
}
} }
/* cleanup curl stuff */ if (!res) {
curl_easy_cleanup(curl_handle); Project::warn("Gitea v1 API didn't work");
curl_global_cleanup(); }
return true; return res;
}
bool Dependency::repoGiteaApi1(
const std::string& protocol,
const std::string& host,
const std::string& owner,
const std::string& repo,
nlohmann::json& json
) {
bool result = false;
std::string url = protocol + "://" + host + "/api/v1/repos/" + owner + "/" + repo;
std::string data;
CURLcode code = httpGet(url, data, {acceptJson});
if (code == CURLE_OK) {
try {
json = nlohmann::json::parse(data);
result = true;
} catch (const nlohmann::json::exception& e) {
Project::warn(e.what());
}
}
return result;
}
CURLcode Dependency::httpGet(const std::string& url, std::string& result, const std::list<std::string_view>& headers) {
CURL* curl = curl_easy_init();
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L);
struct curl_slist* curlHeaders = nullptr;
for (const std::string_view& header : headers)
curlHeaders = curl_slist_append(curlHeaders, header.data());
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curlHeaders);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeString);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &result);
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, trace);
curl_easy_setopt(curl, CURLOPT_DEBUGDATA, this);
CURLcode res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
return res;
} }
size_t Dependency::write(void* file, size_t size, size_t nmemb, void* stream) { bool Dependency::download(const std::string& url, const std::filesystem::path& destination) {
size_t written = fwrite(file, size, nmemb, (FILE *)stream); std::filesystem::create_directories(destination.parent_path());
if (std::filesystem::exists(destination))
Project::minor("File " + destination.string() + " already exists, will be overwritten");
bool result = false;
CURL* curl = curl_easy_init();
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeFile);
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, trace);
curl_easy_setopt(curl, CURLOPT_DEBUGDATA, this);
FILE* pagefile = fopen(destination.c_str(), "wb");
if (pagefile) {
curl_easy_setopt(curl, CURLOPT_WRITEDATA, pagefile);
CURLcode res = curl_easy_perform(curl);
fclose(pagefile);
if (res != CURLE_OK) {
Project::warn("Couldn't download file " + url + ": " + curl_easy_strerror(res));
} else {
uint32_t code;
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code);
if (code == 200)
result = true;
else
Project::warn("Couldn't download file " + url + ": response code " + std::to_string(code));
}
} else {
Project::error(std::string("couldn't open output file ") + destination.c_str());
}
curl_easy_cleanup(curl);
if (!result) {
Project::minor("Removing " + destination.string() + " since the donwload failed");
std::filesystem::remove(destination);
}
return result;
}
size_t Dependency::writeString(void* data, size_t size, size_t nmemb, void* pointer) {
size_t finalSize = size * nmemb;
std::string* string = static_cast<std::string*>(pointer);
string->append(static_cast<char*>(data), finalSize);
return finalSize;
}
size_t Dependency::writeFile(void* data, size_t size, size_t nmemb, void* file) {
size_t written = fwrite(data, size, nmemb, (FILE *)file);
return written; return written;
} }
int Dependency::trace(CURL* handle, curl_infotype type, char* data, size_t size, void* clientp) {
switch (type) {
case CURLINFO_TEXT: {
std::string message(data, size);
if (message[size - 1] == '\n')
message = message.substr(0, size - 1);
Project::debug(message);
} break;
default:
break;
}
return 0;
}

View File

@ -3,10 +3,12 @@
#include <string> #include <string>
#include <optional> #include <optional>
#include <filesystem> #include <filesystem>
#include <list>
#include <curl/curl.h>
#include <stdio.h> #include <stdio.h>
#include <nlohmann/json.hpp>
#include <curl/curl.h>
class Dependency { class Dependency {
public: public:
enum class Type { enum class Type {
@ -31,8 +33,26 @@ public:
const std::string path; const std::string path;
private: private:
static size_t write(void *file, size_t size, size_t nmemb, void *stream); static size_t writeFile(void* data, size_t size, size_t nmemb, void* file);
bool download(const std::filesystem::path& destination); static size_t writeString(void* data, size_t size, size_t nmemb, void* mem);
static int trace(CURL *handle, curl_infotype type, char *data, size_t size, void *clientp);
bool download(const std::string& url, const std::filesystem::path& destination);
bool downloadRepo(
const std::filesystem::path& destination,
const std::string& protocol,
const std::string& host,
const std::string& owner,
const std::string& repo
);
bool repoGiteaApi1(
const std::string& protocol,
const std::string& host,
const std::string& owner,
const std::string& repo,
nlohmann::json& json
);
CURLcode httpGet(const std::string& url, std::string& result, const std::list<std::string_view>& headers = {});
private: private:
Type type; Type type;

View File

@ -24,7 +24,8 @@ constexpr const std::string_view bold("\e[1m");
constexpr const std::string_view regular("\e[22m"); constexpr const std::string_view regular("\e[22m");
constexpr const std::string_view clearStyle("\e[0m"); constexpr const std::string_view clearStyle("\e[0m");
Logger::Logger(Logger::Severity severity): Logger::Logger(Logger::Severity severity, bool accumulate):
accumulate(accumulate),
currentSeverity(severity), currentSeverity(severity),
history(), history(),
readMutex(), readMutex(),
@ -34,26 +35,41 @@ Logger::Logger(Logger::Severity severity):
Logger::~Logger() Logger::~Logger()
{} {}
void Logger::printLog(bool colored) { void Logger::printLog(bool colored) const {
std::cout << std::endl; std::cout << std::endl;
for (const Message& message : history) { std::scoped_lock lock(readMutex);
if (colored) { for (const Message& message : history)
int severity = static_cast<int>(message.first); printMessage(message, colored);
std::cout << logSettings[severity] << bold << logHeaders[severity] << regular;
}
std::cout << message.second << std::endl;
}
if (colored) if (colored)
std::cout << clearStyle << std::flush; std::cout << clearStyle << std::flush;
} }
void Logger::log(Logger::Severity severity, const std::string& comment) const { void Logger::log(Logger::Severity severity, const std::string& comment, bool colored) const {
if (severity < currentSeverity)
return;
if (accumulate) {
std::scoped_lock lock(readMutex, writeMutex); std::scoped_lock lock(readMutex, writeMutex);
if (severity >= currentSeverity)
history.emplace_back(severity, comment); history.emplace_back(severity, comment);
} else {
std::scoped_lock lock(writeMutex);
printMessage({severity, comment}, colored);
if (colored)
std::cout << clearStyle << std::flush;
}
} }
std::list<std::pair<Logger::Severity, std::string>> Logger::getLog() const { std::list<std::pair<Logger::Severity, std::string>> Logger::getLog() const {
std::scoped_lock lock(readMutex); std::scoped_lock lock(readMutex);
return history; return history;
} }
void Logger::printMessage(const Message& message, bool colored) const {
if (colored) {
int severity = static_cast<int>(message.first);
std::cout << logSettings[severity] << bold << logHeaders[severity] << regular;
}
std::cout << message.second << std::endl;
}

View File

@ -20,14 +20,18 @@ public:
}; };
typedef std::pair<Severity, std::string> Message; typedef std::pair<Severity, std::string> Message;
Logger(Severity severity = Severity::info); Logger(Severity severity = Severity::info, bool accumulate = false);
~Logger(); ~Logger();
void log (Severity severity, const std::string& comment) const; void log (Severity severity, const std::string& comment, bool colored = true) const;
std::list<Message> getLog() const; std::list<Message> getLog() const;
void printLog(bool colored = true); void printLog(bool colored = true) const;
private: private:
void printMessage(const Message& message, bool colored = false) const;
private:
bool accumulate;
const Severity currentSeverity; const Severity currentSeverity;
mutable std::list<Message> history; mutable std::list<Message> history;
mutable std::mutex readMutex; mutable std::mutex readMutex;

View File

@ -25,12 +25,16 @@ Project::Project(const std::filesystem::path& location, const std::filesystem::p
dependencies(), dependencies(),
root(logger == nullptr) root(logger == nullptr)
{ {
if (root) if (root) {
logger = new Logger(); curl_global_init(CURL_GLOBAL_ALL);
logger = new Logger(Logger::Severity::debug);
}
} }
Project::~Project() { Project::~Project() {
if (root) { if (root) {
curl_global_cleanup();
delete logger; delete logger;
logger = nullptr; logger = nullptr;
} }
@ -178,6 +182,14 @@ void Project::debug(const std::string& message) {
log(Logger::Severity::debug, message); log(Logger::Severity::debug, message);
} }
void Project::minor(const std::string& message) {
log(Logger::Severity::minor, message);
}
void Project::major(const std::string& message) {
log(Logger::Severity::major, message);
}
void Project::error(const std::string& message) { void Project::error(const std::string& message) {
log(Logger::Severity::error, message); log(Logger::Severity::error, message);
} }

View File

@ -8,6 +8,7 @@
#include <stdint.h> #include <stdint.h>
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
#include <curl/curl.h>
#include "dependency.h" #include "dependency.h"
#include "loggger.h" #include "loggger.h"
@ -33,6 +34,8 @@ public:
static void log(Logger::Severity severity, const std::string& message); static void log(Logger::Severity severity, const std::string& message);
static void debug(const std::string& message); static void debug(const std::string& message);
static void info(const std::string& message); static void info(const std::string& message);
static void minor(const std::string& message);
static void major(const std::string& message);
static void warn(const std::string& message); static void warn(const std::string& message);
static void error(const std::string& message); static void error(const std::string& message);
static void fatal(const std::string& message); static void fatal(const std::string& message);