genode/repos/libports/src/app/fetchurl/component.cc
Josef Söntgen c8c354d1e6 fetchurl: add progress timeout handling
The component will now abort an ongoing download attempt if it stalls
for given amount of time, the default is 10 seconds.

Fixes #3346.
2019-05-27 14:46:52 +02:00

398 lines
9.3 KiB
C++

/*
* \brief Native fetchurl utility for Nix
* \author Emery Hemingway
* \date 2016-03-08
*/
/*
* Copyright (C) 2016-2018 Genode Labs GmbH
*
* This file is part of the Genode OS framework, which is distributed
* under the terms of the GNU Affero General Public License version 3.
*/
/* Genode includes */
#include <timer_session/connection.h>
#include <os/path.h>
#include <os/reporter.h>
#include <libc/component.h>
#include <base/heap.h>
#include <base/log.h>
/* cURL includes */
#include <curl/curl.h>
/* Libc includes */
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <sys/stat.h>
namespace Fetchurl {
class Fetch;
struct User_data;
struct Main;
typedef Genode::String<256> Url;
typedef Genode::Path<256> Path;
}
static size_t write_callback(char *ptr,
size_t size,
size_t nmemb,
void *userdata);
static int progress_callback(void *userdata,
double dltotal, double dlnow,
double ultotal, double ulnow);
class Fetchurl::Fetch : Genode::List<Fetch>::Element
{
friend class Genode::List<Fetch>;
public:
using Genode::List<Fetch>::Element::next;
Main &main;
Url const url;
Path const path;
Url const proxy;
long retry;
double dltotal = 0;
double dlnow = 0;
bool timeout = false;
int fd = -1;
Fetch(Main &main, Url const &url, Path const &path,
Url const &proxy, long retry)
:
main(main), url(url), path(path),
proxy(proxy), retry(retry+1)
{ }
};
struct Fetchurl::User_data
{
Timer::Connection &timer;
Genode::Milliseconds last_ms;
Genode::Milliseconds const max_timeout;
Genode::Milliseconds curr_timeout;
Fetchurl::Fetch &fetch;
};
struct Fetchurl::Main
{
Main(Main const &);
Main &operator = (Main const &);
Libc::Env &_env;
Genode::Heap _heap { _env.pd(), _env.rm() };
Timer::Connection _timer { _env, "reporter" };
Genode::Constructible<Genode::Expanding_reporter> _reporter { };
Genode::List<Fetch> _fetches { };
Timer::One_shot_timeout<Main> _report_timeout {
_timer, *this, &Main::_report };
Genode::Duration _report_delay { Genode::Milliseconds { 0 } };
Genode::Milliseconds _progress_timeout { 10u * 1000 };
void _schedule_report()
{
using namespace Genode;
if ((_report_delay.trunc_to_plain_ms().value > 0) &&
(!_report_timeout.scheduled()))
{
_report_timeout.schedule(_report_delay.trunc_to_plain_us());
}
}
void _report()
{
if (!_reporter.constructed())
return;
_reporter->generate([&] (Genode::Xml_generator &xml) {
for (Fetch *f = _fetches.first(); f; f = f->next()) {
xml.node("fetch", [&] {
xml.attribute("url", f->url);
xml.attribute("total", f->dltotal);
xml.attribute("now", f->dlnow);
if (f->timeout) {
xml.attribute("timeout", true);
}
});
}
});
}
void _report(Genode::Duration) { _report(); }
void parse_config(Genode::Xml_node const &config_node)
{
using namespace Genode;
try {
enum { DEFAULT_DELAY_MS = 100UL };
Xml_node const report_node = config_node.sub_node("report");
if (report_node.attribute_value("progress", false)) {
Milliseconds delay_ms { 0 };
delay_ms.value = report_node.attribute_value(
"delay_ms", (unsigned)DEFAULT_DELAY_MS);
if (delay_ms.value < 1)
delay_ms.value = DEFAULT_DELAY_MS;
_report_delay = Duration(delay_ms);
_schedule_report();
_reporter.construct(_env, "progress", "progress");
}
}
catch (...) { }
_progress_timeout.value = config_node.attribute_value("progress_timeout",
_progress_timeout.value);
auto const parse_fn = [&] (Genode::Xml_node node) {
if (!node.has_attribute("url") || !node.has_attribute("path")) {
Genode::error("error reading 'fetch' XML node");
return;
}
Url const url = node.attribute_value("url", Url());
Path const path = node.attribute_value("path", String<256>());
Url const proxy = node.attribute_value("proxy", Url());
long const retry = node.attribute_value("retry", 0L);
auto *f = new (_heap) Fetch(*this, url, path, proxy, retry);
_fetches.insert(f);
};
config_node.for_each_sub_node("fetch", parse_fn);
}
Main(Libc::Env &e) : _env(e)
{
_env.config([&] (Genode::Xml_node const &config) {
parse_config(config);
});
}
CURLcode _process_fetch(CURL *_curl, Fetch &_fetch)
{
Genode::log("fetch ", _fetch.url);
char const *out_path = _fetch.path.base();
/* create compound directories leading to the path */
for (size_t sub_path_len = 0; ; sub_path_len++) {
bool const end_of_path = (out_path[sub_path_len] == 0);
bool const end_of_elem = (out_path[sub_path_len] == '/');
if (end_of_path)
break;
if (!end_of_elem)
continue;
Genode::String<256> sub_path(Genode::Cstring(out_path, sub_path_len));
/* skip '/' */
sub_path_len++;
/* if sub path is a directory, we are fine */
struct stat sb;
sb.st_mode = 0;
stat(sub_path.string(), &sb);
if (S_ISDIR(sb.st_mode))
continue;
/* create directory for sub path */
if (mkdir(sub_path.string(), 0777) < 0) {
Genode::error("failed to create directory ", sub_path);
return CURLE_FAILED_INIT;
}
}
int fd = open(out_path, O_CREAT | O_RDWR);
if (fd == -1) {
switch (errno) {
case EACCES:
Genode::error("permission denied at ", out_path); break;
case EEXIST:
Genode::error(out_path, " already exists"); break;
case EISDIR:
Genode::error(out_path, " is a directory"); break;
case ENOSPC:
Genode::error("cannot create ", out_path, ", out of space"); break;
default:
Genode::error("creation of ", out_path, " failed (errno=", errno, ")");
}
return CURLE_FAILED_INIT;
}
_fetch.fd = fd;
curl_easy_setopt(_curl, CURLOPT_URL, _fetch.url.string());
curl_easy_setopt(_curl, CURLOPT_FOLLOWLOCATION, true);
curl_easy_setopt(_curl, CURLOPT_NOSIGNAL, true);
curl_easy_setopt(_curl, CURLOPT_FAILONERROR, 1L);
curl_easy_setopt(_curl, CURLOPT_WRITEFUNCTION, write_callback);
curl_easy_setopt(_curl, CURLOPT_WRITEDATA, &_fetch);
curl_easy_setopt(_curl, CURLOPT_NOPROGRESS, 0L);
curl_easy_setopt(_curl, CURLOPT_PROGRESSFUNCTION, progress_callback);
User_data ud {
.timer = _timer,
.last_ms = _timer.curr_time().trunc_to_plain_ms(),
.max_timeout = _progress_timeout,
.curr_timeout = Genode::Milliseconds { .value = 0 },
.fetch = _fetch,
};
curl_easy_setopt(_curl, CURLOPT_PROGRESSDATA, &ud);
curl_easy_setopt(_curl, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt(_curl, CURLOPT_SSL_VERIFYHOST, 0L);
/* check for optional proxy configuration */
if (_fetch.proxy != "") {
curl_easy_setopt(_curl, CURLOPT_PROXY, _fetch.proxy.string());
}
CURLcode res = curl_easy_perform(_curl);
close(_fetch.fd);
_fetch.fd = -1;
if (res != CURLE_OK)
Genode::error(curl_easy_strerror(res), ", failed to fetch ", _fetch.url);
return res;
}
int run()
{
CURLcode exit_res = CURLE_OK;
CURL *curl = curl_easy_init();
if (!curl) {
Genode::error("failed to initialize libcurl");
return -1;
}
while (true) {
_report();
bool retry_some = false;
for (Fetch *f = _fetches.first(); f; f = f->next()) {
if (f->retry < 1) continue;
CURLcode res = _process_fetch(curl, *f);
if (res == CURLE_OK) {
f->retry = 0;
} else {
if (--f->retry > 0)
retry_some = true;
else
exit_res = res;
}
}
if (!retry_some) break;
}
_report();
curl_easy_cleanup(curl);
return exit_res ^ CURLE_OK;
}
};
static size_t write_callback(char *ptr,
size_t size,
size_t nmemb,
void *userdata)
{
Fetchurl::Fetch &fetch = *((Fetchurl::Fetch *)userdata);
return write(fetch.fd, ptr, size*nmemb);
}
static int progress_callback(void *userdata,
double dltotal, double dlnow,
double ultotal, double ulnow)
{
(void)ultotal;
(void)ulnow;
using namespace Fetchurl;
using namespace Genode;
User_data &ud = *reinterpret_cast<User_data*>(userdata);
Timer::Connection &timer = ud.timer;
Fetch &fetch = ud.fetch;
Milliseconds curr { timer.curr_time().trunc_to_plain_ms() };
Milliseconds diff { .value = curr.value - ud.last_ms.value };
ud.last_ms = curr;
/*
* To catch stuck downloads we increase the timeout time whenever
* the current download rate is same as the last one. When we hit
* the max timeout value, we will abort the download attempt.
*/
if (dlnow == fetch.dlnow) {
ud.curr_timeout.value += diff.value;
}
else {
ud.curr_timeout.value = 0;
}
bool const timeout = ud.curr_timeout.value >= ud.max_timeout.value;
fetch.dltotal = dltotal;
fetch.dlnow = dlnow;
fetch.timeout = timeout;
fetch.main._schedule_report();
/* non-zero return is enough to trigger an abort */
return timeout ? CURLE_GOT_NOTHING : CURLE_OK;
}
void Libc::Component::construct(Libc::Env &env)
{
int res = -1;
Libc::with_libc([&]() {
curl_global_init(CURL_GLOBAL_DEFAULT);
static Fetchurl::Main inst(env);
res = inst.run();
curl_global_cleanup();
});
env.parent().exit(res);
}
/* dummies to prevent warnings printed by unimplemented libc functions */
extern "C" int issetugid() { return 1; }
extern "C" pid_t getpid() { return 1; }