ExternalVendorCode/genode
1
0
Fork 0
mirror of https://github.com/genodelabs/genode.git synced 2025-02-20 17:52:52 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fetchurl progress reporting

Browse Source 
Refactor the fetchurl utility to optionally report the initial fetch
state, fetch progress, and the final state.

Fix #2702
This commit is contained in:
Emery Hemingway 2018-03-02 12:08:48 +01:00 committed by Christian Helmuth
parent ed1c87c8d6
commit 84ac5891b2
4 changed files with 234 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
repos/libports/recipes/src/fetchurl/used_apis
View File

@ -6,3 +6,4 @@ vfs
curl
timer_session
nic_session
report_session

8
repos/libports/run/fetchurl.run
View File

@ -34,6 +34,7 @@ append config {
</parent-provides>
<default caps="100"/>
<default-route>
<service name="Report"> <child name="report_rom"/> </service>
<any-service> <parent/> <any-child/> </any-service>
</default-route>}
@ -49,9 +50,16 @@ append config {
<resource name="RAM" quantum="4M"/>
<provides> <service name="Nic"/> </provides>
</start>
<start name="report_rom">
<resource name="RAM" quantum="4M"/>
<provides> <service name="Report"/> </provides>
<config verbose="yes"/>
</start>
<start name="fetchurl" caps="500">
<resource name="RAM" quantum="32M"/>
<config>
<report progress="yes"/>
<vfs>
<dir name="etc">
<inline name="resolv.conf">nameserver 213.73.91.35</inline>

13
repos/libports/src/app/fetchurl/README
View File

@ -10,9 +10,20 @@ Configuration
! </vfs>
! </libc>
! <fetch url="http://genode.org/about/LICENSE" path="LICENSE"/>
! <report progress="yes"/>
!</config>
Optionally, you can use a proxy:
! <fetch url="http://genode.org/about/LICENSE" path="LICENSE"
! proxy="sock5://10.0.0.1:9050 />
! proxy="sock5://10.0.0.1:9050" />
The presence of a 'report' node in the configuration with an
affirmative 'progress' attribute will enable a progress report.
The 'delay_ms' attribute will set the minimum interval between
reports and defauts to 100 miliseconds. The report format is as
follows.
! <progress>
! <fetch url="..." total="100.0" now="50.0"/>
! </progress>

269
repos/libports/src/app/fetchurl/component.cc
View File

@ -14,8 +14,9 @@
/* Genode includes */
#include <timer_session/connection.h>
#include <os/path.h>
#include <base/attached_rom_dataspace.h>
#include <os/reporter.h>
#include <libc/component.h>
#include <base/heap.h>
#include <base/log.h>
/* cURL includes */
@ -27,37 +28,148 @@
#include <errno.h>
#include <sys/stat.h>
namespace Fetchurl {
class Fetch;
struct Main;
typedef Genode::String<256> Url;
typedef Genode::Path<256> Path;
}
static size_t write_callback(char *ptr,
size_t size,
size_t nmemb,
void *userdata)
void *userdata);
static int progress_callback(void *userdata,
double dltotal, double dlnow,
double ultotal, double ulnow);
class Fetchurl::Fetch : Genode::List<Fetch>::Element
{
int *fd = (int*)userdata;
return write(*fd, ptr, size*nmemb);
}
friend class Genode::List<Fetch>;
public:
Main &main;
using Genode::List<Fetch>::Element::next;
Url const url;
Path const path;
Url const proxy;
double dltotal = 0;
double dlnow = 0;
int fd = -1;
Fetch(Main &main, Url const &url, Path const &path, Url const &proxy)
: main(main), url(url), path(path), proxy(proxy) { }
};
static int fetchurl(Genode::Xml_node config_node)
struct Fetchurl::Main
{
Genode::String<256> url;
Genode::Path<256> path;
CURLcode res = CURLE_OK;
Main(Main const &);
Main &operator = (Main const &);
Libc::with_libc([&]() { curl_global_init(CURL_GLOBAL_DEFAULT); });
Libc::Env &_env;
bool verbose = config_node.attribute_value("verbose", false);
Genode::Heap _heap { _env.pd(), _env.rm() };
config_node.for_each_sub_node("fetch", [&] (Genode::Xml_node node) {
Timer::Connection _timer { _env, "reporter" };
if (res != CURLE_OK) return;
Genode::Reporter _reporter { _env, "progress" };
Genode::List<Fetch> _fetches { };
Timer::One_shot_timeout<Main> _report_timeout {
_timer, *this, &Main::_report };
Genode::Duration _report_delay { Genode::Milliseconds { 0 } };
void _schedule_report()
{
using namespace Genode;
if ((_report_delay.trunc_to_plain_ms().value > 0) &&
(!_report_timeout.scheduled()))
{
_report_timeout.schedule(_report_delay.trunc_to_plain_us());
}
}
void _report()
{
using namespace Genode;
Reporter::Xml_generator xml_gen(_reporter, [&] {
for (Fetch *f = _fetches.first(); f; f = f->next()) {
xml_gen.node("fetch", [&] {
xml_gen.attribute("url", f->url);
xml_gen.attribute("total", f->dltotal);
xml_gen.attribute("now", f->dlnow);
});
}
});
}
void _report(Genode::Duration) { _report(); }
void parse_config(Genode::Xml_node const &config_node)
{
using namespace Genode;
try {
node.attribute("url").value(&url);
node.attribute("path").value(path.base(), path.capacity());
} catch (...) { Genode::error("error reading 'fetch' node"); return; }
enum { DEFAULT_DELAY_MS = 100UL };
char const *out_path = path.base();
Xml_node const report_node = config_node.sub_node("report");
if (report_node.attribute_value("progress", false)) {
Milliseconds delay_ms { 0 };
delay_ms.value = report_node.attribute_value(
"delay_ms", (unsigned)DEFAULT_DELAY_MS);
if (delay_ms.value < 1)
delay_ms.value = DEFAULT_DELAY_MS;
_report_delay = Duration(delay_ms);
_schedule_report();
_reporter.enabled(true);
}
}
catch (...) { }
auto const parse_fn = [&] (Genode::Xml_node node) {
Url url;
Path path;
Url proxy;
try {
node.attribute("url").value(&url);
node.attribute("path").value(path.base(), path.capacity());
}
catch (...) { Genode::error("error reading 'fetch' XML node"); return; }
try { config_node.attribute("proxy").value(&proxy); }
catch (...) { }
auto *f = new (_heap) Fetch(*this, url, path, proxy);
_fetches.insert(f);
};
config_node.for_each_sub_node("fetch", parse_fn);
}
Main(Libc::Env &e) : _env(e)
{
_env.config([&] (Genode::Xml_node const &config) {
parse_config(config);
});
}
CURLcode _process_fetch(CURL *_curl, Fetch &_fetch)
{
char const *out_path = _fetch.path.base();
/* create compound directories leading to the path */
for (size_t sub_path_len = 0; ; sub_path_len++) {
@ -86,7 +198,7 @@ static int fetchurl(Genode::Xml_node config_node)
/* create directory for sub path */
if (mkdir(sub_path.string(), 0777) < 0) {
Genode::error("failed to create directory ", sub_path);
break;
return CURLE_FAILED_INIT;
}
}
@ -104,62 +216,107 @@ static int fetchurl(Genode::Xml_node config_node)
default:
Genode::error("creation of ", out_path, " failed (errno=", errno, ")");
}
res = CURLE_FAILED_INIT;
return;
return CURLE_FAILED_INIT;
}
_fetch.fd = fd;
curl_easy_setopt(_curl, CURLOPT_URL, _fetch.url.string());
curl_easy_setopt(_curl, CURLOPT_FOLLOWLOCATION, true);
curl_easy_setopt(_curl, CURLOPT_NOSIGNAL, true);
curl_easy_setopt(_curl, CURLOPT_FAILONERROR, 1L);
curl_easy_setopt(_curl, CURLOPT_WRITEFUNCTION, write_callback);
curl_easy_setopt(_curl, CURLOPT_WRITEDATA, &_fetch);
curl_easy_setopt(_curl, CURLOPT_NOPROGRESS, 0L);
curl_easy_setopt(_curl, CURLOPT_PROGRESSFUNCTION, progress_callback);
curl_easy_setopt(_curl, CURLOPT_PROGRESSDATA, &_fetch);
curl_easy_setopt(_curl, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt(_curl, CURLOPT_SSL_VERIFYHOST, 0L);
/* check for optional proxy configuration */
if (_fetch.proxy != "") {
curl_easy_setopt(_curl, CURLOPT_PROXY, _fetch.proxy.string());
}
CURLcode res = curl_easy_perform(_curl);
close(_fetch.fd);
_fetch.fd = -1;
if (res != CURLE_OK)
Genode::error(curl_easy_strerror(res), ", failed to fetch ", _fetch.url);
return res;
}
int run()
{
CURLcode res = CURLE_OK;
CURL *curl = curl_easy_init();
if (!curl) {
Genode::error("failed to initialize libcurl");
res = CURLE_FAILED_INIT;
return;
return -1;
}
curl_easy_setopt(curl, CURLOPT_URL, url.string());
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true);
if (_reporter.enabled())
_report();
curl_easy_setopt(curl, CURLOPT_VERBOSE, verbose);
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, true);
curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L);
for (Fetch *f = _fetches.first(); f; f = f->next()) {
if (res != CURLE_OK) break;
res = _process_fetch(curl, *f);
}
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &fd);
if (_reporter.enabled())
_report();
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L);
curl_easy_cleanup(curl);
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
return res ^ CURLE_OK;
}
};
/* check for optional proxy configuration */
try {
Genode::String<256> proxy;
node.attribute("proxy").value(&proxy);
curl_easy_setopt(curl, CURLOPT_PROXY, proxy.string());
} catch (...) { }
Libc::with_libc([&]() {
res = curl_easy_perform(curl);
close(fd);
if (res != CURLE_OK)
Genode::error(curl_easy_strerror(res));
curl_easy_cleanup(curl);
});
});
curl_global_cleanup();
Genode::warning("SSL certificates not verified");
return res ^ CURLE_OK;
static size_t write_callback(char *ptr,
size_t size,
size_t nmemb,
void *userdata)
{
Fetchurl::Fetch &fetch = *((Fetchurl::Fetch *)userdata);
return write(fetch.fd, ptr, size*nmemb);
}
static int progress_callback(void *userdata,
double dltotal, double dlnow,
double ultotal, double ulnow)
{
(void)ultotal;
(void)ulnow;
Fetchurl::Fetch &fetch = *((Fetchurl::Fetch *)userdata);
fetch.dltotal = dltotal;
fetch.dlnow = dlnow;
fetch.main._schedule_report();
return CURLE_OK;
}
void Libc::Component::construct(Libc::Env &env)
{
env.config([&env] (Genode::Xml_node config) {
env.parent().exit( fetchurl(config) );
int res = -1;
Libc::with_libc([&]() {
curl_global_init(CURL_GLOBAL_DEFAULT);
static Fetchurl::Main inst(env);
res = inst.run();
curl_global_cleanup();
});
env.parent().exit(res);
}
/* dummies to prevent warnings printed by unimplemented libc functions */