ExternalVendorCode/genode
1
0
Fork 0
mirror of https://github.com/genodelabs/genode.git synced 2025-01-11 23:43:15 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fetchurl: optionally attempt to re-fetch failed URLs.

Browse Source 
Fix #2733
This commit is contained in:
Emery Hemingway 2018-03-25 11:24:19 +02:00 committed by Christian Helmuth
parent 1ca2b6fab3
commit e1a079954b
3 changed files with 42 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
repos/libports/run/fetchurl.run
View File

@ -71,7 +71,7 @@ append config {
<dir name="socket"> <lxip dhcp="yes"/> </dir> <dir name="socket"> <lxip dhcp="yes"/> </dir>
</vfs> </vfs>
<libc stdout="/dev/log" stderr="/dev/log" rtc="/dev/rtc" socket="/socket"/> <libc stdout="/dev/log" stderr="/dev/log" rtc="/dev/rtc" socket="/socket"/>
<fetch url="http://genode.org/about/LICENSE" path="/dev/log"/> <fetch url="http://genode.org/about/LICENSE" path="/dev/log" retry="3"/>
</config> </config>
</start> </start>
</config> </config>

9
repos/libports/src/app/fetchurl/README
View File

@ -9,11 +9,16 @@ Configuration
! <dir name="dev"> <jitterentropy name="random"/> <log/> </dir> ! <dir name="dev"> <jitterentropy name="random"/> <log/> </dir>
! </vfs> ! </vfs>
! </libc> ! </libc>
! <fetch url="http://genode.org/about/LICENSE" path="LICENSE"/> ! <fetch url="http://genode.org/about/LICENSE" path="LICENSE" retry="3"/>
! <report progress="yes"/> ! <report progress="yes"/>
!</config> !</config>
Optionally, you can use a proxy: The two primary options are 'url', the URL to fetch, and 'path', the
file-system path where fetched data is written. Secondary options are
'retry' and 'proxy'. Retry is the number of fetch attempts to make
following failure, and proxy is used to reroute requests.
An example TOR proxying configuration:
! <fetch url="http://genode.org/about/LICENSE" path="LICENSE" ! <fetch url="http://genode.org/about/LICENSE" path="LICENSE"
! proxy="sock5://10.0.0.1:9050" /> ! proxy="sock5://10.0.0.1:9050" />

42
repos/libports/src/app/fetchurl/component.cc
View File

@ -52,21 +52,26 @@ class Fetchurl::Fetch : Genode::List<Fetch>::Element
public: public:
Main &main;
using Genode::List<Fetch>::Element::next; using Genode::List<Fetch>::Element::next;
Main &main;
Url const url; Url const url;
Path const path; Path const path;
Url const proxy; Url const proxy;
long retry;
double dltotal = 0; double dltotal = 0;
double dlnow = 0; double dlnow = 0;
int fd = -1; int fd = -1;
Fetch(Main &main, Url const &url, Path const &path, Url const &proxy) Fetch(Main &main, Url const &url, Path const &path,
: main(main), url(url), path(path), proxy(proxy) { } Url const &proxy, long retry)
:
main(main), url(url), path(path),
proxy(proxy), retry(retry+1)
{ }
}; };
@ -143,6 +148,7 @@ struct Fetchurl::Main
Url url; Url url;
Path path; Path path;
Url proxy; Url proxy;
long retry;
try { try {
node.attribute("url").value(&url); node.attribute("url").value(&url);
@ -150,10 +156,10 @@ struct Fetchurl::Main
} }
catch (...) { Genode::error("error reading 'fetch' XML node"); return; } catch (...) { Genode::error("error reading 'fetch' XML node"); return; }
try { config_node.attribute("proxy").value(&proxy); } proxy = node.attribute_value("proxy", Url());
catch (...) { } retry = node.attribute_value("retry", 0L);
auto *f = new (_heap) Fetch(*this, url, path, proxy); auto *f = new (_heap) Fetch(*this, url, path, proxy, retry);
_fetches.insert(f); _fetches.insert(f);
}; };
@ -169,6 +175,8 @@ struct Fetchurl::Main
CURLcode _process_fetch(CURL *_curl, Fetch &_fetch) CURLcode _process_fetch(CURL *_curl, Fetch &_fetch)
{ {
Genode::log("fetch ", _fetch.url);
char const *out_path = _fetch.path.base(); char const *out_path = _fetch.path.base();
/* create compound directories leading to the path */ /* create compound directories leading to the path */
@ -252,7 +260,7 @@ struct Fetchurl::Main
int run() int run()
{ {
CURLcode res = CURLE_OK; CURLcode exit_res = CURLE_OK;
CURL *curl = curl_easy_init(); CURL *curl = curl_easy_init();
if (!curl) { if (!curl) {
@ -260,12 +268,24 @@ struct Fetchurl::Main
return -1; return -1;
} }
while (true) {
if (_reporter.enabled()) if (_reporter.enabled())
_report(); _report();
bool retry_some = false;
for (Fetch *f = _fetches.first(); f; f = f->next()) { for (Fetch *f = _fetches.first(); f; f = f->next()) {
if (res != CURLE_OK) break; if (f->retry < 1) continue;
res = _process_fetch(curl, *f); CURLcode res = _process_fetch(curl, *f);
if (res == CURLE_OK) {
f->retry = 0;
} else {
if (--f->retry > 0)
retry_some = true;
else
exit_res = res;
}
}
if (!retry_some) break;
} }
if (_reporter.enabled()) if (_reporter.enabled())
@ -273,7 +293,7 @@ struct Fetchurl::Main
curl_easy_cleanup(curl); curl_easy_cleanup(curl);
return res ^ CURLE_OK; return exit_res ^ CURLE_OK;
} }
}; };