From e1a079954bdc2be70ac6fcbe8ec2b5c0a69e7404 Mon Sep 17 00:00:00 2001
From: Emery Hemingway <ehmry@posteo.net>
Date: Sun, 25 Mar 2018 11:24:19 +0200
Subject: [PATCH] Fetchurl: optionally attempt to re-fetch failed URLs.

Fix #2733
---
 repos/libports/run/fetchurl.run              |  2 +-
 repos/libports/src/app/fetchurl/README       |  9 +++-
 repos/libports/src/app/fetchurl/component.cc | 48 ++++++++++++++------
 3 files changed, 42 insertions(+), 17 deletions(-)

diff --git a/repos/libports/run/fetchurl.run b/repos/libports/run/fetchurl.run
index c349ae2e46..f83a4e027b 100644
--- a/repos/libports/run/fetchurl.run
+++ b/repos/libports/run/fetchurl.run
@@ -71,7 +71,7 @@ append config {
 				<dir name="socket"> <lxip dhcp="yes"/> </dir>
 			</vfs>
 			<libc stdout="/dev/log" stderr="/dev/log" rtc="/dev/rtc" socket="/socket"/>
-			<fetch url="http://genode.org/about/LICENSE" path="/dev/log"/>
+			<fetch url="http://genode.org/about/LICENSE" path="/dev/log" retry="3"/>
 		</config>
 	</start>
 </config>
diff --git a/repos/libports/src/app/fetchurl/README b/repos/libports/src/app/fetchurl/README
index 39155ac6f3..8692de6842 100644
--- a/repos/libports/src/app/fetchurl/README
+++ b/repos/libports/src/app/fetchurl/README
@@ -9,11 +9,16 @@ Configuration
 !   <dir name="dev"> <jitterentropy name="random"/> <log/> </dir>
 !  </vfs>
 ! </libc>
-! <fetch url="http://genode.org/about/LICENSE" path="LICENSE"/>
+! <fetch url="http://genode.org/about/LICENSE" path="LICENSE" retry="3"/>
 ! <report progress="yes"/>
 !</config>
 
-Optionally, you can use a proxy:
+The two primary options are 'url', the URL to fetch, and 'path', the
+file-system path where fetched data is written. Secondary options are
+'retry' and 'proxy'. Retry is the number of fetch attempts to make
+following failure, and proxy is used to reroute requests.
+
+An example TOR proxying configuration:
 
 ! <fetch url="http://genode.org/about/LICENSE" path="LICENSE"
 !        proxy="sock5://10.0.0.1:9050" />
diff --git a/repos/libports/src/app/fetchurl/component.cc b/repos/libports/src/app/fetchurl/component.cc
index fc57667a96..7a4a0b3a93 100644
--- a/repos/libports/src/app/fetchurl/component.cc
+++ b/repos/libports/src/app/fetchurl/component.cc
@@ -52,21 +52,26 @@ class Fetchurl::Fetch : Genode::List<Fetch>::Element
 
 	public:
 
-		Main &main;
-
 		using Genode::List<Fetch>::Element::next;
 
+		Main &main;
+
 		Url  const url;
 		Path const path;
 		Url  const proxy;
+		long       retry;
 
 		double dltotal = 0;
 		double dlnow = 0;
 
 		int fd = -1;
 
-		Fetch(Main &main, Url const &url, Path const &path, Url const &proxy)
-		: main(main), url(url), path(path), proxy(proxy) { }
+		Fetch(Main &main, Url const &url, Path const &path,
+		      Url const &proxy, long retry)
+		:
+			main(main), url(url), path(path),
+			proxy(proxy), retry(retry+1)
+		{ }
 };
 
 
@@ -143,6 +148,7 @@ struct Fetchurl::Main
 			Url url;
 			Path path;
 			Url proxy;
+			long retry;
 
 			try {
 				node.attribute("url").value(&url);
@@ -150,10 +156,10 @@ struct Fetchurl::Main
 			}
 			catch (...) { Genode::error("error reading 'fetch' XML node"); return; }
 
-			try { config_node.attribute("proxy").value(&proxy); }
-			catch (...) { }
+			proxy = node.attribute_value("proxy", Url());
+			retry = node.attribute_value("retry", 0L);
 
-			auto *f = new (_heap) Fetch(*this, url, path, proxy);
+			auto *f = new (_heap) Fetch(*this, url, path, proxy, retry);
 			_fetches.insert(f);
 		};
 
@@ -169,6 +175,8 @@ struct Fetchurl::Main
 
 	CURLcode _process_fetch(CURL *_curl, Fetch &_fetch)
 	{
+		Genode::log("fetch ", _fetch.url);
+
 		char const *out_path = _fetch.path.base();
 
 		/* create compound directories leading to the path */
@@ -252,7 +260,7 @@ struct Fetchurl::Main
 
 	int run()
 	{
-		CURLcode res = CURLE_OK;
+		CURLcode exit_res = CURLE_OK;
 
 		CURL *curl = curl_easy_init();
 		if (!curl) {
@@ -260,12 +268,24 @@ struct Fetchurl::Main
 			return -1;
 		}
 
-		if (_reporter.enabled())
-			_report();
+		while (true) {
+			if (_reporter.enabled())
+				_report();
 
-		for (Fetch *f = _fetches.first(); f; f = f->next()) {
-			if (res != CURLE_OK) break;
-			res = _process_fetch(curl, *f);
+			bool retry_some = false;
+			for (Fetch *f = _fetches.first(); f; f = f->next()) {
+				if (f->retry < 1) continue;
+				CURLcode res = _process_fetch(curl, *f);
+				if (res == CURLE_OK) {
+					f->retry = 0;
+				} else {
+					if (--f->retry > 0)
+						retry_some = true;
+					else
+						exit_res = res;
+				}
+			}
+			if (!retry_some) break;
 		}
 
 		if (_reporter.enabled())
@@ -273,7 +293,7 @@ struct Fetchurl::Main
 
 		curl_easy_cleanup(curl);
 
-		return res ^ CURLE_OK;
+		return exit_res ^ CURLE_OK;
 	}
 };