Fetchurl progress reporting

Refactor the fetchurl utility to optionally report the initial fetch
state, fetch progress, and the final state.

Fix #2702

repos/libports/recipes/src/fetchurl/used_apis

@@ -6,3 +6,4 @@ vfs
 curl
 timer_session
 nic_session
+report_session

repos/libports/run/fetchurl.run

@@ -34,6 +34,7 @@ append config {
 	</parent-provides>
 	<default caps="100"/>
 	<default-route>
+		<service name="Report"> <child name="report_rom"/> </service>
 		<any-service> <parent/> <any-child/> </any-service>
 	</default-route>}
 
@@ -49,9 +50,16 @@ append config {
 		<resource name="RAM" quantum="4M"/>
 		<provides> <service name="Nic"/> </provides>
 	</start>
+	<start name="report_rom">
+		<resource name="RAM" quantum="4M"/>
+		<provides> <service name="Report"/> </provides>
+		<config verbose="yes"/>
+	</start>
+
 	<start name="fetchurl" caps="500">
 		<resource name="RAM" quantum="32M"/>
 		<config>
+			<report progress="yes"/>
 			<vfs>
 				<dir name="etc">
 					<inline name="resolv.conf">nameserver 213.73.91.35</inline>

repos/libports/src/app/fetchurl/README

@@ -10,9 +10,20 @@ Configuration
 !  </vfs>
 ! </libc>
 ! <fetch url="http://genode.org/about/LICENSE" path="LICENSE"/>
+! <report progress="yes"/>
 !</config>
 
 Optionally, you can use a proxy:
 
 ! <fetch url="http://genode.org/about/LICENSE" path="LICENSE"
-!        proxy="sock5://10.0.0.1:9050 />
+!        proxy="sock5://10.0.0.1:9050" />
+
+The presence of a 'report' node in the configuration with an
+affirmative 'progress' attribute will enable a progress report.
+The 'delay_ms' attribute will set the minimum interval between
+reports and defauts to 100 miliseconds. The report format is as
+follows.
+
+! <progress>
+!   <fetch url="..." total="100.0" now="50.0"/>
+! </progress>

repos/libports/src/app/fetchurl/component.cc

@@ -14,8 +14,9 @@
 /* Genode includes */
 #include <timer_session/connection.h>
 #include <os/path.h>
-#include <base/attached_rom_dataspace.h>
+#include <os/reporter.h>
 #include <libc/component.h>
+#include <base/heap.h>
 #include <base/log.h>
 
 /* cURL includes */
@@ -27,37 +28,148 @@
 #include <errno.h>
 #include <sys/stat.h>
 
+namespace Fetchurl {
+	class Fetch;
+	struct Main;
+
+	typedef Genode::String<256> Url;
+	typedef Genode::Path<256>   Path;
+}
 
 static size_t write_callback(char   *ptr,
                              size_t  size,
                              size_t  nmemb,
-                             void   *userdata)
+                             void   *userdata);
+
+static int progress_callback(void *userdata,
+                             double dltotal, double dlnow,
+                             double ultotal, double ulnow);
+
+
+class Fetchurl::Fetch : Genode::List<Fetch>::Element
 {
-	int *fd = (int*)userdata;
+	friend class Genode::List<Fetch>;
-	return write(*fd, ptr, size*nmemb);
+
-}
+	public:
+
+		Main &main;
+
+		using Genode::List<Fetch>::Element::next;
+
+		Url  const url;
+		Path const path;
+		Url  const proxy;
+
+		double dltotal = 0;
+		double dlnow = 0;
+
+		int fd = -1;
+
+		Fetch(Main &main, Url const &url, Path const &path, Url const &proxy)
+		: main(main), url(url), path(path), proxy(proxy) { }
+};
 
 
-static int fetchurl(Genode::Xml_node config_node)
+struct Fetchurl::Main
 {
-	Genode::String<256> url;
+	Main(Main const &);
-	Genode::Path<256>   path;
+	Main &operator = (Main const &);
-	CURLcode res = CURLE_OK;
 
-	Libc::with_libc([&]() { curl_global_init(CURL_GLOBAL_DEFAULT); });
+	Libc::Env &_env;
 
-	bool verbose = config_node.attribute_value("verbose", false);
+	Genode::Heap _heap { _env.pd(), _env.rm() };
 
-	config_node.for_each_sub_node("fetch", [&] (Genode::Xml_node node) {
+	Timer::Connection _timer { _env, "reporter" };
 
-		if (res != CURLE_OK) return;
+	Genode::Reporter _reporter { _env, "progress" };
+
+	Genode::List<Fetch> _fetches { };
+
+	Timer::One_shot_timeout<Main> _report_timeout {
+		_timer, *this, &Main::_report };
+
+	Genode::Duration _report_delay { Genode::Milliseconds { 0 } };
+
+	void _schedule_report()
+	{
+		using namespace Genode;
+
+		if ((_report_delay.trunc_to_plain_ms().value > 0) &&
+		    (!_report_timeout.scheduled()))
+		{
+			_report_timeout.schedule(_report_delay.trunc_to_plain_us());
+		}
+	}
+
+	void _report()
+	{
+		using namespace Genode;
+		Reporter::Xml_generator xml_gen(_reporter, [&] {
+			for (Fetch *f = _fetches.first(); f; f = f->next()) {
+				xml_gen.node("fetch", [&] {
+					xml_gen.attribute("url", f->url);
+					xml_gen.attribute("total", f->dltotal);
+					xml_gen.attribute("now", f->dlnow);
+				});
+			}
+		});
+	}
+
+	void _report(Genode::Duration) { _report(); }
+
+	void parse_config(Genode::Xml_node const &config_node)
+	{
+		using namespace Genode;
+
+		try {
+			enum { DEFAULT_DELAY_MS = 100UL };
+
+			Xml_node const report_node = config_node.sub_node("report");
+			if (report_node.attribute_value("progress", false)) {
+				Milliseconds delay_ms { 0 };
+				delay_ms.value = report_node.attribute_value(
+					"delay_ms", (unsigned)DEFAULT_DELAY_MS);
+				if (delay_ms.value < 1)
+					delay_ms.value = DEFAULT_DELAY_MS;
+
+				_report_delay = Duration(delay_ms);
+				_schedule_report();
+				_reporter.enabled(true);
+			}
+		}
+		catch (...) { }
+
+		auto const parse_fn = [&] (Genode::Xml_node node) {
+			Url url;
+			Path path;
+			Url proxy;
 
 			try {
 				node.attribute("url").value(&url);
 				node.attribute("path").value(path.base(), path.capacity());
-		} catch (...) { Genode::error("error reading 'fetch' node"); return; }
+			}
+			catch (...) { Genode::error("error reading 'fetch' XML node"); return; }
 
-		char const *out_path = path.base();
+			try { config_node.attribute("proxy").value(&proxy); }
+			catch (...) { }
+
+			auto *f = new (_heap) Fetch(*this, url, path, proxy);
+			_fetches.insert(f);
+		};
+
+		config_node.for_each_sub_node("fetch", parse_fn);
+	}
+
+	Main(Libc::Env &e) : _env(e)
+	{
+		_env.config([&] (Genode::Xml_node const &config) {
+			parse_config(config);
+		});
+	}
+
+	CURLcode _process_fetch(CURL *_curl, Fetch &_fetch)
+	{
+		char const *out_path = _fetch.path.base();
 
 		/* create compound directories leading to the path */
 		for (size_t sub_path_len = 0; ; sub_path_len++) {
@@ -86,7 +198,7 @@ static int fetchurl(Genode::Xml_node config_node)
 			/* create directory for sub path */
 			if (mkdir(sub_path.string(), 0777) < 0) {
 				Genode::error("failed to create directory ", sub_path);
-				break;
+				return CURLE_FAILED_INIT;
 			}
 		}
 
@@ -104,62 +216,107 @@ static int fetchurl(Genode::Xml_node config_node)
 			default:
 				Genode::error("creation of ", out_path, " failed (errno=", errno, ")");
 			}
-			res = CURLE_FAILED_INIT;
+			return CURLE_FAILED_INIT;
-			return;
 		}
+		_fetch.fd = fd;
+
+		curl_easy_setopt(_curl, CURLOPT_URL, _fetch.url.string());
+		curl_easy_setopt(_curl, CURLOPT_FOLLOWLOCATION, true);
+
+		curl_easy_setopt(_curl, CURLOPT_NOSIGNAL, true);
+		curl_easy_setopt(_curl, CURLOPT_FAILONERROR, 1L);
+
+		curl_easy_setopt(_curl, CURLOPT_WRITEFUNCTION, write_callback);
+		curl_easy_setopt(_curl, CURLOPT_WRITEDATA, &_fetch);
+
+		curl_easy_setopt(_curl, CURLOPT_NOPROGRESS, 0L);
+		curl_easy_setopt(_curl, CURLOPT_PROGRESSFUNCTION, progress_callback);
+		curl_easy_setopt(_curl, CURLOPT_PROGRESSDATA, &_fetch);
+
+		curl_easy_setopt(_curl, CURLOPT_SSL_VERIFYPEER, 0L);
+		curl_easy_setopt(_curl, CURLOPT_SSL_VERIFYHOST, 0L);
+
+		/* check for optional proxy configuration */
+		if (_fetch.proxy != "") {
+			curl_easy_setopt(_curl, CURLOPT_PROXY, _fetch.proxy.string());
+		}
+
+		CURLcode res = curl_easy_perform(_curl);
+		close(_fetch.fd);
+		_fetch.fd = -1;
+
+		if (res != CURLE_OK)
+			Genode::error(curl_easy_strerror(res), ", failed to fetch ", _fetch.url);
+		return res;
+	}
+
+	int run()
+	{
+		CURLcode res = CURLE_OK;
 
 		CURL *curl = curl_easy_init();
 		if (!curl) {
 			Genode::error("failed to initialize libcurl");
-			res = CURLE_FAILED_INIT;
+			return -1;
-			return;
 		}
 
-		curl_easy_setopt(curl, CURLOPT_URL, url.string());
+		if (_reporter.enabled())
-		curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true);
+			_report();
 
-		curl_easy_setopt(curl, CURLOPT_VERBOSE, verbose);
+		for (Fetch *f = _fetches.first(); f; f = f->next()) {
-		curl_easy_setopt(curl, CURLOPT_NOSIGNAL, true);
+			if (res != CURLE_OK) break;
-		curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L);
+			res = _process_fetch(curl, *f);
+		}
 
-		curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback);
+		if (_reporter.enabled())
-		curl_easy_setopt(curl, CURLOPT_WRITEDATA, &fd);
+			_report();
-
-		curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L);
-
-		curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
-		curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
-
-		/* check for optional proxy configuration */
-		try {
-			Genode::String<256> proxy;
-			node.attribute("proxy").value(&proxy);
-			curl_easy_setopt(curl, CURLOPT_PROXY, proxy.string());
-		} catch (...) { }
-
-		Libc::with_libc([&]() {
-			res = curl_easy_perform(curl);
-			close(fd);
-
-			if (res != CURLE_OK)
-				Genode::error(curl_easy_strerror(res));
 
 		curl_easy_cleanup(curl);
-		});
-	});
-
-	curl_global_cleanup();
-
-	Genode::warning("SSL certificates not verified");
 
 		return res ^ CURLE_OK;
+	}
+};
+
+
+static size_t write_callback(char   *ptr,
+                             size_t  size,
+                             size_t  nmemb,
+                             void   *userdata)
+{
+	Fetchurl::Fetch &fetch = *((Fetchurl::Fetch *)userdata);
+	return write(fetch.fd, ptr, size*nmemb);
 }
 
+
+static int progress_callback(void *userdata,
+                             double dltotal, double dlnow,
+                             double ultotal, double ulnow)
+{
+	(void)ultotal;
+	(void)ulnow;
+
+	Fetchurl::Fetch &fetch = *((Fetchurl::Fetch *)userdata);
+	fetch.dltotal = dltotal;
+	fetch.dlnow = dlnow;
+	fetch.main._schedule_report();
+	return CURLE_OK;
+}
+
+
 void Libc::Component::construct(Libc::Env &env)
 {
-	env.config([&env] (Genode::Xml_node config) {
+	int res = -1;
-		env.parent().exit( fetchurl(config) );
+
+	Libc::with_libc([&]() {
+		curl_global_init(CURL_GLOBAL_DEFAULT);
+
+		static Fetchurl::Main inst(env);
+		res = inst.run();
+
+		curl_global_cleanup();
 	});
+
+	env.parent().exit(res);
 }
 
 /* dummies to prevent warnings printed by unimplemented libc functions */