ExternalVendorCode/genode
1
0
Fork 0
mirror of https://github.com/genodelabs/genode.git synced 2025-02-21 02:01:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

depot_download_manager: optimize report processing

Browse Source 
Feed the high-frequent fetchurl reports into a list model to speed up
the subsequent evaluation of the data.

Also limit the rate of state reporting during updates to visible changes
in percent such that most fetchurl reports do not cause any UI activity
(status updating) any longer.

This patch noticeably speeds up the installation of complex packages
(i.e., the morph browser) on the PinePhone.

Issue #5038
This commit is contained in:
Norman Feske 2023-10-20 11:49:30 +02:00 committed by Christian Helmuth
parent 20239dc27f
commit b50a88a95d
2 changed files with 138 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

107
repos/gems/src/app/depot_download_manager/import.h
View File

@ -17,6 +17,7 @@
/* Genode includes */
#include <util/xml_node.h>
#include <util/xml_generator.h>
#include <util/list_model.h>
#include <base/registry.h>
#include <base/allocator.h>
@ -32,18 +33,55 @@ class Depot_download_manager::Import
{
public:
/**
* Interface for obtaining the download progress for a given archive
*/
struct Download_progress : Interface
struct Download : List_model<Download>::Element
{
struct Info
struct Progress
{
typedef String<32> Bytes;
Bytes total, now;
uint64_t total_bytes;
uint64_t downloaded_bytes;
bool complete() const
static Progress from_xml(Xml_node const &node)
{
return { .total_bytes = node.attribute_value("total", 0ULL),
.downloaded_bytes = node.attribute_value("now", 0ULL) };
}
void gen_attr(Xml_generator &xml) const
{
xml.attribute("total", total_bytes);
xml.attribute("now", downloaded_bytes);
}
unsigned percent() const
{
if (total_bytes == 0)
return 0;
return unsigned((downloaded_bytes*100)/total_bytes);
}
};
Url const url;
Progress progress { };
bool complete = false;
Download(Url const &url) : url(url) { };
static Url url_from_xml(Xml_node const &node)
{
return node.attribute_value("url", Url());
}
void update(Xml_node const &node)
{
auto complete_from_xml = [&]
{
using Bytes = String<32>;
Bytes const total = node.attribute_value("total", Bytes()),
now = node.attribute_value("now", Bytes());
/* fetchurl did not return valid download info */
if (total == "")
return false;
@ -53,10 +91,21 @@ class Depot_download_manager::Import
return false;
return now == total;
}
};
};
virtual Info download_progress(Archive::Path const &) const = 0;
progress = Progress::from_xml(node);
complete = complete_from_xml();
}
bool matches(Xml_node const &node) const
{
return url_from_xml(node) == url;
}
static bool type_matches(Xml_node const &node)
{
return node.has_type("fetch") && url_from_xml(node).valid();
}
};
private:
@ -80,6 +129,8 @@ class Depot_download_manager::Import
State state = DOWNLOAD_IN_PROGRESS;
Download::Progress progress { };
bool in_progress() const
{
return state == DOWNLOAD_IN_PROGRESS
@ -313,18 +364,30 @@ class Depot_download_manager::Import
});
}
void apply_download_progress(Download_progress const &progress)
void _with_downloading_item(Url const &current_user_url, Url const &url, auto const &fn)
{
_items.for_each([&] (Item &item) {
if (item.state != Item::DOWNLOAD_IN_PROGRESS)
return;
if (item.state == Item::DOWNLOAD_IN_PROGRESS
&& progress.download_progress(item.path).complete()) {
item.state = Item::DOWNLOAD_COMPLETE;
}
Url const item_url(current_user_url, "/", Archive::download_file_path(item.path));
if (url == item_url)
fn(item);
});
}
void download_complete(Url const &current_user_url, Url const &url)
{
_with_downloading_item(current_user_url, url, [&] (Item &item) {
item.state = Item::DOWNLOAD_COMPLETE; });
}
void download_progress(Url const &current_user_url, Url const &url, Download::Progress progress)
{
_with_downloading_item(current_user_url, url, [&] (Item &item) {
item.progress = progress; });
}
void all_remaining_downloads_unavailable()
{
_items.for_each([&] (Item &item) {
@ -355,19 +418,15 @@ class Depot_download_manager::Import
item.state = Item::UNPACKED; });
}
void report(Xml_generator &xml, Download_progress const &progress) const
void report(Xml_generator &xml) const
{
_items.for_each([&] (Item const &item) {
xml.node("archive", [&] () {
xml.attribute("path", item.path);
xml.attribute("state", item.state_text());
if (item.state == Item::DOWNLOAD_IN_PROGRESS) {
Download_progress::Info const info =
progress.download_progress(item.path);
xml.attribute("total", info.total);
xml.attribute("now", info.now);
}
if (item.state == Item::DOWNLOAD_IN_PROGRESS)
item.progress.gen_attr(xml);
});
});
}

84
repos/gems/src/app/depot_download_manager/main.cc
View File

@ -51,7 +51,7 @@ struct Depot_download_manager::Child_exit_state
};
struct Depot_download_manager::Main : Import::Download_progress
struct Depot_download_manager::Main
{
Env &_env;
@ -117,25 +117,6 @@ struct Depot_download_manager::Main : Import::Download_progress
Constructible<Import> _import { };
/**
* Download_progress interface
*/
Info download_progress(Archive::Path const &path) const override
{
Info result { Info::Bytes(), Info::Bytes() };
try {
Url const url_path(_current_user_url(), "/", Archive::download_file_path(path));
/* search fetchurl progress report for matching 'url_path' */
_fetchurl_progress.xml().for_each_sub_node("fetch", [&] (Xml_node fetch) {
if (fetch.attribute_value("url", Url()) == url_path)
result = { .total = fetch.attribute_value("total", Info::Bytes()),
.now = fetch.attribute_value("now", Info::Bytes()) }; });
} catch (Invalid_download_url) { }
return result;
}
void _update_state_report()
{
_state_reporter.generate([&] (Xml_generator &xml) {
@ -143,7 +124,7 @@ struct Depot_download_manager::Main : Import::Download_progress
/* produce detailed reports while the installation is in progress */
if (_import.constructed()) {
xml.attribute("progress", "yes");
_import->report(xml, *this);
_import->report(xml);
}
/* once all imports have settled, present the final results */
@ -209,22 +190,67 @@ struct Depot_download_manager::Main : Import::Download_progress
/* number of bytes downloaded by current fetchurl instance */
uint64_t _downloaded_bytes { };
using Download = Import::Download;
List_model<Download> _fetchurl_downloads { };
void _handle_fetchurl_progress()
{
_fetchurl_progress.update();
bool visible_progress = false;
update_list_model_from_xml(_fetchurl_downloads, _fetchurl_progress.xml(),
/* create */
[&] (Xml_node const &node) -> Download & {
visible_progress = true;
return *new (_heap) Download(Download::url_from_xml(node));
},
/* destroy */
[&] (Download &e) {
visible_progress = true;
destroy(_heap, &e);
},
/* update */
[&] (Download &download, Xml_node const &node)
{
unsigned const orig_percent = download.progress.percent();
bool const orig_complete = download.complete;
download.update(node);
bool const progress = (orig_percent != download.progress.percent())
|| (orig_complete != download.complete);
visible_progress |= progress;
if (_import.constructed()) {
try {
if (progress)
_import->download_progress(_current_user_url(),
download.url, download.progress);
if (download.complete)
_import->download_complete(_current_user_url(), download.url);
} catch (Invalid_download_url) { }
}
}
);
/* count sum of bytes downloaded by current fetchurl instance */
_downloaded_bytes = 0;
_fetchurl_progress.xml().for_each_sub_node("fetch", [&] (Xml_node fetch) {
_downloaded_bytes += fetch.attribute_value("now", 0ULL); });
_fetchurl_downloads.for_each([&] (Download const &download) {
_downloaded_bytes += download.progress.downloaded_bytes; });
if (_import.constructed()) {
_import->apply_download_progress(*this);
if (!visible_progress)
return;
/* proceed with next import step if all downloads are done or failed */
if (!_import->downloads_in_progress())
_generate_init_config();
}
/* proceed with next import step if all downloads are done or failed */
if (_import.constructed() && !_import->downloads_in_progress())
_generate_init_config();
_update_state_report();
}