ExternalVendorCode/genode
1
0
Fork 0
mirror of https://github.com/genodelabs/genode.git synced 2024-12-21 14:37:50 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

depot_download,depot_query: support system images

Browse Source 
This patch enhances the depot_download subsystem with support for
downloading and querying system images.

The installation ROM support the following two now download types:

  <image_index path="<user>/image/index"/>
  <image       path="<user>/image/<name>"/>

Internally, the depot-download subsystem employs the depot-query
component to determine the missing depot content. This component
accepts the following two new queries:

  <images      user="..."/>
  <image_index user="..."/>

If present in the query, depot_query generates reports labeled as
"images" and "image_index" respectively.

The also tracks the completion of each job depending on the depot-
query results, so that the final report contains a result for each
installation item requested. Prior this patch, the inactivity of the
depot-download manager (indicated by an empty state report) was
interpreted as success. But that prevents the proper association of
results and requested installation items.

Issue #4744
This commit is contained in:
Norman Feske 2023-01-23 16:42:46 +01:00 committed by Christian Helmuth
parent b3bcab6c13
commit 677c8e828c
11 changed files with 397 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
repos/gems/include/depot/archive.h
View File

@ -30,7 +30,7 @@ struct Depot::Archive
typedef String<80> Name;
typedef String<40> Version;
enum Type { PKG, RAW, SRC };
enum Type { PKG, RAW, SRC, IMAGE };
struct Unknown_archive_type : Exception { };
@ -81,9 +81,10 @@ struct Depot::Archive
typedef String<8> Name;
Name const name = _path_element<Name>(path, 1);
if (name == "src") return SRC;
if (name == "pkg") return PKG;
if (name == "raw") return RAW;
if (name == "src") return SRC;
if (name == "pkg") return PKG;
if (name == "raw") return RAW;
if (name == "image") return IMAGE;
throw Unknown_archive_type();
}
@ -96,7 +97,23 @@ struct Depot::Archive
return _path_element<Name>(path, 1) == "index";
}
static Name name (Path const &path) { return _path_element<Name>(path, 2); }
/**
* Return true if 'path' refers to a system-image index file
*/
static bool image_index(Path const &path)
{
return _path_element<Name>(path, 1) == "image" && name(path) == "index";
}
/**
* Return true if 'path' refers to a system image
*/
static bool image(Path const &path)
{
return _path_element<Name>(path, 1) == "image" && name(path) != "index";
}
static Name name (Path const &path) { return _path_element<Name> (path, 2); }
static Version version (Path const &path) { return _path_element<Version>(path, 3); }
static Version index_version(Path const &path) { return _path_element<Version>(path, 2); }
@ -108,10 +125,9 @@ struct Depot::Archive
*/
static Archive::Path download_file_path(Archive::Path path)
{
return Archive::index(path) ? Archive::Path(path, ".xz")
: Archive::Path(path, ".tar.xz");
return (index(path) || image_index(path)) ? Path(path, ".xz")
: Path(path, ".tar.xz");
}
};
#endif /* _INCLUDE__DEPOT__ARCHIVE_H_ */

6
repos/gems/recipes/raw/depot_download/depot_download.config
View File

@ -28,6 +28,10 @@
report="dynamic -> depot_query -> dependencies"/>
<policy label="manager -> index"
report="dynamic -> depot_query -> index"/>
<policy label="manager -> image"
report="dynamic -> depot_query -> image"/>
<policy label="manager -> image_index"
report="dynamic -> depot_query -> image_index"/>
<policy label="manager -> user"
report="dynamic -> depot_query -> user"/>
<policy label="manager -> init_state"
@ -85,6 +89,8 @@
<service name="Report"> <child name="report_rom"/> </service>
<service name="ROM" label="dependencies"> <child name="report_rom"/> </service>
<service name="ROM" label="index"> <child name="report_rom"/> </service>
<service name="ROM" label="image"> <child name="report_rom"/> </service>
<service name="ROM" label="image_index"> <child name="report_rom"/> </service>
<service name="ROM" label="user"> <child name="report_rom"/> </service>
<service name="ROM" label="init_state"> <child name="report_rom"/> </service>
<service name="ROM" label="verified"> <child name="report_rom"/> </service>

47
repos/gems/src/app/depot_download_manager/gen_depot_query.cc
View File

@ -51,11 +51,14 @@ void Depot_download_manager::gen_depot_query_start_content(Xml_generator &xml,
return failed;
};
installation.for_each_sub_node("archive", [&] (Xml_node archive) {
if (job_failed(archive))
return;
auto for_each_install_sub_node = [&] (auto node_type, auto const &fn)
{
installation.for_each_sub_node(node_type, [&] (Xml_node node) {
if (!job_failed(node))
fn(node); });
};
for_each_install_sub_node("archive", [&] (Xml_node const &archive) {
xml.node("dependencies", [&] () {
xml.attribute("path", archive.attribute_value("path", Archive::Path()));
xml.attribute("source", archive.attribute_value("source", true));
@ -63,22 +66,40 @@ void Depot_download_manager::gen_depot_query_start_content(Xml_generator &xml,
});
});
installation.for_each_sub_node("index", [&] (Xml_node index) {
if (job_failed(index))
for_each_install_sub_node("index", [&] (Xml_node const &index) {
Archive::Path const path = index.attribute_value("path", Archive::Path());
if (!Archive::index(path)) {
warning("malformed index path '", path, "'");
return;
}
xml.node("index", [&] () {
Archive::Path const path = index.attribute_value("path", Archive::Path());
if (!Archive::index(path)) {
warning("malformed index path '", path, "'");
return;
}
xml.attribute("user", Archive::user(path));
xml.attribute("version", Archive::_path_element<Archive::Version>(path, 2));
});
});
for_each_install_sub_node("image", [&] (Xml_node const &image) {
Archive::Path const path = image.attribute_value("path", Archive::Path());
if (!Archive::image(path)) {
warning("malformed image path '", path, "'");
return;
}
xml.node("image", [&] () {
xml.attribute("user", Archive::user(path));
xml.attribute("name", Archive::name(path));
});
});
for_each_install_sub_node("image_index", [&] (Xml_node const &image_index) {
Archive::Path const path = image_index.attribute_value("path", Archive::Path());
if (!Archive::index(path) && Archive::name(path) != "index") {
warning("malformed image-index path '", path, "'");
return;
}
xml.node("image_index", [&] () {
xml.attribute("user", Archive::user(path)); });
});
if (next_user.valid())
xml.node("user", [&] () { xml.attribute("name", next_user); });
});

3
repos/gems/src/app/depot_download_manager/gen_extract.cc
View File

@ -65,6 +65,9 @@ void Depot_download_manager::gen_extract_start_content(Xml_generator &xml,
if (Archive::index(path))
xml.attribute("name", Archive::index_version(path));
if (Archive::image_index(path))
xml.attribute("name", "index");
});
});
});

87
repos/gems/src/app/depot_download_manager/import.h
View File

@ -125,8 +125,70 @@ class Depot_download_manager::Import
return result;
}
static Archive::Path _depdendency_path(Xml_node const &item)
{
return item.attribute_value("path", Archive::Path());
}
static Archive::Path _index_path(Xml_node const &item)
{
return Path(item.attribute_value("user", Archive::User()), "/index/",
item.attribute_value("version", Archive::Version()));
}
static Archive::Path _image_path(Xml_node const &item)
{
return Path(item.attribute_value("user", Archive::User()), "/image/",
item.attribute_value("name", Archive::Name()));
}
static Archive::Path _image_index_path(Xml_node const &item)
{
return Path(item.attribute_value("user", Archive::User()), "/image/index");
}
template <typename FN>
static void _for_each_missing_depot_path(Xml_node const &dependencies,
Xml_node const &index,
Xml_node const &image,
Xml_node const &image_index,
FN const &fn)
{
dependencies.for_each_sub_node("missing", [&] (Xml_node const &item) {
fn(_depdendency_path(item)); });
index.for_each_sub_node("missing", [&] (Xml_node const &item) {
fn(_index_path(item)); });
image.for_each_sub_node("missing", [&] (Xml_node const &item) {
fn(_image_path(item)); });
image_index.for_each_sub_node("missing", [&] (Xml_node const &item) {
fn(_image_index_path(item)); });
}
public:
template <typename FN>
static void for_each_present_depot_path(Xml_node const &dependencies,
Xml_node const &index,
Xml_node const &image,
Xml_node const &image_index,
FN const &fn)
{
dependencies.for_each_sub_node("present", [&] (Xml_node const &item) {
fn(_depdendency_path(item)); });
index.for_each_sub_node("index", [&] (Xml_node const &item) {
fn(_index_path(item)); });
image.for_each_sub_node("image", [&] (Xml_node const &item) {
fn(_image_path(item)); });
image_index.for_each_sub_node("present", [&] (Xml_node const &item) {
fn(_image_index_path(item)); });
}
/**
* Constructor
*
@ -139,26 +201,17 @@ class Depot_download_manager::Import
* a future iteration.
*/
Import(Allocator &alloc, Archive::User const &user,
Xml_node dependencies, Xml_node index)
Xml_node const &dependencies,
Xml_node const &index,
Xml_node const &image,
Xml_node const &image_index)
:
_alloc(alloc)
{
dependencies.for_each_sub_node("missing", [&] (Xml_node item) {
Archive::Path const path = item.attribute_value("path", Archive::Path());
if (Archive::user(path) == user)
new (alloc) Item(_items, path);
});
index.for_each_sub_node("missing", [&] (Xml_node item) {
Archive::Path const
path(item.attribute_value("user", Archive::User()),
"/index/",
item.attribute_value("version", Archive::Version()));
if (Archive::user(path) == user)
new (alloc) Item(_items, path);
});
_for_each_missing_depot_path(dependencies, index, image, image_index,
[&] (Archive::Path const &path) {
if (Archive::user(path) == user)
new (alloc) Item(_items, path); });
}
~Import()

1
repos/gems/src/app/depot_download_manager/job.h
View File

@ -30,6 +30,7 @@ struct Depot_download_manager::Job : List_model<Job>::Element
{
bool started = false;
bool failed = false;
bool done = false;
Archive::Path const path;

58
repos/gems/src/app/depot_download_manager/main.cc
View File

@ -60,6 +60,8 @@ struct Depot_download_manager::Main : Import::Download_progress
Attached_rom_dataspace _installation { _env, "installation" };
Attached_rom_dataspace _dependencies { _env, "dependencies" };
Attached_rom_dataspace _index { _env, "index" };
Attached_rom_dataspace _image { _env, "image" };
Attached_rom_dataspace _image_index { _env, "image_index" };
Attached_rom_dataspace _init_state { _env, "init_state" };
Attached_rom_dataspace _fetchurl_progress { _env, "fetchurl_progress" };
@ -143,15 +145,18 @@ struct Depot_download_manager::Main : Import::Download_progress
else {
_jobs.for_each([&] (Job const &job) {
if (!job.started)
if (!job.started && !job.done)
return;
/*
* If a job has been started and has not failed, it must
* have succeeded at the time when the import is finished.
*/
char const *type = Archive::index(job.path) ? "index" : "archive";
xml.node(type, [&] () {
auto type = [] (Archive::Path const &path)
{
if (Archive::index(path)) return "index";
if (Archive::image(path)) return "image";
if (Archive::image_index(path)) return "image_index";
return "archive";
};
xml.node(type(job.path), [&] () {
xml.attribute("path", job.path);
xml.attribute("state", job.failed ? "failed" : "done");
});
@ -259,6 +264,8 @@ struct Depot_download_manager::Main : Import::Download_progress
{
_dependencies .sigh(_query_result_handler);
_index .sigh(_query_result_handler);
_image .sigh(_query_result_handler);
_image_index .sigh(_query_result_handler);
_current_user .sigh(_query_result_handler);
_init_state .sigh(_init_state_handler);
_verified .sigh(_init_state_handler);
@ -353,6 +360,8 @@ void Depot_download_manager::Main::_handle_query_result()
_dependencies.update();
_index.update();
_image.update();
_image_index.update();
_current_user.update();
/* validate completeness of depot-user info */
@ -385,14 +394,21 @@ void Depot_download_manager::Main::_handle_query_result()
Xml_node const dependencies = _dependencies.xml();
Xml_node const index = _index.xml();
Xml_node const image = _image.xml();
Xml_node const image_index = _image_index.xml();
if (dependencies.num_sub_nodes() == 0 && index.num_sub_nodes() == 0)
return;
/* mark jobs referring to existing depot content as unneccessary */
Import::for_each_present_depot_path(dependencies, index, image, image_index,
[&] (Archive::Path const &path) {
_jobs.for_each([&] (Job &job) {
if (job.path == path)
job.done = true; }); });
bool const missing_dependencies = dependencies.has_sub_node("missing");
bool const missing_index_files = index.has_sub_node("missing");
if (!missing_dependencies && !missing_index_files) {
bool const complete = !dependencies.has_sub_node("missing")
&& !index .has_sub_node("missing")
&& !image .has_sub_node("missing")
&& !image_index .has_sub_node("missing");
if (complete) {
log("installation complete.");
_update_state_report();
return;
@ -408,9 +424,16 @@ void Depot_download_manager::Main::_handle_query_result()
{
Archive::User user { };
if (missing_index_files)
index.with_optional_sub_node("missing", [&] (Xml_node missing) {
user = missing.attribute_value("user", Archive::User()); });
auto assign_user_from_missing_xml_sub_node = [&] (Xml_node const node)
{
if (!user.valid())
node.with_optional_sub_node("missing", [&] (Xml_node missing) {
user = missing.attribute_value("user", Archive::User()); });
};
assign_user_from_missing_xml_sub_node(index);
assign_user_from_missing_xml_sub_node(image);
assign_user_from_missing_xml_sub_node(image_index);
if (user.valid())
return user;
@ -435,7 +458,8 @@ void Depot_download_manager::Main::_handle_query_result()
}
/* start new import */
_import.construct(_heap, _current_user_name(), dependencies, index);
_import.construct(_heap, _current_user_name(),
dependencies, index, image, image_index);
/* mark imported jobs as started */
_import->for_each_download([&] (Archive::Path const &path) {

21
repos/gems/src/app/depot_query/main.cc
View File

@ -64,6 +64,9 @@ Depot_query::Main::_find_rom_in_pkg(File_content const &archives,
result = result_from_pkg;
});
break;
case Archive::IMAGE:
break;
}
});
return result;
@ -208,6 +211,7 @@ void Depot_query::Main::_collect_source_dependencies(Archive::Path const &path,
}
case Archive::RAW:
case Archive::IMAGE:
break;
};
}
@ -243,6 +247,9 @@ void Depot_query::Main::_collect_binary_dependencies(Archive::Path const &path,
case Archive::RAW:
dependencies.record(path);
break;
case Archive::IMAGE:
break;
};
}
@ -377,6 +384,20 @@ void Depot_query::Main::_query_index(Archive::User const &user,
}
void Depot_query::Main::_query_image(Archive::User const &user,
Archive::Name const &name,
Xml_generator &xml)
{
Directory::Path const image_path("depot/", user, "/image/", name);
char const *node_type = _root.directory_exists(image_path)
? "image" : "missing";
xml.node(node_type, [&] () {
xml.attribute("user", user);
xml.attribute("name", name);
});
}
void Component::construct(Genode::Env &env)
{
static Depot_query::Main main(env);

35
repos/gems/src/app/depot_query/main.h
View File

@ -267,6 +267,8 @@ struct Depot_query::Main
Constructible_reporter _dependencies_reporter { };
Constructible_reporter _user_reporter { };
Constructible_reporter _index_reporter { };
Constructible_reporter _image_reporter { };
Constructible_reporter _image_index_reporter { };
template <typename T, typename... ARGS>
static void _construct_if(bool condition, Constructible<T> &obj, ARGS &&... args)
@ -341,6 +343,8 @@ struct Depot_query::Main
void _gen_index_node_rec(Xml_generator &, Xml_node const &, unsigned) const;
void _gen_index_for_arch(Xml_generator &, Xml_node const &) const;
void _query_index(Archive::User const &, Archive::Version const &, bool, Xml_generator &);
void _query_image(Archive::User const &, Archive::Name const &, Xml_generator &);
void _query_image_index(Xml_node const &, Xml_generator &);
void _handle_config()
{
@ -371,9 +375,6 @@ struct Depot_query::Main
Xml_node const query = (query_from_rom ? _query_rom->xml() : config);
_construct_if(query.has_sub_node("scan"),
_scan_reporter, _env, "scan", "scan");
/*
* Use 64 KiB as initial report size to avoid the repetitive querying
* when successively expanding the reporter.
@ -382,14 +383,18 @@ struct Depot_query::Main
_blueprint_reporter, _env, "blueprint", "blueprint",
Expanding_reporter::Initial_buffer_size { 64*1024 });
_construct_if(query.has_sub_node("dependencies"),
_dependencies_reporter, _env, "dependencies", "dependencies");
auto construct_reporter_if_needed = [&] (auto &reporter, auto query_type)
{
_construct_if(query.has_sub_node(query_type),
reporter, _env, query_type, query_type);
};
_construct_if(query.has_sub_node("user"),
_user_reporter, _env, "user", "user");
_construct_if(query.has_sub_node("index"),
_index_reporter, _env, "index", "index");
construct_reporter_if_needed(_scan_reporter, "scan");
construct_reporter_if_needed(_dependencies_reporter, "dependencies");
construct_reporter_if_needed(_user_reporter, "user");
construct_reporter_if_needed(_index_reporter, "index");
construct_reporter_if_needed(_image_reporter, "image");
construct_reporter_if_needed(_image_index_reporter, "image_index");
_root.apply_config(config.sub_node("vfs"));
@ -453,6 +458,16 @@ struct Depot_query::Main
node.attribute_value("version", Archive::Version()),
node.attribute_value("content", false),
xml); }); });
_gen_versioned_report(_image_reporter, version, [&] (Xml_generator &xml) {
query.for_each_sub_node("image", [&] (Xml_node node) {
_query_image(node.attribute_value("user", Archive::User()),
node.attribute_value("name", Archive::Name()),
xml); }); });
_gen_versioned_report(_image_index_reporter, version, [&] (Xml_generator &xml) {
query.for_each_sub_node("image_index", [&] (Xml_node node) {
_query_image_index(node, xml); }); });
}
Main(Env &env) : _env(env)

171
repos/gems/src/app/depot_query/query_image_index.cc Normal file
View File

@ -0,0 +1,171 @@
/*
* \brief Querying system-image information from a depot
* \author Norman Feske
* \date 2023-01-26
*/
/*
* Copyright (C) 2023 Genode Labs GmbH
*
* This file is part of the Genode OS framework, which is distributed
* under the terms of the GNU Affero General Public License version 3.
*/
/* Genode includes */
#include <os/buffered_xml.h>
/* local includes */
#include <main.h>
void Depot_query::Main::_query_image_index(Xml_node const &index_query,
Xml_generator &xml)
{
using User = Archive::User;
using Version = String<16>;
using Os = String<16>;
using Board = String<32>;
User const user = index_query.attribute_value("user", User());
Os const os = index_query.attribute_value("os", Os());
Board const board = index_query.attribute_value("board", Board());
struct Version_reverse : Version
{
using Version::Version;
bool operator > (Version_reverse const &other) const
{
return strcmp(string(), other.string()) < 0;
}
};
struct Image_info;
using Image_dict = Dictionary<Image_info, Version_reverse>;
struct Image_info : Image_dict::Element
{
Constructible<Buffered_xml> from_index { };
enum Presence { PRESENT, ABSENT } const presence;
Image_info(Image_dict &dict, Version_reverse const &version, Presence presence)
: Image_dict::Element(dict, version), presence(presence) { }
void generate(Xml_generator &xml) const
{
xml.node("image", [&] {
xml.attribute("version", name);
if (presence == PRESENT)
xml.attribute("present", "yes");
if (!from_index.constructed())
return;
from_index->xml().for_each_sub_node("info", [&] (Xml_node const &info) {
using Text = String<160>;
Text const text = info.attribute_value("text", Text());
if (text.valid())
xml.node("info", [&] {
xml.attribute("text", text); });
});
});
}
};
Image_dict images { };
Directory::Path const prefix(os, "-", board, "-");
/* return version part of the image-file name */
auto version_from_name = [&prefix] (auto name)
{
size_t const prefix_chars = prefix.length() - 1;
if (strcmp(prefix.string(), name.string(), prefix_chars))
return Version_reverse(); /* prefix mismatch */
return Version_reverse(name.string() + prefix_chars);
};
Directory::Path const image_path("depot/", user, "/image");
if (_root.directory_exists(image_path)) {
Directory(_root, image_path).for_each_entry([&] (Directory::Entry const &entry) {
Directory::Entry::Name const name = entry.name();
Version_reverse const version = version_from_name(name);
if (entry.dir() && version.length() > 1)
new (_heap) Image_info { images, version, Image_info::PRESENT };
});
}
/*
* Supplement information found in the index file, if present
*/
Directory::Path const index_path("depot/", user, "/image/index");
bool index_exists = _root.file_exists(index_path);
if (index_exists) {
try {
File_content const
file(_heap, _root, index_path, File_content::Limit{16*1024});
file.xml([&] (Xml_node node) {
node.for_each_sub_node("image", [&] (Xml_node const &image) {
bool const os_and_board_match =
(image.attribute_value("os", Os()) == os) &&
(image.attribute_value("board", Board()) == board);
if (!os_and_board_match)
return;
Version_reverse const version {
image.attribute_value("version", Version()).string() };
if (!images.exists(version))
new (_heap) Image_info(images, version, Image_info::ABSENT);
images.with_element(version,
[&] (Image_info &info) {
info.from_index.construct(_heap, image); },
[&] () { }
);
});
});
}
catch (Directory::Nonexistent_file) {
index_exists = false;
}
}
/*
* Give feedback to depot_download_manager about the absence of the index
* file.
*/
xml.node(index_exists ? "present" : "missing", [&] () {
xml.attribute("user", user); });
/*
* Report aggregated image information with the newest version first.
*/
xml.node("user", [&] () {
xml.attribute("name", user);
xml.attribute("os", os);
xml.attribute("board", board);
images.for_each([&] (Image_info const &info) {
info.generate(xml); });
});
auto destroy_image_info = [&] (Image_info &info) { destroy(_heap, &info); };
while (images.with_any_element(destroy_image_info));
}

2
repos/gems/src/app/depot_query/target.mk
View File

@ -1,4 +1,4 @@
TARGET := depot_query
SRC_CC := main.cc
SRC_CC := main.cc query_image_index.cc
LIBS += base vfs
INC_DIR += $(REP_DIR)/src/app/fs_query $(PRG_DIR)