ExternalVendorCode/genode
1
0
Fork 0
mirror of https://github.com/genodelabs/genode.git synced 2025-02-20 17:52:52 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

New depot_remove component

Browse Source 
The depot_remove component can delete PKG archives with
automatically resolving dependencies and deleting archives that are not
required on the system anymore.

Issue genodelabs#4866
This commit is contained in:
Alice Domage 2023-03-14 14:18:13 +01:00 committed by Norman Feske
parent 40338f9acb
commit fb0e8fffa2
8 changed files with 670 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
repos/gems/recipes/src/depot_remove/content.mk Normal file
View File

@ -0,0 +1,10 @@
SRC_DIR := src/app/depot_remove
include $(GENODE_DIR)/repos/base/recipes/src/content.inc
MIRROR_FROM_REP_DIR := include/depot
content: $(MIRROR_FROM_REP_DIR)
$(MIRROR_FROM_REP_DIR):
$(mirror_from_rep_dir)

1
repos/gems/recipes/src/depot_remove/hash Normal file
View File

@ -0,0 +1 @@
2023-05-09T1814 a46d237336206b23977c7e447cec72c1c49e0799

4
repos/gems/recipes/src/depot_remove/used_apis Normal file
View File

@ -0,0 +1,4 @@
base
os
vfs
report_session

284
repos/gems/run/depot_remove.run Normal file
View File

@ -0,0 +1,284 @@
assert_spec linux
assert_spec x86_64
create_boot_directory
import_from_depot [depot_user]/src/[base_src] \
[depot_user]/src/report_rom \
[depot_user]/src/vfs \
[depot_user]/src/lx_fs \
[depot_user]/src/vfs_import \
[depot_user]/src/init
create_tar_from_depot_binaries [run_dir]/genode/depot.tar \
[depot_user]/pkg/chroot \
[depot_user]/pkg/system_shell \
[depot_user]/pkg/fonts_fs \
[depot_user]/pkg/wm \
[depot_user]/pkg/nano3d \
[depot_user]/pkg/window_layouter \
[depot_user]/pkg/motif_decorator \
[depot_user]/pkg/themed_decorator \
[depot_user]/pkg/sticks_blue_backdrop
if { [get_cmd_switch --autopilot] } {
import_from_depot [depot_user]/src/depot_remove
} else {
build { app/depot_remove }
}
install_config {
<config>
<parent-provides>
<service name="ROM"/>
<service name="IRQ"/>
<service name="IO_MEM"/>
<service name="IO_PORT"/>
<service name="PD"/>
<service name="RM"/>
<service name="CPU"/>
<service name="LOG"/>
</parent-provides>
<default-route>
<any-service> <parent/> <any-child/> </any-service>
</default-route>
<default caps="100"/>
<start name="timer">
<resource name="RAM" quantum="1M"/>
<provides> <service name="Timer"/> </provides>
</start>
<start name="report_rom">
<resource name="RAM" quantum="100M"/>
<provides> <service name="Report"/> <service name="ROM"/> </provides>
<config verbose="yes"/>
</start>
<start name="lx_fs" ld="no">
<resource name="RAM" quantum="100M"/>
<provides> <service name="File_system"/> </provides>
<config>
<policy label="depot_remove -> " root="/depot" writeable="yes"/>
</config>
</start>
<start name="depot_remove">
<resource name="RAM" quantum="2M"/>
<route>
<service name="ROM" label="config"> <parent label="depot_remove_config"/> </service>
<service name="File_system"> <child name="lx_fs"/> </service>
<any-service> <parent/> <any-child/> </any-service>
</route>
</start>
</config>}
exec mkdir [run_dir]/genode/depot
exec tar xvf [run_dir]/genode/depot.tar -C [run_dir]/genode/depot
exec chmod -R +r [run_dir]/genode/depot
if { [get_cmd_switch --autopilot] } {
build_boot_image {}
} else {
build_boot_image [build_artifacts]
}
proc install_test_config { args } {
set fd [open [run_dir]/genode/depot_remove_config w]
puts $fd "[join $args {}]"
close $fd
}
proc depot_state { } {
set archives_to_keep {}
foreach dir [glob -type d [run_dir]/genode/depot/[depot_user]/pkg/*] {
set pkg_path [depot_user]/pkg/[file tail $dir]/[_current_depot_archive_version pkg [file tail $dir]]
set fd [open [run_dir]/genode/depot/$pkg_path/archives r]
set pkg_deps [split [string trim [read $fd]]]
close $fd
foreach dependency $pkg_deps {
set idx [lsearch -exact $archives_to_keep $dependency]
if { $idx == -1 } {
lappend archives_to_keep $dependency
}
}
lappend archives_to_keep $pkg_path
}
set context [dict create]
dict set context pkg ""
dict set context archives_to_delete {}
dict set context archives_to_keep $archives_to_keep
return $context
}
proc depot_state_for_pkg { archive } {
set archive_path [depot_user]/pkg/$archive/[_current_depot_archive_version pkg $archive]
# Dependencies of $archive are archives to delete
set fd [open [run_dir]/genode/depot/$archive_path/archives r]
set archives_to_delete [split [string trim [read $fd]]]
close $fd
set archives_to_keep {}
foreach dir [glob -type d [run_dir]/genode/depot/[depot_user]/pkg/*] {
if { [file tail $dir] != $archive } {
set pkg_path [depot_user]/pkg/[file tail $dir]/[_current_depot_archive_version pkg [file tail $dir]]
set fd [open [run_dir]/genode/depot/$pkg_path/archives r]
set pkg_deps [split [string trim [read $fd]]]
close $fd
foreach dependency $archives_to_delete {
set idx [lsearch -exact $pkg_deps $dependency]
if { $idx != -1 } {
lappend archives_to_keep $dependency
}
}
}
}
# Remove archive to keep from archive to delete
foreach dependency $archives_to_keep {
set idx [lsearch -exact $archives_to_delete $dependency]
set archives_to_delete [lreplace $archives_to_delete $idx $idx]
}
set context [dict create]
dict set context pkg "$archive_path"
dict set context archives_to_delete $archives_to_delete
dict set context archives_to_keep $archives_to_keep
return $context
}
proc check_depot_state { context } {
foreach archive [dict get $context archives_to_delete] {
regexp [_depot_archive_versioned_path_pattern] $archive dummy archive_user archive_type archive_name
if { $archive_type == "src" } {
set archive $archive_user/bin/x86_64/$archive_name/[_current_depot_archive_version src $archive_name]
}
if { [file isdirectory [run_dir]/genode/depot/$archive] } {
puts "ERROR: $archive is present but should has been deleted."
return 1
}
}
foreach archive [dict get $context archives_to_keep] {
regexp [_depot_archive_versioned_path_pattern] $archive dummy archive_user archive_type archive_name
if { $archive_type == "src" } {
set archive $archive_user/bin/x86_64/$archive_name/[_current_depot_archive_version src $archive_name]
}
if { ![file isdirectory [run_dir]/genode/depot/$archive] } {
puts "ERROR: $archive should still be there but it has been deleted."
return 1
}
}
if { [dict get $context pkg] != "" && [file isdirectory [run_dir]/genode/depot/[dict get $context pkg]] } {
puts "ERROR: [dict get $context pkg] is present but shoud have been deleted."
return 1
}
return 0
}
## TEST 1 --- Delete nano3d ---------------------------------------------------
set context [depot_state_for_pkg "nano3d"]
install_test_config {
<config arch="x86_64" report="yes">
<remove user="} [depot_user] {" pkg="nano3d"/>
<vfs> <dir name="depot"> <fs/> </dir> </vfs>
</config>
}
run_genode_until ".*</removed_archives>" 10
if { [check_depot_state $context] } {
puts " TEST 1 --- Delete nano3d -- ERROR"
exit 1
}
puts " TEST 1 --- Delete nano3d -- SUCCESS"
## TEST 2 --- Delete non existing archive -------------------------------------
set context [depot_state]
install_test_config {
<config arch="x86_64" report="yes">
<remove user="} [depot_user] {" pkg="nano3d"/>
<vfs> <dir name="depot"> <fs/> </dir> </vfs>
</config>
}
run_genode_until ".*<removed_archives/>" 10
if { [check_depot_state $context] } {
puts " TEST 2 --- Delete non existing archive -- ERROR"
exit 1
}
puts " TEST 2 --- Delete non existing archive -- SUCCESS"
## TEST 3 --- Delete a PKG archive with deps to keep --------------------------
set context [depot_state_for_pkg "fonts_fs"]
install_test_config {
<config arch="x86_64" report="yes">
<remove user="} [depot_user] {" pkg="fonts_fs"/>
<vfs> <dir name="depot"> <fs/> </dir> </vfs>
</config>
}
run_genode_until ".*</removed_archives>" 10
if { [check_depot_state $context] } {
puts " TEST 3 --- Delete a PKG archive with deps to keep --- ERROR"
exit 1
}
puts " TEST 3 --- Delete a PKG archive with deps to keep --- SUCCESS"
## TEST 4 --- Remove all, keep themed_decorator PKG --------------------------
set context [depot_state_for_pkg "themed_decorator"]
install_test_config {
<config arch="x86_64" report="yes">
<remove-all>
<keep user="} [depot_user] {" pkg="themed_decorator"/>
</remove-all>
<vfs> <dir name="depot"> <fs/> </dir> </vfs>
</config>
}
run_genode_until ".*</removed_archives>" 10
if { ![check_depot_state $context] } {
puts " TEST 4 --- Remove all, keep themed_decorator PKG --- ERROR"
exit 1
}
puts " TEST 4 --- Remove all, keep themed_decorator PKG --- SUCCESS"

73
repos/gems/src/app/depot_remove/README Normal file
View File

@ -0,0 +1,73 @@
This directory contains the depot_remove component. It can delete PKGs and
its dependencies from the depot. It operates by reading its configuration and
processes delete operations based on the provided rules. This component
listens for configuration changes and will reactivate if it has been updated.
Configuration
~~~~~~~~~~~~~
A typical configuration looks as follows.
! <config arch="x86_64" report="yes">
! <remove user="alice" pkg="nano3d"/>
! <remove user="bob" pkg="wm" version="2042-42-42"/>
! <remove-all>
! <keep user="alice" pkg="fonts_fs"/>
! </remove-all>
! </config>
The '<config>' node comes with two possible attributes.
:arch:
This a mandatory attribute used to identify the correct directory for binary
archives.
:report (default "no"):
This is an optional attribute. If set to "yes", a "removed_archives" report
is created, listing each deleted archive.
The '<remove>' node instructs the component to remove a PKG archive from the
depot. The '<remove-all>' node instructs the component to remove all PGK archives.
It can be combined with a '<keep>' node to instruct the component to keep a
given PKG archive while removing all the others.
The '<remove>' and '<keep>' nodes accept the following attributes.
:pkg:
This is a mandatory attribute that is used to identify the targeted PKG
archive.
:user:
This is an optional attribute to identify the depot user for the given pkg.
:version:
This is an optional attribute to identify the version for the given pkg.
Reporting
~~~~~~~~~
When activated, the component reports the following content.
! <removed_archives>
! <removed path="bob/bin/x86_64/wm/2023-04-11T1218"/>
! <removed path="alice/bin/x86_64/nano3d/2023-04-11T1218"/>
! <removed path="bob/bin/x86_64/themed_decorator/2023-04-11T1218"/>
! <removed path="bob/bin/x86_64/decorator/2023-04-11T1218"/>
! <removed path="bob/raw/wm/2020-06-21"/>
! <removed path="bob/raw/window_layouter/2020-02-19"/>
! <removed path="bob/pkg/wm/2023-04-11T1218"/>
! <removed path="alice/pkg/nano3d/2023-04-11T1218"/>
! </removed_archives>
Example
~~~~~~~
Please refer to the _gems/run/depot_remove.run_ script for a practical example
of using the _depot_remove_ component.

291
repos/gems/src/app/depot_remove/main.cc Normal file
View File

@ -0,0 +1,291 @@
/*
* \brief Tool for deleting packages from a depot and resolving unused dependencies
* \author Alice Domage
* \date 2023-06-14
*/
/*
* Copyright (C) 2023 Genode Labs GmbH
* Copyright (C) 2023 gapfruit AG
*
* This file is part of the Genode OS framework, which is distributed
* under the terms of the GNU Affero General Public License version 3.
*/
#include <base/attached_rom_dataspace.h>
#include <base/component.h>
#include <base/heap.h>
#include <base/registry.h>
#include <base/signal.h>
#include <depot/archive.h>
#include <os/reporter.h>
#include <os/vfs.h>
#include <util/reconstructible.h>
#include <util/xml_node.h>
#include <util/xml_generator.h>
namespace Depot_remove {
using namespace Genode;
struct Main;
class Archive_remover;
}
class Depot_remove::Archive_remover
{
public:
using Archive_path = Depot::Archive::Path;
using Registered_path = Genode::Registered_no_delete<Archive_path>;
using Path = Directory::Path;
private:
Allocator &_alloc;
String<32> const _arch;
Registry<Registered_path> _deleted_archives { };
Registry<Registered_path> _pkg_to_delete { };
Registry<Registered_path> _archive_to_delete { };
void _remove_directory(Directory &depot, Path path) const
{
Directory dir { depot, path };
Registry<Registered_path> dirent_files { };
dir.for_each_entry([&] (auto const &entry) {
if (entry.name() == ".." || entry.name() == ".")
return;
else if (entry.type() == Vfs::Directory_service::Dirent_type::DIRECTORY)
_remove_directory(depot, Directory::join(path, entry.name()));
else
/*
* Deleting file within the for_each_entry() confuses lx_fs dirent
* offset computation and some files, such as 'README', is consitently
* omitted, thus the unlink operation fails. Thus create a list
* to delete file out of the lambda.
*/
new (_alloc) Registered_path(dirent_files, Directory::join(path, entry.name())); });
dirent_files.for_each([&](Registered_path &sub_path) {
depot.unlink(sub_path);
destroy(_alloc, &sub_path); });
depot.unlink(path);
}
template<typename FUNC = void(Path const &)>
void _for_each_subdir(Directory &depot, Path const &parent_dir, FUNC fn)
{
Directory pkg { depot, parent_dir };
pkg.for_each_entry([&fn, &parent_dir](auto const &entry) {
if (entry.name() == ".." || entry.name() == ".")
return;
Path subdir_path { Directory::join(parent_dir, entry.name()) };
fn(subdir_path); });
}
template<typename FUNC = void(Path const &)>
void _for_each_pkg(Directory &depot, FUNC fn)
{
depot.for_each_entry([&](auto const &entry) {
Path pkg_path { entry.name(), "/pkg" };
if (depot.directory_exists(pkg_path)) {
_for_each_subdir(depot, pkg_path, [&] (Path const &pkg_path) {
_for_each_subdir(depot, pkg_path, [&] (Path const &pkg_version_path) {
fn(pkg_version_path); }); }); } });
}
void _autoremove_pkg_and_dependencies(Directory &depot)
{
/* collect all archive dependencies to delete */
_pkg_to_delete.for_each([&](Archive_path &elem) {
Path pkg_version_path { elem };
Path archive_file_path { Directory::join(pkg_version_path, "archives") };
File_content archives { _alloc, depot, archive_file_path, { 8192 } };
archives.for_each_line<Path>([&](auto const &dependency_path) {
if (Depot::Archive::type(dependency_path) == Depot::Archive::Type::PKG)
return;
new (_alloc) Registered_path(_archive_to_delete, dependency_path); });
_remove_directory(depot, pkg_version_path);
/* try to delete the parent if it is empty, if not empty the operation fails */
_remove_directory(depot, Genode::Directory::join(pkg_version_path, ".."));
new (_alloc) Registered_path(_deleted_archives, pkg_version_path); });
/* keep archive dependencies that are still referenced by another PKG */
_for_each_pkg(depot, [&](Path const &pkg_version_path) {
Path archive_file_path { Directory::join(pkg_version_path, "archives") };
File_content archives { _alloc, depot, archive_file_path, { 8192 } };
archives.for_each_line<Path>([&] (auto const &dependency_path) {
if (Depot::Archive::type(dependency_path) == Depot::Archive::Type::PKG)
return;
_archive_to_delete.for_each([&](Registered_path &path){
if (dependency_path == path)
destroy(_alloc, &path); }); }); });
/* delete archive dependencies */
_archive_to_delete.for_each([&](Archive_path &path) {
Path archive {};
if (Depot::Archive::type(path) == Depot::Archive::Type::SRC) {
archive = Directory::join(Depot::Archive::user(path), "bin");
archive = Directory::join(archive, _arch);
archive = Directory::join(archive, Depot::Archive::name(path));
archive = Directory::join(archive, Depot::Archive::version(path));
} else {
archive = path;
}
/* if directory does not exist, it might has been deleted before, return silently */
if (!depot.directory_exists(archive))
return;
_remove_directory(depot, archive);
new (_alloc) Registered_path(_deleted_archives, archive);
/* try to delete the parent if it is empty, if not empty the operation fails */
_remove_directory(depot, Directory::join(archive, "..")); });
}
static bool _config_node_match_pkg(Genode::Xml_node const &node, Path pkg)
{
if (!node.has_attribute("user"))
return false;
if (Depot::Archive::user(pkg) != node.attribute_value("user", Archive_path {}))
return false;
if (!node.has_attribute("pkg"))
return true;
if (Depot::Archive::name(pkg) != node.attribute_value("pkg", Archive_path {}))
return false;
if (!node.has_attribute("version"))
return true;
if (Depot::Archive::version(pkg) != node.attribute_value("version", Archive_path {}))
return false;
return true;
};
void _configure_remove_pkgs(Directory &depot, Xml_node const &config)
{
_for_each_pkg(depot, [&] (Path const &pkg_path) {
config.for_each_sub_node("remove", [&](Xml_node const &node) {
if (_config_node_match_pkg(node, pkg_path))
new (_alloc) Registered_path(_pkg_to_delete, pkg_path); }); });
}
void _configure_remove_all_pkgs(Directory &depot, Xml_node const &config)
{
_for_each_pkg(depot, [&] (Path const &pkg_path) {
bool keep = false;
config.for_each_sub_node("remove-all", [&](Xml_node const &remove_all_node) {
remove_all_node.for_each_sub_node("keep", [&](Xml_node const &node) {
if (_config_node_match_pkg(node, pkg_path))
keep = true; }); });
if (!keep)
new (_alloc) Registered_path(_pkg_to_delete, pkg_path); });
}
public:
void generate_report(Expanding_reporter &reporter) const
{
reporter.generate([&](Reporter::Xml_generator &xml) {
_deleted_archives.for_each([&] (auto &path) {
xml.node("removed", [&]() {
xml.attribute("path", path); }); }); });
}
Archive_remover(Allocator &alloc,
Directory &depot,
Xml_node const &config)
:
_alloc { alloc },
_arch { config.attribute_value("arch", String<32>()) }
{
if (config.has_sub_node("remove") && config.has_sub_node("remove-all")) {
warning("<remove/> and <remove-all/> are mutually exclusive");
return;
}
if (config.has_sub_node("remove")) _configure_remove_pkgs(depot, config);
if (config.has_sub_node("remove-all")) _configure_remove_all_pkgs(depot, config);
_autoremove_pkg_and_dependencies(depot);
}
~Archive_remover() {
_pkg_to_delete.for_each([this] (auto &elem) {
destroy(_alloc, &elem); });
_archive_to_delete.for_each([this] (auto &elem) {
destroy(_alloc, &elem); });
_deleted_archives.for_each([this] (auto &elem) {
destroy(_alloc, &elem); });
}
};
struct Depot_remove::Main
{
Env &_env;
Heap _heap { _env.ram(), _env.rm() };
Attached_rom_dataspace _config_rom { _env, "config" };
Signal_handler<Main> _config_handler { _env.ep(), *this, &Main::_handle_config };
Constructible<Expanding_reporter> _reporter { };
Main(Env &env)
:
_env { env },
_config_handler { env.ep(), *this, &Main::_handle_config }
{
_config_rom.sigh(_config_handler);
_handle_config();
}
void _handle_config()
{
_config_rom.update();
Xml_node const &config { _config_rom.xml() };
if (!config.has_attribute("arch")) {
warning("missing arch attribute");
return;
}
if (!config.has_sub_node("vfs")) {
warning("configuration misses a <vfs> configuration node");
return;
}
Directory::Path depot_path { "depot" };
Root_directory root_directory { _env, _heap, config.sub_node("vfs") };
Directory depot { root_directory, depot_path };
try {
Archive_remover archive_cleaner { _heap, depot, config };
_reporter.conditional(config.attribute_value("report", false),
_env, "removed_archives", "archive_list");
if (_reporter.constructed())
archive_cleaner.generate_report(*_reporter);
} catch (...) {
/* catch any exceptions to prevent the component to abort */
error("Depot autoclean job finished with error(s).");
}
}
};
void Component::construct(Genode::Env &env)
{
static Depot_remove::Main main(env);
}

6
repos/gems/src/app/depot_remove/target.mk Normal file
View File

@ -0,0 +1,6 @@
TARGET := depot_remove
SRC_CC := main.cc
LIBS := base vfs

1
tool/autopilot.list
View File

@ -9,6 +9,7 @@ demo
depot_autopilot
depot_download
depot_query
depot_remove
event_filter
extract
fb_bench