ExternalVendorCode/genode
1
0
Fork 0
mirror of https://github.com/genodelabs/genode.git synced 2025-04-26 05:49:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

depot_autopilot: provide repeat mode

Browse Source 
Adds an config attribute to the Depot Autopilot component:

:<config repeat>:

  Can be one of

    "false"         - process the given test list only once,
    "until_forever" - endlessly repeat processing the given test list,
    "until_failed"  - repeat processing the given test list until it fails.

Adds an environment variable to the Depot Autopilot Run script:

:TEST_REPEAT:

  Same as the <config repeat> attribute of the Depot Autopilot.

This is useful when having to debug very sporadic errors during one test
or a series of tests.
This commit is contained in:
Martin Stein 2018-12-19 15:07:46 +01:00 committed by Norman Feske
parent b7cb5839eb
commit f0842a27c5
7 changed files with 291 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

121
repos/gems/run/depot_autopilot.run
View File

@ -1,28 +1,8 @@
####################
## User interface ##
####################
# #
# Additional environment variables considered by this run script # Integration of Depot Autopilot into a Run-tool-based test infrastructure
# #
# TEST_PKGS list of test package-archives if set (default list below) # For a detailed documentation of the user interface and the features of this
# TEST_SRCS list of test source-archive if TEST_PKGS is set (default list below) # Run script please see _repos/gems/src/app/depot_autopilot/README_.
# TEST_BUILDS list of test build-targets that shall be build from repos
# TEST_MODULES list of test boot-modules to overlay the test depot-content
#
# To get a hint which build components and which boot modules you may want to
# enter for a given test package-archive, please see these files:
#
# repos/<REPO>/recipes/pkg/<TEST_PKG>/archives
# repos/<REPO>/recipes/pkg/<TEST_PKG>/runtime (<content> tag)
#
# Example:
#
# ! KERNEL="nova" \
# TEST_PKGS="test-libc_vfs test-log" \
# TEST_BUILDS="server/ram_fs test/libc_vfs" \
# TEST_MODULES="ram_fs test-libc_vfs vfs.lib.so" \
# make run/depot_autopilot
# #
@ -235,6 +215,7 @@ proc prepare_to_run_genode { } {
global test_srcs global test_srcs
global test_builds global test_builds
global test_modules global test_modules
global test_repeat
global last_test_pkg global last_test_pkg
global run_genode_failed global run_genode_failed
global serial_id global serial_id
@ -364,7 +345,7 @@ proc prepare_to_run_genode { } {
<start name="depot_autopilot" priority="-1"> <start name="depot_autopilot" priority="-1">
<resource name="RAM" quantum="2M"/> <resource name="RAM" quantum="2M"/>
<provides> <service name="LOG"/> </provides> <provides> <service name="LOG"/> </provides>
<config arch="} [depot_spec] {" children_label_prefix="dynamic -> "> <config repeat="} $test_repeat {" arch="} [depot_spec] {" children_label_prefix="dynamic -> ">
<static> <static>
<parent-provides> <parent-provides>
<service name="ROM"/> <service name="ROM"/>
@ -482,6 +463,55 @@ proc init_previous_results {} {
set previous_skipped 0 set previous_skipped 0
} }
#
# Initialize variables that can be configured via the user interface
#
proc init_test_setting {} {
global test_pkgs
global test_srcs
global test_builds
global test_modules
global test_repeat
global default_test_pkgs
global default_test_srcs
global skip_test_pkg
#
# Initialize the lists of test package-archives, source-archives,
# build-targets, and boot-modules to be considered by the run script
#
set test_pkgs [get_env_var TEST_PKGS ""]
set test_srcs [get_env_var TEST_SRCS ""]
set test_builds [get_env_var TEST_BUILDS ""]
set test_modules [get_env_var TEST_MODULES ""]
set test_repeat [get_env_var TEST_REPEAT "false"]
#
# If the user didn't declare an individual list of test package-archives, use
# the default lists of test package-archives and source-archives
#
set nr_of_tests_to_run 0
if {$test_pkgs == ""} {
foreach test_pkg $default_test_pkgs {
append test_pkgs " $test_pkg "
if {!([info exists skip_test_pkg($test_pkg)] && $skip_test_pkg($test_pkg))} {
incr nr_of_tests_to_run
}
}
foreach test_src_pkg $default_test_srcs {
append test_srcs " $test_src_pkg "
}
} else {
foreach test_pkg $test_pkgs {
if {!([info exists skip_test_pkg($test_pkg)] && $skip_test_pkg($test_pkg))} {
incr nr_of_tests_to_run
}
}
}
}
rename exit run_tool_exit rename exit run_tool_exit
proc exit {{status 0}} { proc exit {{status 0}} {
@ -677,40 +707,8 @@ if {[info exists qemu_args]} {
set initial_qemu_args $qemu_args set initial_qemu_args $qemu_args
} }
# # initialize global variables
# Initialize the lists of test package-archives, source-archives, init_test_setting
# build-targets, and boot-modules to be considered by the run script
#
set test_pkgs [get_env_var TEST_PKGS ""]
set test_srcs [get_env_var TEST_SRCS ""]
set test_builds [get_env_var TEST_BUILDS ""]
set test_modules [get_env_var TEST_MODULES ""]
#
# If the user didn't declare an individual list of test package-archives, use
# the default lists of test package-archives and source-archives
#
set nr_of_tests_to_run 0
if {$test_pkgs == ""} {
foreach test_pkg $default_test_pkgs {
append test_pkgs " $test_pkg "
if {!([info exists skip_test_pkg($test_pkg)] && $skip_test_pkg($test_pkg))} {
incr nr_of_tests_to_run
}
}
foreach test_src_pkg $default_test_srcs {
append test_srcs " $test_src_pkg "
}
} else {
foreach test_pkg $test_pkgs {
if {!([info exists skip_test_pkg($test_pkg)] && $skip_test_pkg($test_pkg))} {
incr nr_of_tests_to_run
}
}
}
puts "Number of tests to run: $nr_of_tests_to_run"
# initialize variables that hold previous results
init_previous_results init_previous_results
# generic preparation for each system boot # generic preparation for each system boot
@ -787,13 +785,16 @@ while {1} {
puts $last_test_result puts $last_test_result
exit -1 exit -1
} }
if {$previous_results != ""} { if {$previous_results != ""} {
append previous_results \012 append previous_results \012
} }
append previous_results $last_test_result append previous_results $last_test_result
} }
# if the Autopilot is currently repeating, reset repeat-influenced variables
if {[llength $test_pkgs] == 0} {
init_test_setting
init_previous_results
}
# determine timeout for the next timeout # determine timeout for the next timeout
set min_timeout 10 set min_timeout 10
regexp {depot_autopilot\] --- Run "(.*?)" \(max ([0-9]*?) } $output] ignore last_test_pkg min_timeout regexp {depot_autopilot\] --- Run "(.*?)" \(max ([0-9]*?) } $output] ignore last_test_pkg min_timeout

98
repos/gems/src/app/depot_autopilot/README
View File

@ -20,22 +20,21 @@ provided:
platforms are comparable platforms are comparable
The Depot Autopilot is accompanied by the run script The Depot Autopilot is accompanied by the Run script
'repos/gems/run/depot_autopilot.run' which doesn't describe a classical test 'repos/gems/run/depot_autopilot.run' which doesn't describe a classical test
but rather helps integrating the Autopilot into a linux-based test but rather helps integrating the Autopilot into a Linux-based test
environment. The run script provides the following features: environment. The run script provides the following features:
* The user only lists the desired test packages and selectively defines the * Creates a complete Autopilot scenario from a list of local test packages
platform compatibility for tests where this is necessary * Optional platform dependencies can be defined for each test package
* The run script executes the Autopilot scenario on the target platform, * Simple interface for selectively using test ingredients built from source
repeadedly if necessary, until an overview with all tests can be presented * Can be combined with a list of source packages and gcov to show test coverage
* Each time the system gets booted, the Autopilot is configured only with * Exits with 0 if all tests are either "ok" or "skipped", otherwise with < 0
the remaining tests * The target is re-booted if the scenario gets stuck
* Tests that caused the run script to re-boot the target platform are listed * On each re-boot the Autopilot is configured only with the remaining tests
as "failed" with cause "reboot" in the overview * Tests that caused a re-boot are listed as "failed" with cause "reboot"
* Exits with 0 if a complete overview can be accomplished and all tests are * The final Autopilot result overview reflects all results from all boots
either "ok" or "skipped" and otherwise with a value < 0 * If a complete result overview can't be obtained, a partial one is simulated
* Provides a convenient interface for debugging single tests
Depot Autopilot component Depot Autopilot component
@ -46,7 +45,7 @@ Configuration
This is an example configuration of the Depot Autopilot: This is an example configuration of the Depot Autopilot:
! <config arch="x86_64" children_label_prefix="dynamic -> "> ! <config arch="x86_64" children_label_prefix="dynamic -> " repeat="false">
! <static> ! <static>
! <parent-provides> ! <parent-provides>
! <service name="ROM"/> ! <service name="ROM"/>
@ -79,11 +78,19 @@ This is an example configuration of the Depot Autopilot:
appropriate variant of the test packages. Must be one of "x86_64", "x86_32", appropriate variant of the test packages. Must be one of "x86_64", "x86_32",
"arm_v7a". "arm_v7a".
:config children_label_prefix: :<config children_label_prefix>:
Label prefix of LOG sessions of the components of a test. This is required Label prefix of LOG sessions of the components of a test. This is required
to relate incoming LOG-session request to a running test. to relate incoming LOG-session request to a running test.
:<config repeat>:
Can be one of
"false" - process the given test list only once,
"until_forever" - endlessly repeat processing the given test list,
"until_failed" - repeat processing the given test list until it fails.
:<config><static>: :<config><static>:
Contains Init configuration that shall always be added to the configuration Contains Init configuration that shall always be added to the configuration
@ -319,8 +326,65 @@ Integration
components as part of the LOG output of the Depot Autopilot. components as part of the LOG output of the Depot Autopilot.
Depot Autopilot Run script
~~~~~~~~~~~~~~~~~~~~~~~~~~
By default, the Run script tries to evaluate all tests that are internally
configured, that is, normally, all that are available as package recipes in
the main repositories. This includes one test that is prepared for the later
examination by gcov as well as gcov itself, which is executed like a test at
the end of the test list to examine the above mentioned adapted test. The test
list is processed only once and all test ingredients are used from packages,
assuming the depot user is 'genodelabs'. For this default behavior just call
the script like any "normal" Run script:
! make run/depot_autopilot KERNEL="nova"
However, this is normally a pretty time intensive process and meant for the
integration into automatic test infrastructures rather than manual evaluation.
This is why you might want to adapt the scenario, for instance, for debugging
one or more tests that failed during your automated testing. For this purpose,
a user interface is given through specific environment variables. This is an
example containing all of these variables:
! make run/depot_autopilot \
! > TEST_PKGS="test-libc_vfs test-log" \
! > TEST_SRCS="test-xml_generator test-xml_node" \
! > TEST_BUILDS="server/ram_fs test/libc_vfs" \
! > TEST_MODULES="ram_fs test-libc_vfs vfs.lib.so" \
! > TEST_REPEAT="until_failed" \
! > KERNEL="nova"
:TEST_PKGS:
List of test package-archives that supersedes the internal list if set.
:TEST_SRCS:
List of test source-archives that supersedes the internal list if set.
:TEST_BUILDS:
List of test build-targets that be build from the local repositories.
:TEST_MODULES:
List of test boot-modules that overlay the contents of the package-archives.
:TEST_REPEAT:
See the <config repeat> attribute of the Depot Autopilot.
To get a hint which build components (TEST_BUILDS) and which boot modules
(TEST_MODULES) you may want to enter for a given test package, you may have
a look at the package recipe:
! repos/<REPO>/recipes/pkg/<TEST_PKG>/archives
! repos/<REPO>/recipes/pkg/<TEST_PKG>/runtime (<content> tag)
Examples Examples
-------- --------
See the run script 'repos/gems/run/depot_autopilot.run' for a comprehensive Please see the run script 'repos/gems/run/depot_autopilot.run' for a
example of how to use the Depot Autopilot. comprehensive example of how to use the Depot Autopilot.

19
repos/gems/src/app/depot_autopilot/child.cc
View File

@ -375,6 +375,19 @@ Child::Child(Allocator &alloc,
{ } { }
Child::~Child()
{
while (Timeout_event *event = _timeout_events.first()) {
_timeout_events.remove(event);
destroy(_alloc, event);
}
while (Log_event *event = _log_events.first()) {
_log_events.remove(event);
destroy(_alloc, event);
}
}
void Child::log_session_write(Log_event::Line const &log_line) void Child::log_session_write(Log_event::Line const &log_line)
{ {
if (_skip) { if (_skip) {
@ -684,10 +697,14 @@ Child::State_name Child::_padded_state_name() const
} }
void Child::print_conclusion()
{
log(" ", _conclusion);
}
void Child::conclusion(Result &result) void Child::conclusion(Result &result)
{ {
struct Bad_state : Exception { }; struct Bad_state : Exception { };
log(" ", _conclusion);
switch (_state) { switch (_state) {
case SUCCEEDED: result.succeeded++; break; case SUCCEEDED: result.succeeded++; break;
case FAILED: result.failed++; break; case FAILED: result.failed++; break;

4
repos/gems/src/app/depot_autopilot/child.h
View File

@ -221,8 +221,12 @@ class Depot_deploy::Child : public List_model<Child>::Element
Timer::Connection &timer, Timer::Connection &timer,
Genode::Signal_context_capability const &config_handler); Genode::Signal_context_capability const &config_handler);
~Child();
void log_session_write(Log_event::Line const &log_line); void log_session_write(Log_event::Line const &log_line);
void print_conclusion();
void conclusion(Result &result); void conclusion(Result &result);
void event_occured(Event const &event, void event_occured(Event const &event,

7
repos/gems/src/app/depot_autopilot/children.h
View File

@ -158,6 +158,13 @@ class Depot_deploy::Children
return finished; return finished;
} }
void print_conclusion()
{
_children.for_each([&] (Child &child) {
child.print_conclusion();
});
}
void conclusion(Result &result) void conclusion(Result &result)
{ {
_children.for_each([&] (Child &child) { _children.for_each([&] (Child &child) {

9
repos/gems/src/app/depot_autopilot/config.xsd
View File

@ -4,6 +4,14 @@
<xs:include schemaLocation="base_types.xsd"/> <xs:include schemaLocation="base_types.xsd"/>
<xs:include schemaLocation="timeout_types.xsd"/> <xs:include schemaLocation="timeout_types.xsd"/>
<xs:simpleType name="Repeat">
<xs:restriction base="xs:string">
<xs:enumeration value="false" />
<xs:enumeration value="until_forever" />
<xs:enumeration value="until_failed" />
</xs:restriction>
</xs:simpleType><!-- Architecture -->
<xs:simpleType name="Architecture"> <xs:simpleType name="Architecture">
<xs:restriction base="xs:string"> <xs:restriction base="xs:string">
<xs:enumeration value="x86_64" /> <xs:enumeration value="x86_64" />
@ -91,6 +99,7 @@
</xs:choice> </xs:choice>
<xs:attribute name="arch" type="Architecture" /> <xs:attribute name="arch" type="Architecture" />
<xs:attribute name="repeat" type="Repeat" />
<xs:attribute name="children_label_prefix" type="Session_label" /> <xs:attribute name="children_label_prefix" type="Session_label" />
<xs:attribute name="ld_verbose" type="Boolean" /> <xs:attribute name="ld_verbose" type="Boolean" />
</xs:complexType> </xs:complexType>

180
repos/gems/src/app/depot_autopilot/main.cc
View File

@ -112,91 +112,133 @@ class Depot_deploy::Log_root : public Root_component<Log_session_component>
struct Depot_deploy::Main struct Depot_deploy::Main
{ {
typedef String<128> Name; struct Repeatable
Env &_env;
Attached_rom_dataspace _config { _env, "config" };
Attached_rom_dataspace _blueprint { _env, "blueprint" };
Expanding_reporter _query_reporter { _env, "query" , "query"};
Expanding_reporter _init_config_reporter { _env, "config", "init.config"};
Heap _heap { _env.ram(), _env.rm() };
Reconstructible<Session_label> _children_label_prefix { };
Timer::Connection _timer { _env };
Signal_handler<Main> _config_handler { _env.ep(), *this, &Main::_handle_config };
Children _children { _heap, _timer, _config_handler };
Log_root _log_root { _env.ep(), _heap, _children, *_children_label_prefix };
void _handle_config()
{ {
_config.update(); Env &_env;
_blueprint.update(); Heap &_heap;
Signal_transmitter _repeat_handler;
Attached_rom_dataspace _config { _env, "config" };
Attached_rom_dataspace _blueprint { _env, "blueprint" };
Expanding_reporter _query_reporter { _env, "query" , "query"};
Expanding_reporter _init_config_reporter { _env, "config", "init.config"};
Reconstructible<Session_label> _children_label_prefix { };
Timer::Connection _timer { _env };
Signal_handler<Repeatable> _config_handler { _env.ep(), *this, &Repeatable::_handle_config };
Children _children { _heap, _timer, _config_handler };
Xml_node const config = _config.xml(); void _handle_config()
{
_config.update();
_blueprint.update();
_children_label_prefix.construct(config.attribute_value("children_label_prefix", String<160>())); Xml_node const config = _config.xml();
_children.apply_config(config);
_children.apply_blueprint(_blueprint.xml());
/* determine CPU architecture of deployment */ _children_label_prefix.construct(config.attribute_value("children_label_prefix", String<160>()));
typedef String<16> Arch; _children.apply_config(config);
Arch const arch = config.attribute_value("arch", Arch()); _children.apply_blueprint(_blueprint.xml());
if (!arch.valid())
warning("config lacks 'arch' attribute");
/* generate init config containing all configured start nodes */ /* determine CPU architecture of deployment */
bool finished; typedef String<16> Arch;
_init_config_reporter.generate([&] (Xml_generator &xml) { Arch const arch = config.attribute_value("arch", Arch());
Xml_node static_config = config.sub_node("static"); if (!arch.valid())
xml.append(static_config.content_base(), static_config.content_size()); warning("config lacks 'arch' attribute");
Child::Depot_rom_server const parent { };
finished = _children.gen_start_nodes(xml, config.sub_node("common_routes"),
parent, parent);
});
if (finished) {
Result result; /* generate init config containing all configured start nodes */
unsigned long previous_time_sec { 0UL }; bool finished;
if (config.has_sub_node("previous-results")) { _init_config_reporter.generate([&] (Xml_generator &xml) {
Xml_node const previous_results = config.sub_node("previous-results"); Xml_node static_config = config.sub_node("static");
previous_time_sec += previous_results.attribute_value("time_sec", 0UL); xml.append(static_config.content_base(), static_config.content_size());
result.succeeded += previous_results.attribute_value("succeeded", 0UL); Child::Depot_rom_server const parent { };
result.failed += previous_results.attribute_value("failed", 0UL); finished = _children.gen_start_nodes(xml, config.sub_node("common_routes"),
result.skipped += previous_results.attribute_value("skipped", 0UL); parent, parent);
} });
unsigned long const time_us { _timer.curr_time().trunc_to_plain_us().value }; if (finished) {
unsigned long time_ms { time_us / 1000UL };
unsigned long const time_sec { time_ms / 1000UL };
time_ms = time_ms - time_sec * 1000UL;
log("\n--- Finished after ", time_sec + previous_time_sec, ".", time_ms < 10 ? "00" : time_ms < 100 ? "0" : "", time_ms, " sec ---\n"); Result result;
if (config.has_sub_node("previous-results")) { unsigned long previous_time_sec { 0UL };
Xml_node const previous_results = config.sub_node("previous-results"); if (config.has_sub_node("previous-results")) {
if (previous_results.content_size()) { Xml_node const previous_results = config.sub_node("previous-results");
log(Cstring(previous_results.content_base(), previous_results.content_size())); previous_time_sec += previous_results.attribute_value("time_sec", 0UL);
result.succeeded += previous_results.attribute_value("succeeded", 0UL);
result.failed += previous_results.attribute_value("failed", 0UL);
result.skipped += previous_results.attribute_value("skipped", 0UL);
}
unsigned long const time_us { _timer.curr_time().trunc_to_plain_us().value };
unsigned long time_ms { time_us / 1000UL };
unsigned long const time_sec { time_ms / 1000UL };
time_ms = time_ms - time_sec * 1000UL;
_children.conclusion(result);
int exit_code = result.failed ? -1 : 0;
typedef String<12> Repeat;
Repeat repeat = config.attribute_value("repeat", Repeat("false"));
if (repeat == Repeat("until_forever") ||
(repeat == Repeat("until_failed") && exit_code == 0)) {
char const *empty_config_str = "<config/>";
Xml_node const empty_config(empty_config_str, 10);
_children.apply_config(empty_config);
_query_reporter.generate([&] (Xml_generator &xml) { xml.attribute("arch", arch); });
_init_config_reporter.generate([&] (Xml_generator &) { });
_repeat_handler.submit();
return;
} else {
log("\n--- Finished after ", time_sec + previous_time_sec, ".", time_ms < 10 ? "00" : time_ms < 100 ? "0" : "", time_ms, " sec ---\n");
if (config.has_sub_node("previous-results")) {
Xml_node const previous_results = config.sub_node("previous-results");
if (previous_results.content_size()) {
log(Cstring(previous_results.content_base(), previous_results.content_size()));
}
}
_children.print_conclusion();
log("");
log(result);
log("");
_env.parent().exit(exit_code);
} }
} }
_children.conclusion(result);
log(""); /* update query for blueprints of all unconfigured start nodes */
log(result); if (arch.valid()) {
log(""); _query_reporter.generate([&] (Xml_generator &xml) {
_env.parent().exit(result.failed ? -1 : 0); xml.attribute("arch", arch);
_children.gen_queries(xml);
});
}
} }
/* update query for blueprints of all unconfigured start nodes */ Repeatable(Env &env,
if (arch.valid()) { Signal_context_capability const &repeat_handler,
_query_reporter.generate([&] (Xml_generator &xml) { Heap &heap)
xml.attribute("arch", arch); :
_children.gen_queries(xml); _env(env),
}); _heap(heap),
_repeat_handler(repeat_handler)
{
_config .sigh(_config_handler);
_blueprint.sigh(_config_handler);
_handle_config();
} }
};
Env &_env;
Signal_handler<Main> _repeat_handler { _env.ep(), *this, &Main::_handle_repeat };
Heap _heap { _env.ram(), _env.rm() };
Reconstructible<Repeatable> _repeatable { _env, _repeat_handler, _heap };
Log_root _log_root { _env.ep(), _heap, _repeatable->_children, *_repeatable->_children_label_prefix };
void _handle_repeat()
{
_repeatable.construct(_env, _repeat_handler, _heap);
} }
Main(Env &env) : _env(env) Main(Env &env) : _env(env)
{ {
_config .sigh(_config_handler);
_blueprint.sigh(_config_handler);
_handle_config();
_env.parent().announce(_env.ep().manage(_log_root)); _env.parent().announce(_env.ep().manage(_log_root));
} }
}; };