genode/repos/gems/run/depot_autopilot.run

805 lines
23 KiB
Plaintext
Raw Normal View History

####################
## User interface ##
####################
#
# Additional environment variables considered by this run script
#
# TEST_PKGS list of test package-archives if set (default list below)
# TEST_SRCS list of test source-archive if TEST_PKGS is set (default list below)
# TEST_BUILDS list of test build-targets that shall be build from repos
# TEST_MODULES list of test boot-modules to overlay the test depot-content
#
# To get a hint which build components and which boot modules you may want to
# enter for a given test package-archive, please see these files:
#
# repos/<REPO>/recipes/pkg/<TEST_PKG>/archives
# repos/<REPO>/recipes/pkg/<TEST_PKG>/runtime (<content> tag)
#
# Example:
#
# ! KERNEL="nova" \
# TEST_PKGS="test-libc_vfs test-log" \
# TEST_BUILDS="server/ram_fs test/libc_vfs" \
# TEST_MODULES="ram_fs test-libc_vfs vfs.lib.so" \
# make run/depot_autopilot
#
##############################################################
## Local copies of run tool procedures with small adaptions ##
## ##
## FIXME: Adapt original and remove local copies ##
##############################################################
proc autopilot_wait_for_output { wait_for_re timeout_value running_spawn_id } {
global output
global run_genode_failed
if {$wait_for_re == "forever"} {
set timeout -1
interact {
\003 {
send_user "Expect: 'interact' received 'strg+c' and was cancelled\n";
exit
}
-i $running_spawn_id
}
} else {
set timeout $timeout_value
}
set platform_msg [run_boot_string]
if {$platform_msg eq ""} {
set platform_msg "undefined platform command startup string sequence"
}
expect {
-i $running_spawn_id $platform_msg { puts stderr "Error: platform rebooted unexpectedly"; exit -4 }
-i $running_spawn_id -re $wait_for_re { }
# pistachio kernel fault
-i $running_spawn_id -re {--- "KD# Exception caught" ---} {
puts stderr "Error: Kernel fault";
set run_genode_failed 1
return
}
# sel4 unknown fault caught by core
-i $running_spawn_id -re {Error: unexpected exception during fault.*stopped} {
puts stderr "Error: Unknown fault";
set run_genode_failed 1
return
}
# can happen, for instance, on socat TCP-timeout
2018-12-05 12:30:03 +00:00
eof {
puts stderr "Error: Spawned process died unexpectedly";
set run_genode_failed 1
2018-12-05 12:30:03 +00:00
return
}
timeout {
puts stderr "Error: Test execution timed out";
set run_genode_failed 1
return
}
}
append output $expect_out(buffer)
}
proc autopilot_create_tar_from_depot_binaries { archive_path args } {
# filter out api and src archives from requested depot content
set content {}
foreach subdir [_collect_from_depot $args] {
if {[regexp [_depot_archive_versioned_path_pattern] $subdir dummy user type]} {
if {$type == "src"} continue;
if {$type == "api"} continue;
}
lappend content $subdir
}
check_for_missing_depot_archives
eval "exec tar cf $archive_path -T /dev/null -C [depot_dir] [lsort -unique $content]"
}
proc autopilot_run_genode_until {{wait_for_re forever} {timeout_value 0} {running_spawn_id -1}} {
#
# If a running_spawn_id is specified, wait for the expected output
#
if {$running_spawn_id != -1} {
autopilot_wait_for_output $wait_for_re $timeout_value $running_spawn_id
return;
}
set retry 3
while { $retry != 0 } {
if {[expr [run_power_on] == false]} {
puts "Power on step failed, retry."
sleep 3
incr retry -1;
continue
}
if {![run_load]} {
puts "Load step failed, retry."
# kill the spawned load process if there is one
if {[load_spawn_id] != -1} {
set pid [exp_pid -i [load_spawn_id]]
exec kill -9 $pid
}
incr retry -1;
continue;
}
if {![run_log $wait_for_re $timeout_value]} {
puts "Log step failed, retry."
incr retry -1;
continue;
}
return;
}
puts stderr "Boot process failed 3 times in series. I give up!";
exit -1;
}
########################
## Utility procedures ##
########################
proc depot_user {} { return [get_cmd_arg --depot-user genodelabs] }
#
# Get value of an environment variable
#
proc get_env_var { var_name default_value } {
if {[info exists ::env($var_name)]} {
return $::env($var_name)
}
return $default_value
}
#
# Check if archives are available without doing anything with them
#
proc check_archives_available { args } {
# filter out api and src archives from requested depot content
set content {}
foreach subdir [_collect_from_depot $args] {
if {[regexp [_depot_archive_versioned_path_pattern] $subdir dummy user type]} {
if {$type == "src"} continue;
if {$type == "api"} continue;
}
lappend content $subdir
}
check_for_missing_depot_archives
}
#
# Return routes for boot modules that shall overlay the test-depot content
#
proc single_test_module_routes { } {
global test_modules
set result ""
foreach module $test_modules {
append result {
<service name="ROM" label_last="} $module {"> <parent/> </service>}
}
return $result
}
#
# Return autopilot start-nodes for the test packages that shall be run
#
proc test_pkgs_start_nodes { } {
global test_pkgs
global skip_test_pkg
set result ""
foreach test_pkg $test_pkgs {
if {[info exists skip_test_pkg($test_pkg)] && $skip_test_pkg($test_pkg)} {
append result {
<start name="} $test_pkg {" skip="true"/>}
} else {
append result {
<start name="} $test_pkg {" pkg="} [depot_user] {/pkg/} $test_pkg {/} [_current_depot_archive_version pkg $test_pkg] {"/>}
}
}
return $result
}
#
# Prepare to call run_genode_until (again, with a changed setup)
#
proc prepare_to_run_genode { } {
global output
global qemu_args
global previous_results
global previous_time_ms
global previous_succeeded
global previous_failed
global previous_skipped
global skip_test_pkg
global test_pkgs
global test_srcs
global test_builds
global test_modules
global last_test_pkg
global run_genode_failed
global serial_id
global timeout
global initial_qemu_args
set qemu_args $initial_qemu_args
#
# Create a depot archive that contains the test packages
#
create_boot_directory
set depot_tar_archives ""
set depot_tar_src_archives ""
set import_archives ""
foreach test_pkg $test_pkgs {
if { [info exists skip_test_pkg($test_pkg)] } {
append_if [expr !$skip_test_pkg($test_pkg)] depot_tar_archives " [depot_user]/pkg/$test_pkg "
} else {
append depot_tar_archives " [depot_user]/pkg/$test_pkg "
}
}
foreach test_src_pkg $test_srcs {
if { [info exists skip_test_pkg($test_src_pkg)] } {
append_if [expr !$skip_test_pkg($test_src_pkg)] depot_tar_src_archives " [depot_user]/src/$test_src_pkg "
} else {
append depot_tar_src_archives " [depot_user]/src/$test_src_pkg "
}
}
append import_archives {
} [depot_user] {/src/} [base_src] {
} [depot_user] {/src/report_rom
} [depot_user] {/src/fs_rom
} [depot_user] {/src/vfs
} [depot_user] {/src/loader
} [depot_user] {/src/init
} [depot_user] {/src/depot_query
}
if {!$skip_test_pkg(test-lx_block)} {
append import_archives [depot_user] {/raw/test-lx_block }
}
set all_archives [concat $depot_tar_archives $import_archives]
check_archives_available {*}$all_archives
autopilot_create_tar_from_depot_binaries [run_dir]/genode/depot.tar {*}$depot_tar_archives
2018-11-23 18:04:53 +00:00
set depot_tar_wo_src_size [file size [run_dir]/genode/depot.tar]
append_src_and_api_depot_packages_to_tar [run_dir]/genode/depot.tar {*}$depot_tar_src_archives
2018-11-23 18:04:53 +00:00
set depot_tar_size [file size [run_dir]/genode/depot.tar]
puts "Depot archive has a size of $depot_tar_size bytes ($depot_tar_wo_src_size bytes without sources)"
#
# Install the root-init config
#
append config {
<config prio_levels="2">
<parent-provides>
<service name="ROM"/>
<service name="IRQ"/>
<service name="IO_PORT"/>
<service name="IO_MEM"/>
<service name="PD"/>
<service name="RM"/>
<service name="CPU"/>
<service name="LOG"/>
<service name="TRACE"/>
</parent-provides>
<default-route>
<any-service> <parent/> <any-child/> </any-service>
</default-route>
<default caps="100"/>
<start name="timer" priority="0">
<resource name="RAM" quantum="1M"/>
<provides> <service name="Timer"/> </provides>
</start>
<start name="report_rom" priority="-1">
<binary name="report_rom"/>
<resource name="RAM" quantum="2M"/>
<provides> <service name="Report"/> <service name="ROM"/> </provides>
<config verbose="no">
<policy label="depot_autopilot -> blueprint" report="depot_query -> blueprint"/>
<policy label="depot_query -> query" report="depot_autopilot -> query"/>
<policy label="dynamic -> config" report="depot_autopilot -> init.config"/>
</config>
</start>
<start name="vfs" priority="-1">
<resource name="RAM" quantum="16M"/>
<provides> <service name="File_system"/> </provides>
<config>
<vfs>
<dir name="depot"> <tar name="depot.tar"/> </dir>
<dir name="gcov_data"> <ram/> </dir>
</vfs>
<policy label="depot_query -> depot" root="/depot" />
<policy label="fs_rom -> " root="/depot" />
<policy label="dynamic -> gcov -> gcov -> depot" root="/depot" />
<policy label_suffix=" -> gcov_data" root="/gcov_data" writeable="yes" />
</config>
</start>
<start name="fs_rom" priority="-1" caps="300">
<resource name="RAM" quantum="16M"/>
<provides> <service name="ROM"/> </provides>
</start>
<start name="depot_query" priority="-1">
<resource name="RAM" quantum="2M"/>
<config query="rom">
<vfs> <dir name="depot"> <fs label="depot"/> </dir> </vfs>
</config>
<route>
<service name="ROM" label="query"> <child name="report_rom"/> </service>
<any-service> <parent/> <any-child/> </any-service>
</route>
</start>
<start name="depot_autopilot" priority="-1">
<resource name="RAM" quantum="2M"/>
<provides> <service name="LOG"/> </provides>
<config arch="} [depot_spec] {" children_label_prefix="dynamic -> ">
<static>
<parent-provides>
<service name="ROM"/>
<service name="CPU"/>
<service name="PD"/>
<service name="LOG"/>
<service name="RM"/>
<service name="Timer"/>
<service name="TRACE"/>
<service name="File_system"/>
</parent-provides>
</static>
<common_routes>
} [single_test_module_routes] {
<service name="ROM" label_last="ld.lib.so"> <parent/> </service>
<service name="ROM" label_last="init"> <parent/> </service>
<service name="CPU"> <parent/> </service>
<service name="PD"> <parent/> </service>
<service name="LOG"> <parent/> </service>
<service name="Timer"> <parent/> </service>
<service name="RM"> <parent/> </service>
<service name="TRACE"> <parent/> </service>
<service name="File_system" label="gcov -> depot"> <parent/> </service>
<service name="File_system" label_suffix="gcov_data"> <parent/> </service>
</common_routes>
<previous-results time_sec="} [expr ($previous_time_ms + 999) / 1000] {"
succeeded="} $previous_succeeded {"
skipped="} $previous_skipped {"
failed="} $previous_failed {">} [string map {\" .} $previous_results] {</previous-results>
} [test_pkgs_start_nodes] {
</config>
<route>
<service name="PD" unscoped_label="depot_autopilot"> <parent/> </service>
<service name="CPU" unscoped_label="depot_autopilot"> <parent/> </service>
<service name="LOG" unscoped_label="depot_autopilot"> <parent/> </service>
<service name="ROM" unscoped_label="depot_autopilot"> <parent/> </service>
<service name="ROM" unscoped_label="ld.lib.so"> <parent/> </service>
<service name="ROM" label="blueprint"> <child name="report_rom"/> </service>
<service name="Report" label="query"> <child name="report_rom"/> </service>
<service name="Report" label="init.config"> <child name="report_rom"/> </service>
<service name="Timer" label=""> <child name="timer"/> </service>
</route>
</start>
<start name="dynamic" caps="8000" priority="-1">
<resource name="RAM" quantum="420M"/>
<binary name="init"/>
<route>
} [single_test_module_routes] {
<service name="ROM" label_last="ld.lib.so"> <parent/> </service>
<service name="ROM" label_last="init"> <parent/> </service>
<service name="ROM" label="config"> <child name="report_rom"/> </service>
<service name="ROM"> <child name="fs_rom"/> </service>
<service name="LOG" unscoped_label="dynamic"> <parent/> </service>
<service name="LOG"> <child name="depot_autopilot"/> </service>
<service name="Timer"> <child name="timer"/> </service>
<any-service> <parent/> <any-child/> </any-service>
</route>
</start>
</config>}
install_config $config
#
# Create the rest of the boot modules
#
set build_components { app/depot_autopilot }
append build_components $test_builds
build $build_components
import_from_depot {*}$import_archives
#
# Build boot image from boot modules
#
set boot_modules { depot_autopilot }
append boot_modules $test_modules
build_boot_image $boot_modules
set last_test_pkg ""
set run_genode_failed 0
set serial_id -1
set timeout 40
append qemu_args "-nographic -serial mon:stdio "
}
#
# Initialize variables that accumulate test results for possible reboots
#
proc init_previous_results {} {
global previous_results_on_exit
global previous_results
global previous_time_ms
global previous_succeeded
global previous_failed
global previous_skipped
set previous_results_on_exit 1
set previous_results ""
set previous_time_ms 0
set previous_succeeded 0
set previous_failed 0
set previous_skipped 0
}
rename exit run_tool_exit
proc exit {{status 0}} {
global previous_results_on_exit
global previous_results
global previous_time_ms
global previous_succeeded
global previous_failed
global previous_skipped
if {![info exists previous_results_on_exit] || \
![info exists previous_results] || \
![info exists previous_time_ms] || \
![info exists previous_succeeded] || \
![info exists previous_failed] || \
![info exists previous_skipped]} \
{
init_previous_results
}
if {$previous_results_on_exit != 0} {
set previous_time_sec [expr $previous_time_ms / 1000]
set previous_time_sec_frac [expr $previous_time_ms - $previous_time_sec * 1000]
set previous_results_list [split $previous_results "\n"]
puts ""
puts "Failed to let Depot Autopilot finish!"
puts "Result overview simulated by run script:"
puts ""
puts "\[init -> depot_autopilot] --- Finished after $previous_time_sec.$previous_time_sec_frac sec ---"
puts "\[init -> depot_autopilot] "
foreach previous_result $previous_results_list {
puts "\[init -> depot_autopilot] $previous_result"
}
puts "\[init -> depot_autopilot] "
puts "\[init -> depot_autopilot] succeeded: $previous_succeeded failed: $previous_failed skipped: $previous_skipped"
puts "\[init -> depot_autopilot] "
}
run_tool_exit $status
}
##################
## Main routine ##
##################
#
# Check platform support
#
if {[expr ![have_spec x86] && \
![have_spec arm_v6] && \
![have_spec arm_v7a]] || \
[expr [have_spec foc] && [have_spec panda]]} \
{
puts "\n Run script is not supported on this platform. \n";
exit 0
}
#
# Default list of test package-archives
#
# Was obtained by issuing:
#
# ! cd <GENODE_DIR>/repos
# ! find . -type d -wholename *recipes/pkg/test-* -printf '%f\n' | sort
#
set default_test_pkgs {
test-ada
test-ada_exception
test-ada_secondary_stack
test-blk
test-blk_cache
test-clipboard
test-ds_ownership
test-dynamic_config
test-dynamic_config_loader
test-dynamic_config_slave
test-expat
test-fault_detection
test-fs_log
test-fs_packet
test-fs_report
test-fs_rom_update
test-fs_rom_update_fs
test-fs_rom_update_ram
test-init
test-init_loop
test-ldso
test-libc
test-libc_counter
test-libc_getenv
test-libc_pipe
test-libc_vfs
test-libc_vfs_audit
test-libc_vfs_block
test-libc_vfs_counter
test-libc_vfs_fs
test-libc_vfs_fs_chained
test-libc_vfs_ram
test-log
test-lx_block
test-magic_ring_buffer
test-mmio
test-new_delete
test-nic_loopback
test-part_blk_gpt
test-part_blk_mbr
test-pipe
test-pthread
test-python
test-ram_fs_chunk
test-read_only_rom
test-reconstructible
test-registry
test-report_rom
test-resource_request
test-resource_yield
test-rm_fault
test-rm_fault_no_nox
test-rm_nested
test-rom_blk
test-rom_filter
test-rust
2018-12-06 04:19:19 +00:00
test-sanitizer
test-sequence
test-signal
test-slab
test-solo5
test-stdcxx
test-synced_interface
test-tcp_bulk_lwip
test-tcp_bulk_lxip
test-terminal_crosslink
test-timed_semaphore
test-timer
test-trace
test-trace_logger
test-utf8
test-vfs_stress_fs
test-vfs_stress_ram
test-weak_ptr
test-xml_generator
test-xml_node
gcov
}
#
# Default list of test source-archives
#
set default_test_srcs {
test-xml_generator
}
#
# Whether the platform supports non-executable dataspaces
#
proc non_executable_supported { } {
if {[have_spec hw] && [have_spec x86_64]} { return true }
if {[have_spec hw] && [have_spec arm]} { return true }
if {[have_spec nova] && [have_spec x86_64]} { return true }
if {[have_spec foc] && [have_spec x86_64]} { return true }
if {[have_spec foc] && [have_spec arm]} { return true }
if {[have_spec sel4] && [have_spec arm]} { return true }
return false
}
#
# Whether to skip a test - if undefined for a test, the test is not skipped
#
set skip_test_pkg(test-python) [expr ![have_spec x86]]
set skip_test_pkg(test-slab) [expr [get_cmd_switch --autopilot] && [have_include "power_on/qemu"]]
set skip_test_pkg(test-rm_nested) [expr [have_spec linux]]
set skip_test_pkg(test-fault_detection) [expr [have_spec pistachio] || [have_spec fiasco]]
set skip_test_pkg(test-fs_packet) [expr [get_cmd_switch --autopilot] && [have_include "power_on/qemu"]]
set skip_test_pkg(test-rm_fault) [expr [have_spec linux] || ![non_executable_supported]]
set skip_test_pkg(test-rm_fault_no_nox) [expr [have_spec linux] || !$skip_test_pkg(test-rm_fault)]
set skip_test_pkg(test-lx_block) [expr ![have_spec linux]]
set skip_test_pkg(test-tcp_bulk_lwip) [expr ![have_spec x86]]
set skip_test_pkg(test-tcp_bulk_lxip) [expr ![have_spec x86]]
set skip_test_pkg(test-solo5) [expr ![have_spec x86_64]]
set skip_test_pkg(test-libc) [expr [have_spec sel4] || [have_spec rpi] || [expr [have_spec pbxa9] && [have_spec foc]] || [expr [have_spec imx53] && [have_spec trustzone]]]
#
# FIXME
#
# When doing the libc_getenv test on autopilot+foc+x86 and one of the
# subsequent tests crashes the system so it gets rebooted by the run script,
# the system doesn't come up again. It gets stuck after core initialization.
#
set skip_test_pkg(test-libc_getenv) [expr [get_cmd_switch --autopilot] && [have_spec foc] && [have_spec x86]]
# remember initial qemu args in case we have to re-boot later
set initial_qemu_args ""
if {[info exists qemu_args]} {
set initial_qemu_args $qemu_args
}
#
# Initialize the lists of test package-archives, source-archives,
# build-targets, and boot-modules to be considered by the run script
#
set test_pkgs [get_env_var TEST_PKGS ""]
set test_srcs [get_env_var TEST_SRCS ""]
set test_builds [get_env_var TEST_BUILDS ""]
set test_modules [get_env_var TEST_MODULES ""]
#
# If the user didn't declare an individual list of test package-archives, use
# the default lists of test package-archives and source-archives
#
set nr_of_tests_to_run 0
if {$test_pkgs == ""} {
foreach test_pkg $default_test_pkgs {
append test_pkgs " $test_pkg "
if {!([info exists skip_test_pkg($test_pkg)] && $skip_test_pkg($test_pkg))} {
incr nr_of_tests_to_run
}
}
foreach test_src_pkg $default_test_srcs {
append test_srcs " $test_src_pkg "
}
} else {
foreach test_pkg $test_pkgs {
if {!([info exists skip_test_pkg($test_pkg)] && $skip_test_pkg($test_pkg))} {
incr nr_of_tests_to_run
}
}
}
puts "Number of tests to run: $nr_of_tests_to_run"
# initialize variables that hold previous results
init_previous_results
# generic preparation for each system boot
prepare_to_run_genode
while {1} {
# wait for the next autopilot event
if {$serial_id == -1} {
autopilot_run_genode_until {depot_autopilot\] --- .*?\n} $timeout
set serial_id [output_spawn_id]
# if the system didn't even boot, exit (prints previous results)
if {$run_genode_failed} {
exit -1
}
} else {
set init_time_ms [clock clicks -millisec]
autopilot_run_genode_until {depot_autopilot\] --- .*?\n} $timeout $serial_id
set previous_time_ms [expr $previous_time_ms + [expr ([clock clicks -millisec] - $init_time_ms)] ]
set serial_id [output_spawn_id]
# remove last test from list and check if we have to reboot the system
set test_pkgs [lsearch -all -inline -not -exact $test_pkgs $last_test_pkg]
if {$run_genode_failed} {
# shut-down running system
exec kill -9 [exp_pid -i $serial_id]
run_power_off
# remember result of last test
if {$previous_results != ""} {
append previous_results \012
}
append previous_results { } [format {%-31s %-6s %7s} $last_test_pkg "failed " "$timeout.000"] { reboot}
incr previous_failed
# prepare system re-boot
prepare_to_run_genode
continue
}
}
# if the autopilot finished all tests, evaluate its return value
if {[regexp {depot_autopilot\] --- Finished} $output]} {
set output ""
run_genode_until {child "depot_autopilot" exited with exit value.*?\n} 10 $serial_id
set previous_results_on_exit 0
grep_output {^\[init\] }
compare_output_to {[init] child "depot_autopilot" exited with exit value 0}
exit 0
}
# if the autopilot started a new test, set a new timeout
if {[regexp {depot_autopilot\] --- Run} $output]} {
if {$last_test_pkg != ""} {
# remember result of last test in case the system must be restartet
set last_test_result ""
regexp {depot_autopilot\] ( [^\033]+)} $output ignored last_test_result
regsub -all {<} $last_test_result {\&lt;} last_test_result
set failed_off [string first " failed" $last_test_result]
set skipped_off [string first " skipped" $last_test_result]
set ok_off [string first " ok" $last_test_result]
if {$failed_off > 0 && ($skipped_off < 0 || $failed_off < $skipped_off) && ($ok_off < 0 || $failed_off < $ok_off)} {
incr previous_failed
} elseif {$skipped_off > 0 && ($ok_off < 0 || $skipped_off < $ok_off)} {
incr previous_skipped
} elseif {$ok_off > 0} {
incr previous_succeeded
} else {
puts "Error: malformed test result"
puts $last_test_result
exit -1
}
if {$previous_results != ""} {
append previous_results \012
}
append previous_results $last_test_result
}
# determine timeout for the next timeout
set min_timeout 10
regexp {depot_autopilot\] --- Run "(.*?)" \(max ([0-9]*?) } $output] ignore last_test_pkg min_timeout
set timeout [expr $min_timeout + 10]
set output ""
}
}