# # Integration of Depot Autopilot into a Run-tool-based test infrastructure # # For a detailed documentation of the user interface and the features of this # Run script please see _repos/gems/src/app/depot_autopilot/README_. # ############################################################## ## Local copies of run tool procedures with small adaptions ## ## ## ## FIXME: Adapt original and remove local copies ## ############################################################## proc autopilot_wait_for_output { wait_for_re timeout_value running_spawn_id } { global output global run_genode_failed if {$wait_for_re == "forever"} { set timeout -1 interact { \003 { send_user "Expect: 'interact' received 'strg+c' and was cancelled\n"; exit } -i $running_spawn_id } } else { set timeout $timeout_value } set platform_msg [run_boot_string] if {$platform_msg eq ""} { set platform_msg "undefined platform command startup string sequence" } expect { -i $running_spawn_id $platform_msg { puts stderr "Error: platform rebooted unexpectedly"; exit -4 } -i $running_spawn_id -re $wait_for_re { } # sel4 -i $running_spawn_id -re {Error: ~Cnode - not implemented.*?\n} { puts stderr "Error: Core presumably went out of resources"; set run_genode_failed 1 return } # pistachio kernel fault -i $running_spawn_id -re {--- "KD# Exception caught" ---.*?\n} { puts stderr "Error: Kernel fault"; set run_genode_failed 1 return } # sel4 unknown fault caught by core -i $running_spawn_id -re {Error: unexpected exception during fault.*?stopped.*?\n} { puts stderr "Error: Unknown fault"; set run_genode_failed 1 return } # can happen, for instance, on socat TCP-timeout eof { puts stderr "Error: Spawned process died unexpectedly"; set run_genode_failed 1 return } timeout { puts stderr "Error: Test execution timed out"; set run_genode_failed 1 return } } append output $expect_out(buffer) } proc autopilot_create_tar_from_depot_binaries { archive_path args } { # filter out api and src archives from requested depot content set content {} foreach subdir [_collect_from_depot $args] { if {[regexp [_depot_archive_versioned_path_pattern] $subdir dummy user type]} { if {$type == "src"} continue; if {$type == "api"} continue; } lappend content $subdir } check_for_missing_depot_archives eval "exec tar cf $archive_path -T /dev/null -C [depot_dir] [lsort -unique $content]" } proc autopilot_run_genode_until {{wait_for_re forever} {timeout_value 0} {running_spawn_id -1}} { # # If a running_spawn_id is specified, wait for the expected output # if {$running_spawn_id != -1} { autopilot_wait_for_output $wait_for_re $timeout_value $running_spawn_id return; } set retry 3 while { $retry != 0 } { if {[expr [run_power_on] == false]} { puts "Power on step failed, retry." sleep 3 incr retry -1; continue } if {![run_load]} { puts "Load step failed, retry." # kill the spawned load process if there is one if {[load_spawn_id] != -1} { set pid [exp_pid -i [load_spawn_id]] exec kill -9 $pid } incr retry -1; continue; } if {![run_log $wait_for_re $timeout_value]} { puts "Log step failed, retry." incr retry -1; continue; } return; } puts stderr "Boot process failed 3 times in series. I give up!"; exit -1; } ######################## ## Utility procedures ## ######################## proc depot_user {} { return [get_cmd_arg --depot-user genodelabs] } # # Get value of an environment variable # proc get_env_var { var_name default_value } { if {[info exists ::env($var_name)]} { return $::env($var_name) } return $default_value } # # Check if archives are available without doing anything with them # proc check_archives_available { args } { # filter out api and src archives from requested depot content set content {} foreach subdir [_collect_from_depot $args] { if {[regexp [_depot_archive_versioned_path_pattern] $subdir dummy user type]} { if {$type == "src"} continue; if {$type == "api"} continue; } lappend content $subdir } check_for_missing_depot_archives } # # Return routes for boot modules that shall overlay the test-depot content # proc single_test_module_routes { } { global test_modules set result "" foreach module $test_modules { append result { } } return $result } # # Return autopilot start-nodes for the test packages that shall be run # proc test_pkgs_start_nodes { } { global test_pkgs global skip_test_pkg set result "" foreach test_pkg $test_pkgs { if {[info exists skip_test_pkg($test_pkg)] && $skip_test_pkg($test_pkg)} { append result { } } else { append result { } } } return $result } # # Prepare to call run_genode_until (again, with a changed setup) # proc prepare_to_run_genode { } { global output global qemu_args global previous_results global previous_time_ms global previous_succeeded global previous_failed global previous_skipped global skip_test_pkg global test_pkgs global test_srcs global test_builds global test_modules global test_repeat global last_test_pkg global run_genode_failed global serial_id global timeout global initial_qemu_args set qemu_args $initial_qemu_args # # Create a depot archive that contains the test packages # create_boot_directory set depot_tar_archives "" set depot_tar_src_archives "" set import_archives "" foreach test_pkg $test_pkgs { if { [info exists skip_test_pkg($test_pkg)] } { append_if [expr !$skip_test_pkg($test_pkg)] depot_tar_archives " [depot_user]/pkg/$test_pkg " } else { append depot_tar_archives " [depot_user]/pkg/$test_pkg " } } foreach test_src_pkg $test_srcs { if { [info exists skip_test_pkg($test_src_pkg)] } { append_if [expr !$skip_test_pkg($test_src_pkg)] depot_tar_src_archives " [depot_user]/src/$test_src_pkg " } else { append depot_tar_src_archives " [depot_user]/src/$test_src_pkg " } } append import_archives { } [depot_user] {/src/} [base_src] { } [depot_user] {/src/report_rom } [depot_user] {/src/fs_rom } [depot_user] {/src/vfs } [depot_user] {/src/loader } [depot_user] {/src/init } [depot_user] {/src/depot_query } if {!$skip_test_pkg(test-lx_block)} { append import_archives [depot_user] {/raw/test-lx_block } } set all_archives [concat $depot_tar_archives $import_archives] check_archives_available {*}$all_archives autopilot_create_tar_from_depot_binaries [run_dir]/genode/depot.tar {*}$depot_tar_archives set depot_tar_wo_src_size [file size [run_dir]/genode/depot.tar] append_src_and_api_depot_packages_to_tar [run_dir]/genode/depot.tar {*}$depot_tar_src_archives set depot_tar_size [file size [run_dir]/genode/depot.tar] puts "Depot archive has a size of $depot_tar_size bytes ($depot_tar_wo_src_size bytes without sources)" # # Install the root-init config # append config { } [single_test_module_routes] { } [string map {\" .} $previous_results] { } [test_pkgs_start_nodes] { } [single_test_module_routes] { } install_config $config # # Create the rest of the boot modules # set build_components { app/depot_autopilot } append build_components $test_builds build $build_components import_from_depot {*}$import_archives # # Build boot image from boot modules # set boot_modules { depot_autopilot } append boot_modules $test_modules build_boot_image $boot_modules set last_test_pkg "" set run_genode_failed 0 set serial_id -1 set timeout 40 append qemu_args "-nographic -serial mon:stdio " } # # Initialize variables that accumulate test results for possible reboots # proc init_previous_results {} { global previous_results_on_exit global previous_results global previous_time_ms global previous_succeeded global previous_failed global previous_skipped set previous_results_on_exit 1 set previous_results "" set previous_time_ms 0 set previous_succeeded 0 set previous_failed 0 set previous_skipped 0 } # # Initialize variables that can be configured via the user interface # proc init_test_setting {} { global test_pkgs global test_srcs global test_builds global test_modules global test_repeat global default_test_pkgs global default_test_srcs global skip_test_pkg # # Initialize the lists of test package-archives, source-archives, # build-targets, and boot-modules to be considered by the run script # set test_pkgs [get_env_var TEST_PKGS ""] set test_srcs [get_env_var TEST_SRCS ""] set test_builds [get_env_var TEST_BUILDS ""] set test_modules [get_env_var TEST_MODULES ""] set test_repeat [get_env_var TEST_REPEAT "false"] # # If the user didn't declare an individual list of test package-archives, use # the default lists of test package-archives and source-archives # set nr_of_tests_to_run 0 if {$test_pkgs == ""} { foreach test_pkg $default_test_pkgs { append test_pkgs " $test_pkg " if {!([info exists skip_test_pkg($test_pkg)] && $skip_test_pkg($test_pkg))} { incr nr_of_tests_to_run } } foreach test_src_pkg $default_test_srcs { append test_srcs " $test_src_pkg " } } else { foreach test_pkg $test_pkgs { if {!([info exists skip_test_pkg($test_pkg)] && $skip_test_pkg($test_pkg))} { incr nr_of_tests_to_run } } } } rename exit run_tool_exit proc exit {{status 0}} { global previous_results_on_exit global previous_results global previous_time_ms global previous_succeeded global previous_failed global previous_skipped if {![info exists previous_results_on_exit] || \ ![info exists previous_results] || \ ![info exists previous_time_ms] || \ ![info exists previous_succeeded] || \ ![info exists previous_failed] || \ ![info exists previous_skipped]} \ { init_previous_results } if {$previous_results_on_exit != 0} { set previous_time_sec [expr $previous_time_ms / 1000] set previous_time_sec_frac [expr $previous_time_ms - $previous_time_sec * 1000] set previous_results_list [split $previous_results "\n"] puts "" puts "Failed to let Depot Autopilot finish!" puts "Result overview simulated by run script:" puts "" puts "\[init -> depot_autopilot] --- Finished after $previous_time_sec.$previous_time_sec_frac sec ---" puts "\[init -> depot_autopilot] " foreach previous_result $previous_results_list { puts "\[init -> depot_autopilot] $previous_result" } puts "\[init -> depot_autopilot] " puts "\[init -> depot_autopilot] succeeded: $previous_succeeded failed: $previous_failed skipped: $previous_skipped" puts "\[init -> depot_autopilot] " } run_tool_exit $status } ################## ## Main routine ## ################## # # Check platform support # if {[expr ![have_spec x86] && \ ![have_spec arm_v6] && \ ![have_spec arm_v7a]] || \ [expr [have_spec foc] && [have_spec panda]]} \ { puts "\n Run script is not supported on this platform. \n"; exit 0 } # # Default list of test package-archives # # Was obtained by issuing: # # ! cd /repos # ! find . -type d -wholename *recipes/pkg/test-* -printf '%f\n' | sort # set default_test_pkgs { test-ada test-ada_exception test-ada_secondary_stack test-block test-block_cache test-clipboard test-ds_ownership test-dynamic_config test-dynamic_config_loader test-dynamic_config_slave test-expat test-fault_detection test-fs_log test-fs_packet test-fs_report test-fs_rom_update test-fs_rom_update_fs test-fs_rom_update_ram test-gnatio test-init test-init_loop test-ldso test-libc test-libc_connect_lwip test-libc_connect_lxip test-libc_connect_vfs_server_lwip test-libc_connect_vfs_server_lxip test-libc_counter test-libc_getenv test-libc_pipe test-libc_vfs test-libc_vfs_audit test-libc_vfs_block test-libc_vfs_counter test-libc_vfs_fs test-libc_vfs_fs_chained test-libc_vfs_ram test-log test-lx_block test-magic_ring_buffer test-mmio test-new_delete test-nic_loopback test-part_block_gpt test-part_block_mbr test-pipe test-pthread test-python test-ram_fs_chunk test-read_only_rom test-reconstructible test-registry test-report_rom test-resource_request test-resource_yield test-rm_fault test-rm_fault_no_nox test-rm_nested test-rom_block test-rom_filter test-rust test-sanitizer test-sequence test-signal test-slab test-solo5 test-stdcxx test-synced_interface test-tcp_bulk_lwip test-tcp_bulk_lxip test-terminal_crosslink test-timed_semaphore test-timer test-trace test-trace_logger test-utf8 test-vfs_stress_fs test-vfs_stress_ram test-weak_ptr test-xml_generator test-xml_node gcov } # # Default list of test source-archives # set default_test_srcs { test-xml_generator } # # Whether the platform supports non-executable dataspaces # proc non_executable_supported { } { if {[have_spec hw] && [have_spec x86_64]} { return true } if {[have_spec hw] && [have_spec arm]} { return true } if {[have_spec nova] && [have_spec x86_64]} { return true } if {[have_spec foc] && [have_spec x86_64]} { return true } if {[have_spec foc] && [have_spec arm]} { return true } if {[have_spec sel4] && [have_spec arm]} { return true } return false } # # Whether to skip a test - if undefined for a test, the test is not skipped # set skip_test_pkg(test-python) [expr ![have_spec x86]] set skip_test_pkg(test-slab) [expr [get_cmd_switch --autopilot] && [have_include "power_on/qemu"]] set skip_test_pkg(test-rm_nested) [expr [have_spec linux]] set skip_test_pkg(test-fault_detection) [expr [have_spec pistachio] || [have_spec fiasco]] set skip_test_pkg(test-fs_packet) [expr [get_cmd_switch --autopilot] && [have_include "power_on/qemu"]] set skip_test_pkg(test-rm_fault) [expr [have_spec linux] || ![non_executable_supported]] set skip_test_pkg(test-rm_fault_no_nox) [expr [have_spec linux] || !$skip_test_pkg(test-rm_fault)] set skip_test_pkg(test-lx_block) [expr ![have_spec linux]] set skip_test_pkg(test-tcp_bulk_lwip) [expr ![have_spec x86]] set skip_test_pkg(test-tcp_bulk_lxip) [expr ![have_spec x86]] set skip_test_pkg(test-solo5) [expr ![have_spec x86_64]] set skip_test_pkg(test-libc) [expr [have_spec sel4] || [have_spec rpi] || [expr [have_spec pbxa9] && [have_spec foc]] || [expr [have_spec imx53] && [have_spec trustzone]]] # # FIXME # # When doing the libc_getenv test on autopilot+foc+x86 and one of the # subsequent tests crashes the system so it gets rebooted by the run script, # the system doesn't come up again. It gets stuck after core initialization. # set skip_test_pkg(test-libc_getenv) [expr [get_cmd_switch --autopilot] && [have_spec foc] && [have_spec x86]] # remember initial qemu args in case we have to re-boot later set initial_qemu_args "" if {[info exists qemu_args]} { set initial_qemu_args $qemu_args } # initialize global variables init_test_setting init_previous_results # generic preparation for each system boot prepare_to_run_genode while {1} { # wait for the next autopilot event if {$serial_id == -1} { autopilot_run_genode_until {depot_autopilot\] --- .*?\n} $timeout set serial_id [output_spawn_id] # if the system didn't even boot, exit (prints previous results) if {$run_genode_failed} { exit -1 } } else { set init_time_ms [clock clicks -millisec] autopilot_run_genode_until {depot_autopilot\] --- .*?\n} $timeout $serial_id set previous_time_ms [expr $previous_time_ms + [expr ([clock clicks -millisec] - $init_time_ms)] ] set serial_id [output_spawn_id] # remove last test from list and check if we have to reboot the system set test_pkgs [lsearch -all -inline -not -exact $test_pkgs $last_test_pkg] if {$run_genode_failed} { # shut-down running system exec kill -9 [exp_pid -i $serial_id] run_power_off # remember result of last test if {$previous_results != ""} { append previous_results \012 } append previous_results { } [format {%-31s %-6s %7s} $last_test_pkg "failed " "$timeout.000"] { reboot} incr previous_failed # prepare system re-boot prepare_to_run_genode continue } } # if the autopilot finished all tests, evaluate its return value if {[regexp {depot_autopilot\] --- Finished} $output]} { set output "" run_genode_until {child "depot_autopilot" exited with exit value.*?\n} 10 $serial_id set previous_results_on_exit 0 grep_output {^\[init\] } compare_output_to {[init] child "depot_autopilot" exited with exit value 0} exit 0 } # if the autopilot started a new test, set a new timeout if {[regexp {depot_autopilot\] --- Run} $output]} { if {$last_test_pkg != ""} { # remember result of last test in case the system must be restartet set last_test_result "" regexp {depot_autopilot\] ( [^\033]+)} $output ignored last_test_result regsub -all {<} $last_test_result {\<} last_test_result set failed_off [string first " failed" $last_test_result] set skipped_off [string first " skipped" $last_test_result] set ok_off [string first " ok" $last_test_result] if {$failed_off > 0 && ($skipped_off < 0 || $failed_off < $skipped_off) && ($ok_off < 0 || $failed_off < $ok_off)} { incr previous_failed } elseif {$skipped_off > 0 && ($ok_off < 0 || $skipped_off < $ok_off)} { incr previous_skipped } elseif {$ok_off > 0} { incr previous_succeeded } else { puts "Error: malformed test result" puts $last_test_result exit -1 } if {$previous_results != ""} { append previous_results \012 } append previous_results $last_test_result } # if the Autopilot is currently repeating, reset repeat-influenced variables if {[llength $test_pkgs] == 0} { init_test_setting init_previous_results } # determine timeout for the next timeout set min_timeout 10 regexp {depot_autopilot\] --- Run "(.*?)" \(max ([0-9]*?) } $output] ignore last_test_pkg min_timeout set timeout [expr $min_timeout + 10] set output "" } }