ExternalVendorCode/onefuzz
1
0
Fork 0
mirror of https://github.com/microsoft/onefuzz.git synced 2025-06-17 20:38:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

local run UI (#663)

Browse Source 
## Summary of the Pull Request

This PR add a UI to the local run. 
- The UI currently monitors the logs and  some of the directory created (the rest will be wired in a coming PR)
- pressing 'q' will quit the PR
- By default, the job directory is deleted when the ui quits unless the parameter 'keep_job_dir' is specified
This commit is contained in:
Cheick Keita
2021-04-06 14:44:37 -07:00
committed by GitHub
parent 794400adf1
commit 3e7b3df34f
34 changed files with 775 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
src/agent/onefuzz-agent/Cargo.toml
View File

@ -11,11 +11,13 @@ integration_test=[]
[dependencies]
anyhow = "1.0"
arraydeque = "0.4.5"
appinsights = "0.1"
async-trait = "0.1"
atexit = { path = "../atexit" }
backoff = { version = "0.3", features = ["async-std"] }
clap = "2.33"
tempfile = "3.2"
crossterm = "0.18"
env_logger = "0.8"
futures = "0.3"
hex = "0.4"
@ -25,18 +27,18 @@ num_cpus = "1.13"
reqwest = { version = "0.10", features = ["json", "stream"] }
serde = "1.0"
serde_json = "1.0"
onefuzz = { path = "../onefuzz" }
onefuzz-telemetry = { path = "../onefuzz-telemetry" }
path-absolutize = "3.0.6"
reqwest-retry = { path = "../reqwest-retry" }
remove_dir_all = "0.7"
stacktrace-parser = { path = "../stacktrace-parser" }
storage-queue = { path = "../storage-queue" }
tempfile = "3.2"
thiserror = "1.0"
tokio = { version = "0.2", features = ["full"] }
tokio-util = { version = "0.3", features = ["full"] }
tokio-stream = "0.1.3"
tui = { version = "0.14", default-features = false, features = ['crossterm'] }
url = { version = "2.2", features = ["serde"] }
uuid = { version = "0.8", features = ["serde", "v4"] }
onefuzz = { path = "../onefuzz" }
storage-queue = { path = "../storage-queue" }
reqwest-retry = { path = "../reqwest-retry" }
onefuzz-telemetry = { path = "../onefuzz-telemetry" }
stacktrace-parser = { path = "../stacktrace-parser" }
path-absolutize = "3.0.6"
atexit = { path = "../atexit" }
remove_dir_all = "0.7"
[dev-dependencies]
tempfile = "3.2"
uuid = { version = "0.8", features = ["serde", "v4"] }

7
src/agent/onefuzz-agent/build.rs
View File

@ -1,14 +1,17 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
use std::env;
use std::error::Error;
use std::fs::File;
use std::io::prelude::*;
use std::process::Command;
use std::{env, process::Stdio};
fn run_cmd(args: &[&str]) -> Result<String, Box<dyn Error>> {
let cmd = Command::new(args[0]).args(&args[1..]).output()?;
let cmd = Command::new(args[0])
.args(&args[1..])
.stdin(Stdio::null())
.output()?;
if cmd.status.success() {
Ok(String::from_utf8_lossy(&cmd.stdout).trim().to_string())
} else {

45
src/agent/onefuzz-agent/src/local/cmd.rs
View File

@ -5,12 +5,13 @@ use std::time::Duration;
use anyhow::Result;
use clap::{App, Arg, SubCommand};
use tokio::time::timeout;
use crossterm::tty::IsTty;
use tokio::{select, time::timeout};
use crate::local::{
common::add_common_config, generic_analysis, generic_crash_report, generic_generator,
libfuzzer, libfuzzer_coverage, libfuzzer_crash_report, libfuzzer_fuzz, libfuzzer_merge,
libfuzzer_regression, libfuzzer_test_input, radamsa, test_input,
libfuzzer_regression, libfuzzer_test_input, radamsa, test_input, tui::TerminalUi,
};
const RADAMSA: &str = "radamsa";
@ -25,16 +26,21 @@ const GENERIC_CRASH_REPORT: &str = "crash-report";
const GENERIC_GENERATOR: &str = "generator";
const GENERIC_ANALYSIS: &str = "analysis";
const GENERIC_TEST_INPUT: &str = "test-input";
const TIMEOUT: &str = "timeout";
pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
pub async fn run(args: clap::ArgMatches<'static>) -> Result<()> {
let running_duration = value_t!(args, TIMEOUT, u64).ok();
let run = async {
let terminal = if std::io::stdout().is_tty() {
Some(TerminalUi::init()?)
} else {
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
None
};
let event_sender = terminal.as_ref().map(|t| t.task_events.clone());
let command_run = tokio::spawn(async move {
match args.subcommand() {
(RADAMSA, Some(sub)) => radamsa::run(sub).await,
(LIBFUZZER, Some(sub)) => libfuzzer::run(sub).await,
(LIBFUZZER, Some(sub)) => libfuzzer::run(sub, event_sender).await,
(LIBFUZZER_FUZZ, Some(sub)) => libfuzzer_fuzz::run(sub).await,
(LIBFUZZER_COVERAGE, Some(sub)) => libfuzzer_coverage::run(sub).await,
(LIBFUZZER_CRASH_REPORT, Some(sub)) => libfuzzer_crash_report::run(sub).await,
@ -49,17 +55,29 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
anyhow::bail!("missing subcommand\nUSAGE: {}", args.usage());
}
}
};
});
if let Some(seconds) = running_duration {
if let Ok(run) = timeout(Duration::from_secs(seconds), run).await {
run
if let Some(terminal) = terminal {
let timeout = running_duration.map(Duration::from_secs);
let ui_run = tokio::spawn(terminal.run(timeout));
select! {
ui_result = ui_run => {
ui_result??
},
command_run_result = command_run => {
command_run_result??
}
};
Ok(())
} else if let Some(seconds) = running_duration {
if let Ok(run) = timeout(Duration::from_secs(seconds), command_run).await {
run?
} else {
info!("The running timeout period has elapsed");
Ok(())
}
} else {
run.await
command_run.await?
}
}
@ -70,8 +88,7 @@ pub fn args(name: &str) -> App<'static, 'static> {
Arg::with_name(TIMEOUT)
.long(TIMEOUT)
.help("The maximum running time in seconds")
.takes_value(true)
.required(false),
.takes_value(true),
)
.subcommand(add_common_config(radamsa::args(RADAMSA)))
.subcommand(add_common_config(libfuzzer::args(LIBFUZZER)))

66
src/agent/onefuzz-agent/src/local/common.rs
View File

@ -11,9 +11,12 @@ use reqwest::Url;
use std::task::Poll;
use std::{
collections::HashMap,
env::current_dir,
path::{Path, PathBuf},
time::Duration,
};
use tokio::stream::StreamExt;
use tokio::{sync::mpsc::UnboundedSender, task::JoinHandle, time::delay_for};
use uuid::Uuid;
pub const SETUP_DIR: &str = "setup_dir";
@ -62,6 +65,12 @@ pub enum CmdType {
// Supervisor,
}
#[derive(Clone, Debug)]
pub struct LocalContext {
pub job_path: PathBuf,
pub common_config: CommonConfig,
}
pub fn get_hash_map(args: &clap::ArgMatches<'_>, name: &str) -> Result<HashMap<String, String>> {
let mut env = HashMap::new();
for opt in args.values_of_lossy(name).unwrap_or_default() {
@ -203,7 +212,7 @@ pub fn get_synced_dir(
// fuzzing tasks from generating random task id to using UUID::nil(). This
// enables making the one-shot crash report generation, which isn't really a task,
// consistent across multiple runs.
pub fn build_common_config(args: &ArgMatches<'_>, generate_task_id: bool) -> Result<CommonConfig> {
pub fn build_local_context(args: &ArgMatches<'_>, generate_task_id: bool) -> Result<LocalContext> {
let job_id = get_uuid("job_id", args).unwrap_or_else(|_| Uuid::nil());
let task_id = get_uuid("task_id", args).unwrap_or_else(|_| {
if generate_task_id {
@ -229,14 +238,19 @@ pub fn build_common_config(args: &ArgMatches<'_>, generate_task_id: bool) -> Res
register_cleanup(job_id)?;
}
let config = CommonConfig {
let common_config = CommonConfig {
job_id,
task_id,
instance_id,
setup_dir,
..Default::default()
};
Ok(config)
let current_dir = current_dir()?;
let job_path = current_dir.join(format!("{}", job_id));
Ok(LocalContext {
job_path,
common_config,
})
}
/// Information about a local path being monitored
@ -302,3 +316,49 @@ pub async fn wait_for_dir(path: impl AsRef<Path>) -> Result<()> {
)
.await
}
pub fn spawn_file_count_monitor(
dir: PathBuf,
sender: UnboundedSender<UiEvent>,
) -> JoinHandle<Result<()>> {
tokio::spawn(async move {
wait_for_dir(&dir).await?;
loop {
let mut rd = tokio::fs::read_dir(&dir).await?;
let mut count: usize = 0;
while let Some(Ok(entry)) = rd.next().await {
if entry.path().is_file() {
count += 1;
}
}
if sender
.send(UiEvent::FileCount {
dir: dir.clone(),
count,
})
.is_err()
{
return Ok(());
}
delay_for(Duration::from_secs(5)).await;
}
})
}
pub fn monitor_file_urls(
urls: &[Option<impl AsRef<Path>>],
event_sender: UnboundedSender<UiEvent>,
) -> Vec<JoinHandle<Result<()>>> {
urls.iter()
.filter_map(|x| x.as_ref())
.map(|path| spawn_file_count_monitor(path.as_ref().into(), event_sender.clone()))
.collect::<Vec<_>>()
}
#[derive(Debug)]
pub enum UiEvent {
FileCount { dir: PathBuf, count: usize },
}

6
src/agent/onefuzz-agent/src/local/generic_analysis.rs
View File

@ -3,7 +3,7 @@
use crate::{
local::common::{
build_common_config, get_cmd_arg, get_cmd_exe, get_hash_map, get_synced_dir, CmdType,
build_local_context, get_cmd_arg, get_cmd_exe, get_hash_map, get_synced_dir, CmdType,
ANALYSIS_DIR, ANALYZER_ENV, ANALYZER_EXE, ANALYZER_OPTIONS, CRASHES_DIR, NO_REPRO_DIR,
REPORTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TOOLS_DIR, UNIQUE_REPORTS_DIR,
},
@ -60,8 +60,8 @@ pub fn build_analysis_config(
}
pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
let common = build_common_config(args, true)?;
let config = build_analysis_config(args, None, common)?;
let context = build_local_context(args, true)?;
let config = build_analysis_config(args, None, context.common_config.clone())?;
run_analysis(config).await
}

6
src/agent/onefuzz-agent/src/local/generic_crash_report.rs
View File

@ -3,7 +3,7 @@
use crate::{
local::common::{
build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, CmdType,
build_local_context, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, CmdType,
CHECK_ASAN_LOG, CHECK_RETRY_COUNT, CRASHES_DIR, DISABLE_CHECK_DEBUGGER,
DISABLE_CHECK_QUEUE, NO_REPRO_DIR, REPORTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS,
TARGET_TIMEOUT, UNIQUE_REPORTS_DIR,
@ -71,8 +71,8 @@ pub fn build_report_config(
}
pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
let common = build_common_config(args, true)?;
let config = build_report_config(args, None, common)?;
let context = build_local_context(args, true)?;
let config = build_report_config(args, None, context.common_config.clone())?;
ReportTask::new(config).managed_run().await
}

6
src/agent/onefuzz-agent/src/local/generic_generator.rs
View File

@ -3,7 +3,7 @@
use crate::{
local::common::{
build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir,
build_local_context, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir,
get_synced_dirs, CmdType, CHECK_ASAN_LOG, CHECK_RETRY_COUNT, CRASHES_DIR,
DISABLE_CHECK_DEBUGGER, GENERATOR_ENV, GENERATOR_EXE, GENERATOR_OPTIONS, READONLY_INPUTS,
RENAME_OUTPUT, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT, TOOLS_DIR,
@ -60,8 +60,8 @@ pub fn build_fuzz_config(args: &clap::ArgMatches<'_>, common: CommonConfig) -> R
}
pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
let common = build_common_config(args, true)?;
let config = build_fuzz_config(args, common)?;
let context = build_local_context(args, true)?;
let config = build_fuzz_config(args, context.common_config.clone())?;
GeneratorTask::new(config).run().await
}

74
src/agent/onefuzz-agent/src/local/libfuzzer.rs
View File

@ -4,8 +4,8 @@
use crate::{
local::{
common::{
build_common_config, wait_for_dir, DirectoryMonitorQueue, ANALYZER_EXE, COVERAGE_DIR,
REGRESSION_REPORTS_DIR, UNIQUE_REPORTS_DIR,
build_local_context, monitor_file_urls, wait_for_dir, DirectoryMonitorQueue,
ANALYZER_EXE, COVERAGE_DIR, REGRESSION_REPORTS_DIR, UNIQUE_REPORTS_DIR,
},
generic_analysis::{build_analysis_config, build_shared_args as build_analysis_args},
libfuzzer_coverage::{build_coverage_config, build_shared_args as build_coverage_args},
@ -25,12 +25,27 @@ use anyhow::Result;
use clap::{App, SubCommand};
use onefuzz::utils::try_wait_all_join_handles;
use std::collections::HashSet;
use tokio::task::spawn;
use tokio::{sync::mpsc::UnboundedSender, task::spawn};
use uuid::Uuid;
pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
let common = build_common_config(args, true)?;
let fuzz_config = build_fuzz_config(args, common.clone())?;
use super::common::UiEvent;
pub async fn run(
args: &clap::ArgMatches<'_>,
event_sender: Option<UnboundedSender<UiEvent>>,
) -> Result<()> {
let mut task_handles = vec![];
let context = build_local_context(args, true)?;
let fuzz_config = build_fuzz_config(args, context.common_config.clone())?;
if let Some(event_sender) = event_sender.clone() {
task_handles.append(&mut monitor_file_urls(
&[
fuzz_config.crashes.url.as_file_path(),
fuzz_config.inputs.url.as_file_path(),
],
event_sender,
));
}
let crash_dir = fuzz_config
.crashes
.url
@ -45,6 +60,7 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
wait_for_dir(&crash_dir).await?;
task_handles.push(fuzz_task);
if args.is_present(UNIQUE_REPORTS_DIR) {
let crash_report_input_monitor =
DirectoryMonitorQueue::start_monitoring(crash_dir.clone()).await?;
@ -54,11 +70,32 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
Some(crash_report_input_monitor.queue_client),
CommonConfig {
task_id: Uuid::new_v4(),
..common.clone()
..context.common_config.clone()
},
)?;
if let Some(event_sender) = event_sender.clone() {
task_handles.append(&mut monitor_file_urls(
&[
report_config
.no_repro
.clone()
.and_then(|u| u.url.as_file_path()),
report_config
.reports
.clone()
.and_then(|u| u.url.as_file_path()),
report_config
.unique_reports
.clone()
.and_then(|u| u.url.as_file_path()),
],
event_sender,
));
}
let mut report = ReportTask::new(report_config);
let report_task = spawn(async move { report.managed_run().await });
task_handles.push(report_task);
task_handles.push(crash_report_input_monitor.handle);
}
@ -72,9 +109,26 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
Some(coverage_input_monitor.queue_client),
CommonConfig {
task_id: Uuid::new_v4(),
..common.clone()
..context.common_config.clone()
},
)?;
if let Some(event_sender) = event_sender {
task_handles.append(&mut monitor_file_urls(
&coverage_config
.readonly_inputs
.iter()
.cloned()
.map(|input| input.url.as_file_path())
.collect::<Vec<_>>(),
event_sender.clone(),
));
task_handles.append(&mut monitor_file_urls(
&[coverage_config.coverage.url.as_file_path()],
event_sender,
));
}
let mut coverage = CoverageTask::new(coverage_config);
let coverage_task = spawn(async move { coverage.managed_run().await });
@ -89,7 +143,7 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
Some(analysis_input_monitor.queue_client),
CommonConfig {
task_id: Uuid::new_v4(),
..common.clone()
..context.common_config.clone()
},
)?;
let analysis_task = spawn(async move { run_analysis(analysis_config).await });
@ -103,7 +157,7 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
args,
CommonConfig {
task_id: Uuid::new_v4(),
..common
..context.common_config.clone()
},
)?;
let regression = LibFuzzerRegressionTask::new(regression_config);

7
src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs
View File

@ -3,7 +3,7 @@
use crate::{
local::common::{
build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir,
build_local_context, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir,
get_synced_dirs, CmdType, CHECK_FUZZER_HELP, COVERAGE_DIR, INPUTS_DIR, READONLY_INPUTS,
TARGET_ENV, TARGET_EXE, TARGET_OPTIONS,
},
@ -55,8 +55,9 @@ pub fn build_coverage_config(
}
pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
let common = build_common_config(args, true)?;
let config = build_coverage_config(args, false, None, common)?;
let context = build_local_context(args, true)?;
let config = build_coverage_config(args, false, None, context.common_config.clone())?;
let mut task = CoverageTask::new(config);
task.managed_run().await
}

6
src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs
View File

@ -3,7 +3,7 @@
use crate::{
local::common::{
build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, CmdType,
build_local_context, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, CmdType,
CHECK_FUZZER_HELP, CHECK_RETRY_COUNT, CRASHES_DIR, DISABLE_CHECK_QUEUE, NO_REPRO_DIR,
REPORTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT, UNIQUE_REPORTS_DIR,
},
@ -63,8 +63,8 @@ pub fn build_report_config(
}
pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
let common = build_common_config(args, true)?;
let config = build_report_config(args, None, common)?;
let context = build_local_context(args, true)?;
let config = build_report_config(args, None, context.common_config.clone())?;
ReportTask::new(config).managed_run().await
}

6
src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs
View File

@ -3,7 +3,7 @@
use crate::{
local::common::{
build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, CmdType,
build_local_context, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, CmdType,
CHECK_FUZZER_HELP, CRASHES_DIR, INPUTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS,
TARGET_WORKERS,
},
@ -50,8 +50,8 @@ pub fn build_fuzz_config(args: &clap::ArgMatches<'_>, common: CommonConfig) -> R
}
pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
let common = build_common_config(args, true)?;
let config = build_fuzz_config(args, common)?;
let context = build_local_context(args, true)?;
let config = build_fuzz_config(args, context.common_config.clone())?;
LibFuzzerFuzzTask::new(config)?.run().await
}

6
src/agent/onefuzz-agent/src/local/libfuzzer_merge.rs
View File

@ -3,7 +3,7 @@
use crate::{
local::common::{
build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir,
build_local_context, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir,
get_synced_dirs, CmdType, ANALYSIS_INPUTS, ANALYSIS_UNIQUE_INPUTS, CHECK_FUZZER_HELP,
INPUTS_DIR, PRESERVE_EXISTING_OUTPUTS, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS,
},
@ -45,8 +45,8 @@ pub fn build_merge_config(
}
pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
let common = build_common_config(args, true)?;
let config = build_merge_config(args, None, common)?;
let context = build_local_context(args, true)?;
let config = build_merge_config(args, None, context.common_config.clone())?;
spawn(std::sync::Arc::new(config)).await
}

6
src/agent/onefuzz-agent/src/local/libfuzzer_regression.rs
View File

@ -3,7 +3,7 @@
use crate::{
local::common::{
build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, CmdType,
build_local_context, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, CmdType,
CHECK_FUZZER_HELP, CHECK_RETRY_COUNT, COVERAGE_DIR, CRASHES_DIR, NO_REPRO_DIR,
REGRESSION_REPORTS_DIR, REPORTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS,
TARGET_TIMEOUT, UNIQUE_REPORTS_DIR,
@ -65,8 +65,8 @@ pub fn build_regression_config(
}
pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
let common = build_common_config(args, true)?;
let config = build_regression_config(args, common)?;
let context = build_local_context(args, true)?;
let config = build_regression_config(args, context.common_config.clone())?;
LibFuzzerRegressionTask::new(config).run().await
}

10
src/agent/onefuzz-agent/src/local/libfuzzer_test_input.rs
View File

@ -3,7 +3,7 @@
use crate::{
local::common::{
build_common_config, get_cmd_arg, get_cmd_env, CmdType, CHECK_RETRY_COUNT, TARGET_ENV,
build_local_context, get_cmd_arg, get_cmd_env, CmdType, CHECK_RETRY_COUNT, TARGET_ENV,
TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT,
},
tasks::report::libfuzzer_report::{test_input, TestInputArgs},
@ -13,7 +13,7 @@ use clap::{App, Arg, SubCommand};
use std::path::PathBuf;
pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
let common = build_common_config(args, true)?;
let context = build_local_context(args, true)?;
let target_exe = value_t!(args, TARGET_EXE, PathBuf)?;
let target_env = get_cmd_env(CmdType::Target, args)?;
@ -28,11 +28,11 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
target_options: &target_options,
input_url: None,
input: input.as_path(),
job_id: common.job_id,
task_id: common.task_id,
job_id: context.common_config.job_id,
task_id: context.common_config.task_id,
target_timeout,
check_retry_count,
setup_dir: &common.setup_dir,
setup_dir: &context.common_config.setup_dir,
minimized_stack_depth: None,
};

1
src/agent/onefuzz-agent/src/local/mod.rs
View File

@ -15,3 +15,4 @@ pub mod libfuzzer_regression;
pub mod libfuzzer_test_input;
pub mod radamsa;
pub mod test_input;
pub mod tui;

9
src/agent/onefuzz-agent/src/local/radamsa.rs
View File

@ -3,7 +3,7 @@
use crate::{
local::{
common::{build_common_config, DirectoryMonitorQueue},
common::{build_local_context, DirectoryMonitorQueue},
generic_crash_report::{build_report_config, build_shared_args as build_crash_args},
generic_generator::{build_fuzz_config, build_shared_args as build_fuzz_args},
},
@ -14,11 +14,12 @@ use clap::{App, SubCommand};
use onefuzz::utils::try_wait_all_join_handles;
use std::collections::HashSet;
use tokio::task::spawn;
use uuid::Uuid;
pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
let common = build_common_config(args, true)?;
let fuzz_config = build_fuzz_config(args, common.clone())?;
let context = build_local_context(args, true)?;
let fuzz_config = build_fuzz_config(args, context.common_config.clone())?;
let crash_dir = fuzz_config
.crashes
.url
@ -34,7 +35,7 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
Some(crash_report_input_monitor.queue_client),
CommonConfig {
task_id: Uuid::new_v4(),
..common
..context.common_config.clone()
},
)?;
let report_task = spawn(async move { ReportTask::new(report_config).managed_run().await });

10
src/agent/onefuzz-agent/src/local/test_input.rs
View File

@ -3,7 +3,7 @@
use crate::{
local::common::{
build_common_config, get_cmd_arg, get_cmd_env, CmdType, CHECK_ASAN_LOG, CHECK_RETRY_COUNT,
build_local_context, get_cmd_arg, get_cmd_env, CmdType, CHECK_ASAN_LOG, CHECK_RETRY_COUNT,
DISABLE_CHECK_DEBUGGER, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT,
},
tasks::report::generic::{test_input, TestInputArgs},
@ -13,7 +13,7 @@ use clap::{App, Arg, SubCommand};
use std::path::PathBuf;
pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
let common = build_common_config(args, false)?;
let context = build_local_context(args, false)?;
let target_exe = value_t!(args, TARGET_EXE, PathBuf)?;
let target_env = get_cmd_env(CmdType::Target, args)?;
@ -30,11 +30,11 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
target_options: &target_options,
input_url: None,
input: input.as_path(),
job_id: common.job_id,
task_id: common.task_id,
job_id: context.common_config.job_id,
task_id: context.common_config.task_id,
target_timeout,
check_retry_count,
setup_dir: &common.setup_dir,
setup_dir: &context.common_config.setup_dir,
minimized_stack_depth: None,
check_asan_log,
check_debugger,

384
src/agent/onefuzz-agent/src/local/tui.rs Normal file
View File

@ -0,0 +1,384 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
use crate::local::common::UiEvent;
use anyhow::{Context, Result};
use crossterm::{
event::{self, Event, KeyCode},
execute,
terminal::{disable_raw_mode, enable_raw_mode, EnterAlternateScreen, LeaveAlternateScreen},
};
use futures::{StreamExt, TryStreamExt};
use log::Level;
use onefuzz::utils::try_wait_all_join_handles;
use std::{
collections::HashMap,
io::{self, Stdout, Write},
path::PathBuf,
thread::{self, JoinHandle},
time::Duration,
};
use tokio::{
sync::mpsc::{self, UnboundedReceiver},
time,
};
use tui::{
backend::CrosstermBackend,
layout::{Constraint, Corner, Direction, Layout},
style::{Color, Modifier, Style},
text::{Span, Spans},
widgets::{Block, Borders},
widgets::{List, ListItem, ListState},
Terminal,
};
use arraydeque::{ArrayDeque, Wrapping};
#[derive(Debug, thiserror::Error)]
enum UiLoopError {
#[error("program exiting")]
Exit,
#[error("error")]
Anyhow(anyhow::Error),
}
impl From<anyhow::Error> for UiLoopError {
fn from(e: anyhow::Error) -> Self {
Self::Anyhow(e)
}
}
impl From<std::io::Error> for UiLoopError {
fn from(e: std::io::Error) -> Self {
Self::Anyhow(e.into())
}
}
/// Maximum number of log message to display, arbitrarily chosen
const LOGS_BUFFER_SIZE: usize = 100;
const TICK_RATE: Duration = Duration::from_millis(250);
/// Event driving the refresh of the UI
#[derive(Debug)]
enum TerminalEvent {
Input(Event),
Tick,
FileCount { dir: PathBuf, count: usize },
Quit,
}
struct UiLoopState {
pub logs: ArrayDeque<[(Level, String); LOGS_BUFFER_SIZE], Wrapping>,
pub file_count: HashMap<PathBuf, usize>,
pub file_count_state: ListState,
pub file_monitors: Vec<JoinHandle<Result<()>>>,
pub log_event_receiver: mpsc::UnboundedReceiver<(Level, String)>,
pub terminal: Terminal<CrosstermBackend<Stdout>>,
}
impl UiLoopState {
fn new(
terminal: Terminal<CrosstermBackend<Stdout>>,
log_event_receiver: mpsc::UnboundedReceiver<(Level, String)>,
) -> Self {
Self {
log_event_receiver,
logs: Default::default(),
file_count: Default::default(),
file_count_state: Default::default(),
file_monitors: Default::default(),
terminal,
}
}
}
pub struct TerminalUi {
pub task_events: mpsc::UnboundedSender<UiEvent>,
task_event_receiver: mpsc::UnboundedReceiver<UiEvent>,
ui_event_tx: mpsc::UnboundedSender<TerminalEvent>,
ui_event_rx: mpsc::UnboundedReceiver<TerminalEvent>,
}
impl TerminalUi {
pub fn init() -> Result<Self> {
let (task_event_sender, task_event_receiver) = mpsc::unbounded_channel();
let (ui_event_tx, ui_event_rx) = mpsc::unbounded_channel();
Ok(Self {
task_events: task_event_sender,
task_event_receiver,
ui_event_tx,
ui_event_rx,
})
}
pub async fn run(self, timeout: Option<Duration>) -> Result<()> {
enable_raw_mode()?;
let mut stdout = io::stdout();
execute!(stdout, EnterAlternateScreen)?;
let backend = CrosstermBackend::new(stdout);
let mut terminal = Terminal::new(backend)?;
terminal.clear()?;
let (log_event_sender, log_event_receiver) = mpsc::unbounded_channel();
let initial_state = UiLoopState::new(terminal, log_event_receiver);
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info"))
.format(move |_buf, record| {
let _r = log_event_sender.send((record.level(), format!("{}", record.args())));
Ok(())
})
.init();
let tick_event_tx_clone = self.ui_event_tx.clone();
let tick_event_handle =
tokio::spawn(async { Self::ticking(tick_event_tx_clone).await.context("ticking") });
let keyboard_ui_event_tx = self.ui_event_tx.clone();
let _keyboard_event_handle = Self::read_keyboard_events(keyboard_ui_event_tx);
let task_event_receiver = self.task_event_receiver;
let ui_event_tx = self.ui_event_tx.clone();
let external_event_handle =
tokio::spawn(Self::read_commands(ui_event_tx, task_event_receiver));
let ui_loop = tokio::spawn(Self::ui_loop(initial_state, self.ui_event_rx));
let mut task_handles = vec![tick_event_handle, ui_loop, external_event_handle];
if let Some(timeout) = timeout {
let ui_event_tx = self.ui_event_tx.clone();
let timeout_task = tokio::spawn(async move {
time::delay_for(timeout).await;
let _ = ui_event_tx.send(TerminalEvent::Quit);
Ok(())
});
task_handles.push(timeout_task);
}
try_wait_all_join_handles(task_handles)
.await
.context("ui_loop")?;
Ok(())
}
async fn ticking(ui_event_tx: mpsc::UnboundedSender<TerminalEvent>) -> Result<()> {
let mut interval = tokio::time::interval(TICK_RATE);
loop {
interval.tick().await;
if let Err(_err) = ui_event_tx.send(TerminalEvent::Tick) {
break;
}
}
Ok(())
}
fn read_keyboard_events(
ui_event_tx: mpsc::UnboundedSender<TerminalEvent>,
) -> JoinHandle<Result<()>> {
thread::spawn(move || loop {
if event::poll(Duration::from_secs(1))? {
let event = event::read()?;
if let Err(_err) = ui_event_tx.send(TerminalEvent::Input(event)) {
return Ok(());
}
}
})
}
async fn read_commands(
ui_event_tx: mpsc::UnboundedSender<TerminalEvent>,
mut external_event_rx: mpsc::UnboundedReceiver<UiEvent>,
) -> Result<()> {
while let Some(UiEvent::FileCount { dir, count }) = external_event_rx.recv().await {
if ui_event_tx
.send(TerminalEvent::FileCount { dir, count })
.is_err()
{
break;
}
}
Ok(())
}
fn take_available_logs<T>(
receiver: &mut UnboundedReceiver<T>,
size: usize,
buffer: &mut ArrayDeque<[T; LOGS_BUFFER_SIZE], Wrapping>,
) {
let mut count = 0;
while let Ok(v) = receiver.try_recv() {
count += 1;
buffer.push_front(v);
if count >= size {
break;
}
}
}
async fn refresh_ui(ui_state: UiLoopState) -> Result<UiLoopState, UiLoopError> {
let mut logs = ui_state.logs;
let mut file_count_state = ui_state.file_count_state;
let file_count = ui_state.file_count;
let mut log_event_receiver = ui_state.log_event_receiver;
let mut terminal = ui_state.terminal;
Self::take_available_logs(&mut log_event_receiver, 10, &mut logs);
terminal.draw(|f| {
let chunks = Layout::default()
.direction(Direction::Vertical)
.constraints([Constraint::Percentage(25), Constraint::Percentage(75)].as_ref())
.split(f.size());
let mut sorted_file_count = file_count.iter().collect::<Vec<_>>();
sorted_file_count.sort_by(|(p1, _), (p2, _)| p1.cmp(p2));
let files = sorted_file_count
.iter()
.map(|(path, count)| {
ListItem::new(Spans::from(vec![
Span::raw(
path.file_name()
.map(|f| f.to_string_lossy())
.unwrap_or_default(),
),
Span::raw(": "),
Span::raw(format!("{}", count)),
]))
})
.collect::<Vec<_>>();
let log_list = List::new(files)
.block(Block::default().borders(Borders::ALL).title("files"))
.highlight_style(Style::default().add_modifier(Modifier::BOLD))
.start_corner(Corner::TopLeft);
f.render_stateful_widget(log_list, chunks[0], &mut file_count_state);
let log_items = logs
.iter()
.map(|(level, log)| {
let style = match level {
Level::Debug => Style::default().fg(Color::Magenta),
Level::Error => Style::default().fg(Color::Red),
Level::Warn => Style::default().fg(Color::Yellow),
Level::Info => Style::default().fg(Color::Blue),
Level::Trace => Style::default(),
};
ListItem::new(Spans::from(vec![
Span::styled(format!("{:<9}", level), style),
Span::raw(" "),
Span::raw(log),
]))
})
.collect::<Vec<_>>();
let log_list = List::new(log_items)
.block(Block::default().borders(Borders::ALL).title("Logs"))
.start_corner(Corner::BottomLeft);
f.render_widget(log_list, chunks[1]);
})?;
Ok(UiLoopState {
logs,
file_count_state,
file_count,
terminal,
log_event_receiver,
..ui_state
})
}
async fn on_key_down(ui_state: UiLoopState) -> Result<UiLoopState, UiLoopError> {
let mut file_count_state = ui_state.file_count_state;
let count = ui_state.file_count.len();
let i = file_count_state
.selected()
.map(|i| {
if count == 0 {
0
} else {
(i + count + 1) % count
}
})
.unwrap_or_default();
file_count_state.select(Some(i));
Ok(UiLoopState {
file_count_state,
..ui_state
})
}
async fn on_key_up(ui_state: UiLoopState) -> Result<UiLoopState, UiLoopError> {
let mut file_count_state = ui_state.file_count_state;
let count = ui_state.file_count.len();
let i = file_count_state
.selected()
.map(|i| {
if count == 0 {
0
} else {
(i + count - 1) % count
}
})
.unwrap_or_default();
file_count_state.select(Some(i));
Ok(UiLoopState {
file_count_state,
..ui_state
})
}
async fn on_quit(ui_state: UiLoopState) -> Result<UiLoopState, UiLoopError> {
let mut terminal = ui_state.terminal;
disable_raw_mode().map_err(|e| anyhow!("{:?}", e))?;
execute!(terminal.backend_mut(), LeaveAlternateScreen).map_err(|e| anyhow!("{:?}", e))?;
terminal.show_cursor()?;
Err(UiLoopError::Exit)
}
async fn on_file_count(
ui_state: UiLoopState,
dir: PathBuf,
count: usize,
) -> Result<UiLoopState, UiLoopError> {
let mut file_count = ui_state.file_count;
file_count.insert(dir, count);
Ok(UiLoopState {
file_count,
..ui_state
})
}
async fn ui_loop(
initial_state: UiLoopState,
ui_event_rx: mpsc::UnboundedReceiver<TerminalEvent>,
) -> Result<()> {
let loop_result = ui_event_rx
.map(Ok)
.try_fold(initial_state, |ui_state, event| async {
match event {
TerminalEvent::Tick => Self::refresh_ui(ui_state).await,
TerminalEvent::Input(Event::Key(k)) => match k.code {
KeyCode::Char('q') => Self::on_quit(ui_state).await,
KeyCode::Down => Self::on_key_down(ui_state).await,
KeyCode::Up => Self::on_key_up(ui_state).await,
_ => Ok(ui_state),
},
TerminalEvent::FileCount { dir, count } => {
Self::on_file_count(ui_state, dir, count).await
}
TerminalEvent::Quit => Self::on_quit(ui_state).await,
_ => Ok(ui_state),
}
})
.await;
match loop_result {
Err(UiLoopError::Exit) | Ok(_) => Ok(()),
Err(UiLoopError::Anyhow(e)) => Err(e),
}
}
}

6
src/agent/onefuzz-agent/src/main.rs
View File

@ -22,8 +22,6 @@ const LOCAL_CMD: &str = "local";
const MANAGED_CMD: &str = "managed";
fn main() -> Result<()> {
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
let built_version = format!(
"{} onefuzz:{} git:{}",
crate_version!(),
@ -45,10 +43,10 @@ fn main() -> Result<()> {
result
}
async fn run(args: ArgMatches<'_>) -> Result<()> {
async fn run(args: ArgMatches<'static>) -> Result<()> {
match args.subcommand() {
(LICENSE_CMD, Some(_)) => licenses(),
(LOCAL_CMD, Some(sub)) => local::cmd::run(sub).await,
(LOCAL_CMD, Some(sub)) => local::cmd::run(sub.to_owned()).await,
(MANAGED_CMD, Some(sub)) => managed::cmd::run(sub).await,
_ => {
anyhow::bail!("missing subcommand\nUSAGE: {}", args.usage());

1
src/agent/onefuzz-agent/src/managed/cmd.rs
View File

@ -7,6 +7,7 @@ use clap::{App, Arg, SubCommand};
use std::path::PathBuf;
pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
let config_path = value_t!(args, "config", PathBuf)?;
let setup_dir = value_t!(args, "setup_dir", PathBuf)?;
let config = Config::from_file(config_path, setup_dir)?;

2
src/agent/onefuzz-agent/src/tasks/coverage/recorder.rs
View File

@ -123,6 +123,7 @@ impl CoverageRecorder {
test_input.to_string_lossy(),
output.to_string_lossy(),
))
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.kill_on_drop(true);
@ -160,6 +161,7 @@ impl CoverageRecorder {
.arg(cdb_cmd)
.arg(&self.config.target_exe)
.arg(test_input)
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.kill_on_drop(true);

3
src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs
View File

@ -196,8 +196,9 @@ impl LibFuzzerFuzzTask {
&self.config.common.setup_dir,
);
let mut running = fuzzer.fuzz(crash_dir.path(), local_inputs, &inputs)?;
let running_id = running.id();
let sys_info = task::spawn(report_fuzzer_sys_info(worker_id, run_id, running.id()));
let sys_info = task::spawn(report_fuzzer_sys_info(worker_id, run_id, running_id));
// Splitting borrow.
let stderr = running

1
src/agent/onefuzz-agent/src/tasks/fuzz/supervisor.rs
View File

@ -211,6 +211,7 @@ async fn start_supervisor(
let cmd = cmd
.kill_on_drop(true)
.env_remove("RUST_LOG")
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped());

1
src/agent/onefuzz-agent/src/tasks/merge/generic.rs
View File

@ -155,6 +155,7 @@ async fn merge(config: &Config, output_dir: impl AsRef<Path>) -> Result<()> {
cmd.kill_on_drop(true)
.env_remove("RUST_LOG")
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped());