local run UI (#663)

## Summary of the Pull Request

This PR add a UI to the local run. 
- The UI currently monitors the logs and  some of the directory created (the rest will be wired in a coming PR)
- pressing 'q' will quit the PR
- By default, the job directory is deleted when the ui quits unless the parameter 'keep_job_dir' is specified
This commit is contained in:
Cheick Keita
2021-04-06 14:44:37 -07:00
committed by GitHub
parent 794400adf1
commit 3e7b3df34f
34 changed files with 775 additions and 103 deletions

151
src/agent/Cargo.lock generated
View File

@ -73,6 +73,12 @@ dependencies = [
"uuid", "uuid",
] ]
[[package]]
name = "arraydeque"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0ffd3d69bd89910509a5d31d1f1353f38ccffdd116dd0099bbd6627f7bd8ad8"
[[package]] [[package]]
name = "async-channel" name = "async-channel"
version = "1.6.1" version = "1.6.1"
@ -331,6 +337,12 @@ version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "631ae5198c9be5e753e5cc215e1bd73c2b466a3565173db433f52bb9d3e66dba" checksum = "631ae5198c9be5e753e5cc215e1bd73c2b466a3565173db433f52bb9d3e66dba"
[[package]]
name = "cassowary"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53"
[[package]] [[package]]
name = "cc" name = "cc"
version = "1.0.67" version = "1.0.67"
@ -521,6 +533,31 @@ dependencies = [
"lazy_static", "lazy_static",
] ]
[[package]]
name = "crossterm"
version = "0.18.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e86d73f2a0b407b5768d10a8c720cf5d2df49a9efc10ca09176d201ead4b7fb"
dependencies = [
"bitflags",
"crossterm_winapi",
"lazy_static",
"libc",
"mio 0.7.10",
"parking_lot",
"signal-hook",
"winapi 0.3.9",
]
[[package]]
name = "crossterm_winapi"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2265c3f8e080075d9b6417aa72293fc71662f34b4af2612d8d1b074d29510db"
dependencies = [
"winapi 0.3.9",
]
[[package]] [[package]]
name = "ctor" name = "ctor"
version = "0.1.19" version = "0.1.19"
@ -1014,7 +1051,7 @@ dependencies = [
"http", "http",
"indexmap", "indexmap",
"slab", "slab",
"tokio", "tokio 0.2.25",
"tokio-util", "tokio-util",
"tracing", "tracing",
"tracing-futures", "tracing-futures",
@ -1118,7 +1155,7 @@ dependencies = [
"itoa", "itoa",
"pin-project", "pin-project",
"socket2", "socket2",
"tokio", "tokio 0.2.25",
"tower-service", "tower-service",
"tracing", "tracing",
"want", "want",
@ -1133,7 +1170,7 @@ dependencies = [
"bytes 0.5.6", "bytes 0.5.6",
"hyper", "hyper",
"native-tls", "native-tls",
"tokio", "tokio 0.2.25",
"tokio-tls", "tokio-tls",
] ]
@ -1313,6 +1350,15 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "lock_api"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312"
dependencies = [
"scopeguard",
]
[[package]] [[package]]
name = "log" name = "log"
version = "0.4.14" version = "0.4.14"
@ -1687,7 +1733,7 @@ dependencies = [
"strum_macros", "strum_macros",
"sysinfo 0.16.4", "sysinfo 0.16.4",
"tempfile", "tempfile",
"tokio", "tokio 0.2.25",
"tokio-util", "tokio-util",
"url", "url",
"urlparse", "urlparse",
@ -1701,10 +1747,12 @@ version = "0.2.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"appinsights", "appinsights",
"arraydeque",
"async-trait", "async-trait",
"atexit", "atexit",
"backoff", "backoff",
"clap", "clap",
"crossterm",
"env_logger", "env_logger",
"futures", "futures",
"hex", "hex",
@ -1722,8 +1770,11 @@ dependencies = [
"stacktrace-parser", "stacktrace-parser",
"storage-queue", "storage-queue",
"tempfile", "tempfile",
"tokio", "thiserror",
"tokio 0.2.25",
"tokio-stream",
"tokio-util", "tokio-util",
"tui",
"url", "url",
"uuid", "uuid",
] ]
@ -1748,7 +1799,7 @@ dependencies = [
"serde_json", "serde_json",
"storage-queue", "storage-queue",
"structopt", "structopt",
"tokio", "tokio 0.2.25",
"url", "url",
"users", "users",
"uuid", "uuid",
@ -1762,7 +1813,7 @@ dependencies = [
"iced-x86", "iced-x86",
"log", "log",
"serde", "serde",
"tokio", "tokio 0.2.25",
"uuid", "uuid",
"z3-sys", "z3-sys",
] ]
@ -1831,6 +1882,31 @@ version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72"
[[package]]
name = "parking_lot"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb"
dependencies = [
"instant",
"lock_api",
"parking_lot_core",
]
[[package]]
name = "parking_lot_core"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018"
dependencies = [
"cfg-if 1.0.0",
"instant",
"libc",
"redox_syscall",
"smallvec",
"winapi 0.3.9",
]
[[package]] [[package]]
name = "paste" name = "paste"
version = "0.1.18" version = "0.1.18"
@ -2278,7 +2354,7 @@ dependencies = [
"serde", "serde",
"serde_json", "serde_json",
"serde_urlencoded", "serde_urlencoded",
"tokio", "tokio 0.2.25",
"tokio-tls", "tokio-tls",
"url", "url",
"wasm-bindgen", "wasm-bindgen",
@ -2297,7 +2373,7 @@ dependencies = [
"log", "log",
"onefuzz-telemetry", "onefuzz-telemetry",
"reqwest", "reqwest",
"tokio", "tokio 0.2.25",
] ]
[[package]] [[package]]
@ -2499,6 +2575,17 @@ dependencies = [
"opaque-debug", "opaque-debug",
] ]
[[package]]
name = "signal-hook"
version = "0.1.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e31d442c16f047a671b5a71e2161d6e68814012b7f5379d269ebd915fac2729"
dependencies = [
"libc",
"mio 0.7.10",
"signal-hook-registry",
]
[[package]] [[package]]
name = "signal-hook-registry" name = "signal-hook-registry"
version = "1.3.0" version = "1.3.0"
@ -2534,6 +2621,12 @@ dependencies = [
"syn 0.15.44", "syn 0.15.44",
] ]
[[package]]
name = "smallvec"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"
[[package]] [[package]]
name = "snafu" name = "snafu"
version = "0.6.10" version = "0.6.10"
@ -2611,7 +2704,7 @@ dependencies = [
"serde-xml-rs", "serde-xml-rs",
"serde_derive", "serde_derive",
"serde_json", "serde_json",
"tokio", "tokio 0.2.25",
"uuid", "uuid",
"yaque", "yaque",
] ]
@ -2830,6 +2923,16 @@ dependencies = [
"winapi 0.3.9", "winapi 0.3.9",
] ]
[[package]]
name = "tokio"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8190d04c665ea9e6b6a0dc45523ade572c088d2e6566244c1122671dbf4ae3a"
dependencies = [
"autocfg",
"pin-project-lite 0.2.6",
]
[[package]] [[package]]
name = "tokio-macros" name = "tokio-macros"
version = "0.2.6" version = "0.2.6"
@ -2841,6 +2944,17 @@ dependencies = [
"syn 1.0.64", "syn 1.0.64",
] ]
[[package]]
name = "tokio-stream"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1981ad97df782ab506a1f43bf82c967326960d278acf3bf8279809648c3ff3ea"
dependencies = [
"futures-core",
"pin-project-lite 0.2.6",
"tokio 1.2.0",
]
[[package]] [[package]]
name = "tokio-tls" name = "tokio-tls"
version = "0.3.1" version = "0.3.1"
@ -2848,7 +2962,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a70f4fcd7b3b24fb194f837560168208f669ca8cb70d0c4b862944452396343" checksum = "9a70f4fcd7b3b24fb194f837560168208f669ca8cb70d0c4b862944452396343"
dependencies = [ dependencies = [
"native-tls", "native-tls",
"tokio", "tokio 0.2.25",
] ]
[[package]] [[package]]
@ -2863,7 +2977,7 @@ dependencies = [
"futures-sink", "futures-sink",
"log", "log",
"pin-project-lite 0.1.12", "pin-project-lite 0.1.12",
"tokio", "tokio 0.2.25",
] ]
[[package]] [[package]]
@ -2909,6 +3023,19 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
[[package]]
name = "tui"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ced152a8e9295a5b168adc254074525c17ac4a83c90b2716274cc38118bddc9"
dependencies = [
"bitflags",
"cassowary",
"crossterm",
"unicode-segmentation",
"unicode-width",
]
[[package]] [[package]]
name = "typenum" name = "typenum"
version = "1.13.0" version = "1.13.0"

View File

@ -1,7 +1,7 @@
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT License. // Licensed under the MIT License.
use std::process::Command; use std::{path::PathBuf, process::Command, process::Stdio};
use anyhow::Result; use anyhow::Result;
use coverage::code::{CmdFilter, CmdFilterDef}; use coverage::code::{CmdFilter, CmdFilterDef};
@ -63,7 +63,7 @@ fn main() -> Result<()> {
let filter = opt.load_filter_or_default()?; let filter = opt.load_filter_or_default()?;
let mut cmd = Command::new(&opt.cmd[0]); let mut cmd = Command::new(&opt.cmd[0]);
cmd.args(&opt.cmd[1..]); cmd.stdin(Stdio::null()).args(&opt.cmd[1..]);
let mut cache = ModuleCache::default(); let mut cache = ModuleCache::default();
let mut recorder = Recorder::new(&mut cache, filter); let mut recorder = Recorder::new(&mut cache, filter);

View File

@ -1035,6 +1035,7 @@ impl AppVerifierController {
); );
let child = Command::new(&self.appverif_path) let child = Command::new(&self.appverif_path)
.args(args) .args(args)
.stdin(Stdio::null())
.stdout(Stdio::piped()) .stdout(Stdio::piped())
.stderr(Stdio::piped()) .stderr(Stdio::piped())
.spawn() .spawn()

View File

@ -9,7 +9,7 @@ use std::{
fs, fs,
io::Write, io::Write,
path::Path, path::Path,
process::{Command, Output}, process::{Command, Output, Stdio},
time::{Duration, Instant}, time::{Duration, Instant},
}; };
@ -317,6 +317,7 @@ pub fn test_process<'a>(
let mut command = Command::new(app_path); let mut command = Command::new(app_path);
command command
.args(args) .args(args)
.stdin(Stdio::null())
.stdout(stdout_writer) .stdout(stdout_writer)
.stderr(stderr_writer); .stderr(stderr_writer);

View File

@ -11,11 +11,13 @@ integration_test=[]
[dependencies] [dependencies]
anyhow = "1.0" anyhow = "1.0"
arraydeque = "0.4.5"
appinsights = "0.1" appinsights = "0.1"
async-trait = "0.1" async-trait = "0.1"
atexit = { path = "../atexit" }
backoff = { version = "0.3", features = ["async-std"] } backoff = { version = "0.3", features = ["async-std"] }
clap = "2.33" clap = "2.33"
tempfile = "3.2" crossterm = "0.18"
env_logger = "0.8" env_logger = "0.8"
futures = "0.3" futures = "0.3"
hex = "0.4" hex = "0.4"
@ -25,18 +27,18 @@ num_cpus = "1.13"
reqwest = { version = "0.10", features = ["json", "stream"] } reqwest = { version = "0.10", features = ["json", "stream"] }
serde = "1.0" serde = "1.0"
serde_json = "1.0" serde_json = "1.0"
onefuzz = { path = "../onefuzz" }
onefuzz-telemetry = { path = "../onefuzz-telemetry" }
path-absolutize = "3.0.6"
reqwest-retry = { path = "../reqwest-retry" }
remove_dir_all = "0.7"
stacktrace-parser = { path = "../stacktrace-parser" }
storage-queue = { path = "../storage-queue" }
tempfile = "3.2"
thiserror = "1.0"
tokio = { version = "0.2", features = ["full"] } tokio = { version = "0.2", features = ["full"] }
tokio-util = { version = "0.3", features = ["full"] } tokio-util = { version = "0.3", features = ["full"] }
tokio-stream = "0.1.3"
tui = { version = "0.14", default-features = false, features = ['crossterm'] }
url = { version = "2.2", features = ["serde"] } url = { version = "2.2", features = ["serde"] }
uuid = { version = "0.8", features = ["serde", "v4"] } uuid = { version = "0.8", features = ["serde", "v4"] }
onefuzz = { path = "../onefuzz" }
storage-queue = { path = "../storage-queue" }
reqwest-retry = { path = "../reqwest-retry" }
onefuzz-telemetry = { path = "../onefuzz-telemetry" }
stacktrace-parser = { path = "../stacktrace-parser" }
path-absolutize = "3.0.6"
atexit = { path = "../atexit" }
remove_dir_all = "0.7"
[dev-dependencies]
tempfile = "3.2"

View File

@ -1,14 +1,17 @@
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT License. // Licensed under the MIT License.
use std::env;
use std::error::Error; use std::error::Error;
use std::fs::File; use std::fs::File;
use std::io::prelude::*; use std::io::prelude::*;
use std::process::Command; use std::process::Command;
use std::{env, process::Stdio};
fn run_cmd(args: &[&str]) -> Result<String, Box<dyn Error>> { fn run_cmd(args: &[&str]) -> Result<String, Box<dyn Error>> {
let cmd = Command::new(args[0]).args(&args[1..]).output()?; let cmd = Command::new(args[0])
.args(&args[1..])
.stdin(Stdio::null())
.output()?;
if cmd.status.success() { if cmd.status.success() {
Ok(String::from_utf8_lossy(&cmd.stdout).trim().to_string()) Ok(String::from_utf8_lossy(&cmd.stdout).trim().to_string())
} else { } else {

View File

@ -5,12 +5,13 @@ use std::time::Duration;
use anyhow::Result; use anyhow::Result;
use clap::{App, Arg, SubCommand}; use clap::{App, Arg, SubCommand};
use tokio::time::timeout; use crossterm::tty::IsTty;
use tokio::{select, time::timeout};
use crate::local::{ use crate::local::{
common::add_common_config, generic_analysis, generic_crash_report, generic_generator, common::add_common_config, generic_analysis, generic_crash_report, generic_generator,
libfuzzer, libfuzzer_coverage, libfuzzer_crash_report, libfuzzer_fuzz, libfuzzer_merge, libfuzzer, libfuzzer_coverage, libfuzzer_crash_report, libfuzzer_fuzz, libfuzzer_merge,
libfuzzer_regression, libfuzzer_test_input, radamsa, test_input, libfuzzer_regression, libfuzzer_test_input, radamsa, test_input, tui::TerminalUi,
}; };
const RADAMSA: &str = "radamsa"; const RADAMSA: &str = "radamsa";
@ -25,16 +26,21 @@ const GENERIC_CRASH_REPORT: &str = "crash-report";
const GENERIC_GENERATOR: &str = "generator"; const GENERIC_GENERATOR: &str = "generator";
const GENERIC_ANALYSIS: &str = "analysis"; const GENERIC_ANALYSIS: &str = "analysis";
const GENERIC_TEST_INPUT: &str = "test-input"; const GENERIC_TEST_INPUT: &str = "test-input";
const TIMEOUT: &str = "timeout"; const TIMEOUT: &str = "timeout";
pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { pub async fn run(args: clap::ArgMatches<'static>) -> Result<()> {
let running_duration = value_t!(args, TIMEOUT, u64).ok(); let running_duration = value_t!(args, TIMEOUT, u64).ok();
let terminal = if std::io::stdout().is_tty() {
let run = async { Some(TerminalUi::init()?)
} else {
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
None
};
let event_sender = terminal.as_ref().map(|t| t.task_events.clone());
let command_run = tokio::spawn(async move {
match args.subcommand() { match args.subcommand() {
(RADAMSA, Some(sub)) => radamsa::run(sub).await, (RADAMSA, Some(sub)) => radamsa::run(sub).await,
(LIBFUZZER, Some(sub)) => libfuzzer::run(sub).await, (LIBFUZZER, Some(sub)) => libfuzzer::run(sub, event_sender).await,
(LIBFUZZER_FUZZ, Some(sub)) => libfuzzer_fuzz::run(sub).await, (LIBFUZZER_FUZZ, Some(sub)) => libfuzzer_fuzz::run(sub).await,
(LIBFUZZER_COVERAGE, Some(sub)) => libfuzzer_coverage::run(sub).await, (LIBFUZZER_COVERAGE, Some(sub)) => libfuzzer_coverage::run(sub).await,
(LIBFUZZER_CRASH_REPORT, Some(sub)) => libfuzzer_crash_report::run(sub).await, (LIBFUZZER_CRASH_REPORT, Some(sub)) => libfuzzer_crash_report::run(sub).await,
@ -49,17 +55,29 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
anyhow::bail!("missing subcommand\nUSAGE: {}", args.usage()); anyhow::bail!("missing subcommand\nUSAGE: {}", args.usage());
} }
} }
}; });
if let Some(seconds) = running_duration { if let Some(terminal) = terminal {
if let Ok(run) = timeout(Duration::from_secs(seconds), run).await { let timeout = running_duration.map(Duration::from_secs);
run let ui_run = tokio::spawn(terminal.run(timeout));
select! {
ui_result = ui_run => {
ui_result??
},
command_run_result = command_run => {
command_run_result??
}
};
Ok(())
} else if let Some(seconds) = running_duration {
if let Ok(run) = timeout(Duration::from_secs(seconds), command_run).await {
run?
} else { } else {
info!("The running timeout period has elapsed"); info!("The running timeout period has elapsed");
Ok(()) Ok(())
} }
} else { } else {
run.await command_run.await?
} }
} }
@ -70,8 +88,7 @@ pub fn args(name: &str) -> App<'static, 'static> {
Arg::with_name(TIMEOUT) Arg::with_name(TIMEOUT)
.long(TIMEOUT) .long(TIMEOUT)
.help("The maximum running time in seconds") .help("The maximum running time in seconds")
.takes_value(true) .takes_value(true),
.required(false),
) )
.subcommand(add_common_config(radamsa::args(RADAMSA))) .subcommand(add_common_config(radamsa::args(RADAMSA)))
.subcommand(add_common_config(libfuzzer::args(LIBFUZZER))) .subcommand(add_common_config(libfuzzer::args(LIBFUZZER)))

View File

@ -11,9 +11,12 @@ use reqwest::Url;
use std::task::Poll; use std::task::Poll;
use std::{ use std::{
collections::HashMap, collections::HashMap,
env::current_dir,
path::{Path, PathBuf}, path::{Path, PathBuf},
time::Duration, time::Duration,
}; };
use tokio::stream::StreamExt;
use tokio::{sync::mpsc::UnboundedSender, task::JoinHandle, time::delay_for};
use uuid::Uuid; use uuid::Uuid;
pub const SETUP_DIR: &str = "setup_dir"; pub const SETUP_DIR: &str = "setup_dir";
@ -62,6 +65,12 @@ pub enum CmdType {
// Supervisor, // Supervisor,
} }
#[derive(Clone, Debug)]
pub struct LocalContext {
pub job_path: PathBuf,
pub common_config: CommonConfig,
}
pub fn get_hash_map(args: &clap::ArgMatches<'_>, name: &str) -> Result<HashMap<String, String>> { pub fn get_hash_map(args: &clap::ArgMatches<'_>, name: &str) -> Result<HashMap<String, String>> {
let mut env = HashMap::new(); let mut env = HashMap::new();
for opt in args.values_of_lossy(name).unwrap_or_default() { for opt in args.values_of_lossy(name).unwrap_or_default() {
@ -203,7 +212,7 @@ pub fn get_synced_dir(
// fuzzing tasks from generating random task id to using UUID::nil(). This // fuzzing tasks from generating random task id to using UUID::nil(). This
// enables making the one-shot crash report generation, which isn't really a task, // enables making the one-shot crash report generation, which isn't really a task,
// consistent across multiple runs. // consistent across multiple runs.
pub fn build_common_config(args: &ArgMatches<'_>, generate_task_id: bool) -> Result<CommonConfig> { pub fn build_local_context(args: &ArgMatches<'_>, generate_task_id: bool) -> Result<LocalContext> {
let job_id = get_uuid("job_id", args).unwrap_or_else(|_| Uuid::nil()); let job_id = get_uuid("job_id", args).unwrap_or_else(|_| Uuid::nil());
let task_id = get_uuid("task_id", args).unwrap_or_else(|_| { let task_id = get_uuid("task_id", args).unwrap_or_else(|_| {
if generate_task_id { if generate_task_id {
@ -229,14 +238,19 @@ pub fn build_common_config(args: &ArgMatches<'_>, generate_task_id: bool) -> Res
register_cleanup(job_id)?; register_cleanup(job_id)?;
} }
let config = CommonConfig { let common_config = CommonConfig {
job_id, job_id,
task_id, task_id,
instance_id, instance_id,
setup_dir, setup_dir,
..Default::default() ..Default::default()
}; };
Ok(config) let current_dir = current_dir()?;
let job_path = current_dir.join(format!("{}", job_id));
Ok(LocalContext {
job_path,
common_config,
})
} }
/// Information about a local path being monitored /// Information about a local path being monitored
@ -302,3 +316,49 @@ pub async fn wait_for_dir(path: impl AsRef<Path>) -> Result<()> {
) )
.await .await
} }
pub fn spawn_file_count_monitor(
dir: PathBuf,
sender: UnboundedSender<UiEvent>,
) -> JoinHandle<Result<()>> {
tokio::spawn(async move {
wait_for_dir(&dir).await?;
loop {
let mut rd = tokio::fs::read_dir(&dir).await?;
let mut count: usize = 0;
while let Some(Ok(entry)) = rd.next().await {
if entry.path().is_file() {
count += 1;
}
}
if sender
.send(UiEvent::FileCount {
dir: dir.clone(),
count,
})
.is_err()
{
return Ok(());
}
delay_for(Duration::from_secs(5)).await;
}
})
}
pub fn monitor_file_urls(
urls: &[Option<impl AsRef<Path>>],
event_sender: UnboundedSender<UiEvent>,
) -> Vec<JoinHandle<Result<()>>> {
urls.iter()
.filter_map(|x| x.as_ref())
.map(|path| spawn_file_count_monitor(path.as_ref().into(), event_sender.clone()))
.collect::<Vec<_>>()
}
#[derive(Debug)]
pub enum UiEvent {
FileCount { dir: PathBuf, count: usize },
}

View File

@ -3,7 +3,7 @@
use crate::{ use crate::{
local::common::{ local::common::{
build_common_config, get_cmd_arg, get_cmd_exe, get_hash_map, get_synced_dir, CmdType, build_local_context, get_cmd_arg, get_cmd_exe, get_hash_map, get_synced_dir, CmdType,
ANALYSIS_DIR, ANALYZER_ENV, ANALYZER_EXE, ANALYZER_OPTIONS, CRASHES_DIR, NO_REPRO_DIR, ANALYSIS_DIR, ANALYZER_ENV, ANALYZER_EXE, ANALYZER_OPTIONS, CRASHES_DIR, NO_REPRO_DIR,
REPORTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TOOLS_DIR, UNIQUE_REPORTS_DIR, REPORTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TOOLS_DIR, UNIQUE_REPORTS_DIR,
}, },
@ -60,8 +60,8 @@ pub fn build_analysis_config(
} }
pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
let common = build_common_config(args, true)?; let context = build_local_context(args, true)?;
let config = build_analysis_config(args, None, common)?; let config = build_analysis_config(args, None, context.common_config.clone())?;
run_analysis(config).await run_analysis(config).await
} }

View File

@ -3,7 +3,7 @@
use crate::{ use crate::{
local::common::{ local::common::{
build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, CmdType, build_local_context, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, CmdType,
CHECK_ASAN_LOG, CHECK_RETRY_COUNT, CRASHES_DIR, DISABLE_CHECK_DEBUGGER, CHECK_ASAN_LOG, CHECK_RETRY_COUNT, CRASHES_DIR, DISABLE_CHECK_DEBUGGER,
DISABLE_CHECK_QUEUE, NO_REPRO_DIR, REPORTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, DISABLE_CHECK_QUEUE, NO_REPRO_DIR, REPORTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS,
TARGET_TIMEOUT, UNIQUE_REPORTS_DIR, TARGET_TIMEOUT, UNIQUE_REPORTS_DIR,
@ -71,8 +71,8 @@ pub fn build_report_config(
} }
pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
let common = build_common_config(args, true)?; let context = build_local_context(args, true)?;
let config = build_report_config(args, None, common)?; let config = build_report_config(args, None, context.common_config.clone())?;
ReportTask::new(config).managed_run().await ReportTask::new(config).managed_run().await
} }

View File

@ -3,7 +3,7 @@
use crate::{ use crate::{
local::common::{ local::common::{
build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, build_local_context, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir,
get_synced_dirs, CmdType, CHECK_ASAN_LOG, CHECK_RETRY_COUNT, CRASHES_DIR, get_synced_dirs, CmdType, CHECK_ASAN_LOG, CHECK_RETRY_COUNT, CRASHES_DIR,
DISABLE_CHECK_DEBUGGER, GENERATOR_ENV, GENERATOR_EXE, GENERATOR_OPTIONS, READONLY_INPUTS, DISABLE_CHECK_DEBUGGER, GENERATOR_ENV, GENERATOR_EXE, GENERATOR_OPTIONS, READONLY_INPUTS,
RENAME_OUTPUT, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT, TOOLS_DIR, RENAME_OUTPUT, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT, TOOLS_DIR,
@ -60,8 +60,8 @@ pub fn build_fuzz_config(args: &clap::ArgMatches<'_>, common: CommonConfig) -> R
} }
pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
let common = build_common_config(args, true)?; let context = build_local_context(args, true)?;
let config = build_fuzz_config(args, common)?; let config = build_fuzz_config(args, context.common_config.clone())?;
GeneratorTask::new(config).run().await GeneratorTask::new(config).run().await
} }

View File

@ -4,8 +4,8 @@
use crate::{ use crate::{
local::{ local::{
common::{ common::{
build_common_config, wait_for_dir, DirectoryMonitorQueue, ANALYZER_EXE, COVERAGE_DIR, build_local_context, monitor_file_urls, wait_for_dir, DirectoryMonitorQueue,
REGRESSION_REPORTS_DIR, UNIQUE_REPORTS_DIR, ANALYZER_EXE, COVERAGE_DIR, REGRESSION_REPORTS_DIR, UNIQUE_REPORTS_DIR,
}, },
generic_analysis::{build_analysis_config, build_shared_args as build_analysis_args}, generic_analysis::{build_analysis_config, build_shared_args as build_analysis_args},
libfuzzer_coverage::{build_coverage_config, build_shared_args as build_coverage_args}, libfuzzer_coverage::{build_coverage_config, build_shared_args as build_coverage_args},
@ -25,12 +25,27 @@ use anyhow::Result;
use clap::{App, SubCommand}; use clap::{App, SubCommand};
use onefuzz::utils::try_wait_all_join_handles; use onefuzz::utils::try_wait_all_join_handles;
use std::collections::HashSet; use std::collections::HashSet;
use tokio::task::spawn; use tokio::{sync::mpsc::UnboundedSender, task::spawn};
use uuid::Uuid; use uuid::Uuid;
pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { use super::common::UiEvent;
let common = build_common_config(args, true)?;
let fuzz_config = build_fuzz_config(args, common.clone())?; pub async fn run(
args: &clap::ArgMatches<'_>,
event_sender: Option<UnboundedSender<UiEvent>>,
) -> Result<()> {
let mut task_handles = vec![];
let context = build_local_context(args, true)?;
let fuzz_config = build_fuzz_config(args, context.common_config.clone())?;
if let Some(event_sender) = event_sender.clone() {
task_handles.append(&mut monitor_file_urls(
&[
fuzz_config.crashes.url.as_file_path(),
fuzz_config.inputs.url.as_file_path(),
],
event_sender,
));
}
let crash_dir = fuzz_config let crash_dir = fuzz_config
.crashes .crashes
.url .url
@ -45,6 +60,7 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
wait_for_dir(&crash_dir).await?; wait_for_dir(&crash_dir).await?;
task_handles.push(fuzz_task); task_handles.push(fuzz_task);
if args.is_present(UNIQUE_REPORTS_DIR) { if args.is_present(UNIQUE_REPORTS_DIR) {
let crash_report_input_monitor = let crash_report_input_monitor =
DirectoryMonitorQueue::start_monitoring(crash_dir.clone()).await?; DirectoryMonitorQueue::start_monitoring(crash_dir.clone()).await?;
@ -54,11 +70,32 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
Some(crash_report_input_monitor.queue_client), Some(crash_report_input_monitor.queue_client),
CommonConfig { CommonConfig {
task_id: Uuid::new_v4(), task_id: Uuid::new_v4(),
..common.clone() ..context.common_config.clone()
}, },
)?; )?;
if let Some(event_sender) = event_sender.clone() {
task_handles.append(&mut monitor_file_urls(
&[
report_config
.no_repro
.clone()
.and_then(|u| u.url.as_file_path()),
report_config
.reports
.clone()
.and_then(|u| u.url.as_file_path()),
report_config
.unique_reports
.clone()
.and_then(|u| u.url.as_file_path()),
],
event_sender,
));
}
let mut report = ReportTask::new(report_config); let mut report = ReportTask::new(report_config);
let report_task = spawn(async move { report.managed_run().await }); let report_task = spawn(async move { report.managed_run().await });
task_handles.push(report_task); task_handles.push(report_task);
task_handles.push(crash_report_input_monitor.handle); task_handles.push(crash_report_input_monitor.handle);
} }
@ -72,9 +109,26 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
Some(coverage_input_monitor.queue_client), Some(coverage_input_monitor.queue_client),
CommonConfig { CommonConfig {
task_id: Uuid::new_v4(), task_id: Uuid::new_v4(),
..common.clone() ..context.common_config.clone()
}, },
)?; )?;
if let Some(event_sender) = event_sender {
task_handles.append(&mut monitor_file_urls(
&coverage_config
.readonly_inputs
.iter()
.cloned()
.map(|input| input.url.as_file_path())
.collect::<Vec<_>>(),
event_sender.clone(),
));
task_handles.append(&mut monitor_file_urls(
&[coverage_config.coverage.url.as_file_path()],
event_sender,
));
}
let mut coverage = CoverageTask::new(coverage_config); let mut coverage = CoverageTask::new(coverage_config);
let coverage_task = spawn(async move { coverage.managed_run().await }); let coverage_task = spawn(async move { coverage.managed_run().await });
@ -89,7 +143,7 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
Some(analysis_input_monitor.queue_client), Some(analysis_input_monitor.queue_client),
CommonConfig { CommonConfig {
task_id: Uuid::new_v4(), task_id: Uuid::new_v4(),
..common.clone() ..context.common_config.clone()
}, },
)?; )?;
let analysis_task = spawn(async move { run_analysis(analysis_config).await }); let analysis_task = spawn(async move { run_analysis(analysis_config).await });
@ -103,7 +157,7 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
args, args,
CommonConfig { CommonConfig {
task_id: Uuid::new_v4(), task_id: Uuid::new_v4(),
..common ..context.common_config.clone()
}, },
)?; )?;
let regression = LibFuzzerRegressionTask::new(regression_config); let regression = LibFuzzerRegressionTask::new(regression_config);

View File

@ -3,7 +3,7 @@
use crate::{ use crate::{
local::common::{ local::common::{
build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, build_local_context, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir,
get_synced_dirs, CmdType, CHECK_FUZZER_HELP, COVERAGE_DIR, INPUTS_DIR, READONLY_INPUTS, get_synced_dirs, CmdType, CHECK_FUZZER_HELP, COVERAGE_DIR, INPUTS_DIR, READONLY_INPUTS,
TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS,
}, },
@ -55,8 +55,9 @@ pub fn build_coverage_config(
} }
pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
let common = build_common_config(args, true)?; let context = build_local_context(args, true)?;
let config = build_coverage_config(args, false, None, common)?; let config = build_coverage_config(args, false, None, context.common_config.clone())?;
let mut task = CoverageTask::new(config); let mut task = CoverageTask::new(config);
task.managed_run().await task.managed_run().await
} }

View File

@ -3,7 +3,7 @@
use crate::{ use crate::{
local::common::{ local::common::{
build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, CmdType, build_local_context, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, CmdType,
CHECK_FUZZER_HELP, CHECK_RETRY_COUNT, CRASHES_DIR, DISABLE_CHECK_QUEUE, NO_REPRO_DIR, CHECK_FUZZER_HELP, CHECK_RETRY_COUNT, CRASHES_DIR, DISABLE_CHECK_QUEUE, NO_REPRO_DIR,
REPORTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT, UNIQUE_REPORTS_DIR, REPORTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT, UNIQUE_REPORTS_DIR,
}, },
@ -63,8 +63,8 @@ pub fn build_report_config(
} }
pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
let common = build_common_config(args, true)?; let context = build_local_context(args, true)?;
let config = build_report_config(args, None, common)?; let config = build_report_config(args, None, context.common_config.clone())?;
ReportTask::new(config).managed_run().await ReportTask::new(config).managed_run().await
} }

View File

@ -3,7 +3,7 @@
use crate::{ use crate::{
local::common::{ local::common::{
build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, CmdType, build_local_context, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, CmdType,
CHECK_FUZZER_HELP, CRASHES_DIR, INPUTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, CHECK_FUZZER_HELP, CRASHES_DIR, INPUTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS,
TARGET_WORKERS, TARGET_WORKERS,
}, },
@ -50,8 +50,8 @@ pub fn build_fuzz_config(args: &clap::ArgMatches<'_>, common: CommonConfig) -> R
} }
pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
let common = build_common_config(args, true)?; let context = build_local_context(args, true)?;
let config = build_fuzz_config(args, common)?; let config = build_fuzz_config(args, context.common_config.clone())?;
LibFuzzerFuzzTask::new(config)?.run().await LibFuzzerFuzzTask::new(config)?.run().await
} }

View File

@ -3,7 +3,7 @@
use crate::{ use crate::{
local::common::{ local::common::{
build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, build_local_context, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir,
get_synced_dirs, CmdType, ANALYSIS_INPUTS, ANALYSIS_UNIQUE_INPUTS, CHECK_FUZZER_HELP, get_synced_dirs, CmdType, ANALYSIS_INPUTS, ANALYSIS_UNIQUE_INPUTS, CHECK_FUZZER_HELP,
INPUTS_DIR, PRESERVE_EXISTING_OUTPUTS, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, INPUTS_DIR, PRESERVE_EXISTING_OUTPUTS, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS,
}, },
@ -45,8 +45,8 @@ pub fn build_merge_config(
} }
pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
let common = build_common_config(args, true)?; let context = build_local_context(args, true)?;
let config = build_merge_config(args, None, common)?; let config = build_merge_config(args, None, context.common_config.clone())?;
spawn(std::sync::Arc::new(config)).await spawn(std::sync::Arc::new(config)).await
} }

View File

@ -3,7 +3,7 @@
use crate::{ use crate::{
local::common::{ local::common::{
build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, CmdType, build_local_context, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, CmdType,
CHECK_FUZZER_HELP, CHECK_RETRY_COUNT, COVERAGE_DIR, CRASHES_DIR, NO_REPRO_DIR, CHECK_FUZZER_HELP, CHECK_RETRY_COUNT, COVERAGE_DIR, CRASHES_DIR, NO_REPRO_DIR,
REGRESSION_REPORTS_DIR, REPORTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, REGRESSION_REPORTS_DIR, REPORTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS,
TARGET_TIMEOUT, UNIQUE_REPORTS_DIR, TARGET_TIMEOUT, UNIQUE_REPORTS_DIR,
@ -65,8 +65,8 @@ pub fn build_regression_config(
} }
pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
let common = build_common_config(args, true)?; let context = build_local_context(args, true)?;
let config = build_regression_config(args, common)?; let config = build_regression_config(args, context.common_config.clone())?;
LibFuzzerRegressionTask::new(config).run().await LibFuzzerRegressionTask::new(config).run().await
} }

View File

@ -3,7 +3,7 @@
use crate::{ use crate::{
local::common::{ local::common::{
build_common_config, get_cmd_arg, get_cmd_env, CmdType, CHECK_RETRY_COUNT, TARGET_ENV, build_local_context, get_cmd_arg, get_cmd_env, CmdType, CHECK_RETRY_COUNT, TARGET_ENV,
TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT, TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT,
}, },
tasks::report::libfuzzer_report::{test_input, TestInputArgs}, tasks::report::libfuzzer_report::{test_input, TestInputArgs},
@ -13,7 +13,7 @@ use clap::{App, Arg, SubCommand};
use std::path::PathBuf; use std::path::PathBuf;
pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
let common = build_common_config(args, true)?; let context = build_local_context(args, true)?;
let target_exe = value_t!(args, TARGET_EXE, PathBuf)?; let target_exe = value_t!(args, TARGET_EXE, PathBuf)?;
let target_env = get_cmd_env(CmdType::Target, args)?; let target_env = get_cmd_env(CmdType::Target, args)?;
@ -28,11 +28,11 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
target_options: &target_options, target_options: &target_options,
input_url: None, input_url: None,
input: input.as_path(), input: input.as_path(),
job_id: common.job_id, job_id: context.common_config.job_id,
task_id: common.task_id, task_id: context.common_config.task_id,
target_timeout, target_timeout,
check_retry_count, check_retry_count,
setup_dir: &common.setup_dir, setup_dir: &context.common_config.setup_dir,
minimized_stack_depth: None, minimized_stack_depth: None,
}; };

View File

@ -15,3 +15,4 @@ pub mod libfuzzer_regression;
pub mod libfuzzer_test_input; pub mod libfuzzer_test_input;
pub mod radamsa; pub mod radamsa;
pub mod test_input; pub mod test_input;
pub mod tui;

View File

@ -3,7 +3,7 @@
use crate::{ use crate::{
local::{ local::{
common::{build_common_config, DirectoryMonitorQueue}, common::{build_local_context, DirectoryMonitorQueue},
generic_crash_report::{build_report_config, build_shared_args as build_crash_args}, generic_crash_report::{build_report_config, build_shared_args as build_crash_args},
generic_generator::{build_fuzz_config, build_shared_args as build_fuzz_args}, generic_generator::{build_fuzz_config, build_shared_args as build_fuzz_args},
}, },
@ -14,11 +14,12 @@ use clap::{App, SubCommand};
use onefuzz::utils::try_wait_all_join_handles; use onefuzz::utils::try_wait_all_join_handles;
use std::collections::HashSet; use std::collections::HashSet;
use tokio::task::spawn; use tokio::task::spawn;
use uuid::Uuid; use uuid::Uuid;
pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
let common = build_common_config(args, true)?; let context = build_local_context(args, true)?;
let fuzz_config = build_fuzz_config(args, common.clone())?; let fuzz_config = build_fuzz_config(args, context.common_config.clone())?;
let crash_dir = fuzz_config let crash_dir = fuzz_config
.crashes .crashes
.url .url
@ -34,7 +35,7 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
Some(crash_report_input_monitor.queue_client), Some(crash_report_input_monitor.queue_client),
CommonConfig { CommonConfig {
task_id: Uuid::new_v4(), task_id: Uuid::new_v4(),
..common ..context.common_config.clone()
}, },
)?; )?;
let report_task = spawn(async move { ReportTask::new(report_config).managed_run().await }); let report_task = spawn(async move { ReportTask::new(report_config).managed_run().await });

View File

@ -3,7 +3,7 @@
use crate::{ use crate::{
local::common::{ local::common::{
build_common_config, get_cmd_arg, get_cmd_env, CmdType, CHECK_ASAN_LOG, CHECK_RETRY_COUNT, build_local_context, get_cmd_arg, get_cmd_env, CmdType, CHECK_ASAN_LOG, CHECK_RETRY_COUNT,
DISABLE_CHECK_DEBUGGER, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT, DISABLE_CHECK_DEBUGGER, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT,
}, },
tasks::report::generic::{test_input, TestInputArgs}, tasks::report::generic::{test_input, TestInputArgs},
@ -13,7 +13,7 @@ use clap::{App, Arg, SubCommand};
use std::path::PathBuf; use std::path::PathBuf;
pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
let common = build_common_config(args, false)?; let context = build_local_context(args, false)?;
let target_exe = value_t!(args, TARGET_EXE, PathBuf)?; let target_exe = value_t!(args, TARGET_EXE, PathBuf)?;
let target_env = get_cmd_env(CmdType::Target, args)?; let target_env = get_cmd_env(CmdType::Target, args)?;
@ -30,11 +30,11 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
target_options: &target_options, target_options: &target_options,
input_url: None, input_url: None,
input: input.as_path(), input: input.as_path(),
job_id: common.job_id, job_id: context.common_config.job_id,
task_id: common.task_id, task_id: context.common_config.task_id,
target_timeout, target_timeout,
check_retry_count, check_retry_count,
setup_dir: &common.setup_dir, setup_dir: &context.common_config.setup_dir,
minimized_stack_depth: None, minimized_stack_depth: None,
check_asan_log, check_asan_log,
check_debugger, check_debugger,

View File

@ -0,0 +1,384 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
use crate::local::common::UiEvent;
use anyhow::{Context, Result};
use crossterm::{
event::{self, Event, KeyCode},
execute,
terminal::{disable_raw_mode, enable_raw_mode, EnterAlternateScreen, LeaveAlternateScreen},
};
use futures::{StreamExt, TryStreamExt};
use log::Level;
use onefuzz::utils::try_wait_all_join_handles;
use std::{
collections::HashMap,
io::{self, Stdout, Write},
path::PathBuf,
thread::{self, JoinHandle},
time::Duration,
};
use tokio::{
sync::mpsc::{self, UnboundedReceiver},
time,
};
use tui::{
backend::CrosstermBackend,
layout::{Constraint, Corner, Direction, Layout},
style::{Color, Modifier, Style},
text::{Span, Spans},
widgets::{Block, Borders},
widgets::{List, ListItem, ListState},
Terminal,
};
use arraydeque::{ArrayDeque, Wrapping};
#[derive(Debug, thiserror::Error)]
enum UiLoopError {
#[error("program exiting")]
Exit,
#[error("error")]
Anyhow(anyhow::Error),
}
impl From<anyhow::Error> for UiLoopError {
fn from(e: anyhow::Error) -> Self {
Self::Anyhow(e)
}
}
impl From<std::io::Error> for UiLoopError {
fn from(e: std::io::Error) -> Self {
Self::Anyhow(e.into())
}
}
/// Maximum number of log message to display, arbitrarily chosen
const LOGS_BUFFER_SIZE: usize = 100;
const TICK_RATE: Duration = Duration::from_millis(250);
/// Event driving the refresh of the UI
#[derive(Debug)]
enum TerminalEvent {
Input(Event),
Tick,
FileCount { dir: PathBuf, count: usize },
Quit,
}
struct UiLoopState {
pub logs: ArrayDeque<[(Level, String); LOGS_BUFFER_SIZE], Wrapping>,
pub file_count: HashMap<PathBuf, usize>,
pub file_count_state: ListState,
pub file_monitors: Vec<JoinHandle<Result<()>>>,
pub log_event_receiver: mpsc::UnboundedReceiver<(Level, String)>,
pub terminal: Terminal<CrosstermBackend<Stdout>>,
}
impl UiLoopState {
fn new(
terminal: Terminal<CrosstermBackend<Stdout>>,
log_event_receiver: mpsc::UnboundedReceiver<(Level, String)>,
) -> Self {
Self {
log_event_receiver,
logs: Default::default(),
file_count: Default::default(),
file_count_state: Default::default(),
file_monitors: Default::default(),
terminal,
}
}
}
pub struct TerminalUi {
pub task_events: mpsc::UnboundedSender<UiEvent>,
task_event_receiver: mpsc::UnboundedReceiver<UiEvent>,
ui_event_tx: mpsc::UnboundedSender<TerminalEvent>,
ui_event_rx: mpsc::UnboundedReceiver<TerminalEvent>,
}
impl TerminalUi {
pub fn init() -> Result<Self> {
let (task_event_sender, task_event_receiver) = mpsc::unbounded_channel();
let (ui_event_tx, ui_event_rx) = mpsc::unbounded_channel();
Ok(Self {
task_events: task_event_sender,
task_event_receiver,
ui_event_tx,
ui_event_rx,
})
}
pub async fn run(self, timeout: Option<Duration>) -> Result<()> {
enable_raw_mode()?;
let mut stdout = io::stdout();
execute!(stdout, EnterAlternateScreen)?;
let backend = CrosstermBackend::new(stdout);
let mut terminal = Terminal::new(backend)?;
terminal.clear()?;
let (log_event_sender, log_event_receiver) = mpsc::unbounded_channel();
let initial_state = UiLoopState::new(terminal, log_event_receiver);
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info"))
.format(move |_buf, record| {
let _r = log_event_sender.send((record.level(), format!("{}", record.args())));
Ok(())
})
.init();
let tick_event_tx_clone = self.ui_event_tx.clone();
let tick_event_handle =
tokio::spawn(async { Self::ticking(tick_event_tx_clone).await.context("ticking") });
let keyboard_ui_event_tx = self.ui_event_tx.clone();
let _keyboard_event_handle = Self::read_keyboard_events(keyboard_ui_event_tx);
let task_event_receiver = self.task_event_receiver;
let ui_event_tx = self.ui_event_tx.clone();
let external_event_handle =
tokio::spawn(Self::read_commands(ui_event_tx, task_event_receiver));
let ui_loop = tokio::spawn(Self::ui_loop(initial_state, self.ui_event_rx));
let mut task_handles = vec![tick_event_handle, ui_loop, external_event_handle];
if let Some(timeout) = timeout {
let ui_event_tx = self.ui_event_tx.clone();
let timeout_task = tokio::spawn(async move {
time::delay_for(timeout).await;
let _ = ui_event_tx.send(TerminalEvent::Quit);
Ok(())
});
task_handles.push(timeout_task);
}
try_wait_all_join_handles(task_handles)
.await
.context("ui_loop")?;
Ok(())
}
async fn ticking(ui_event_tx: mpsc::UnboundedSender<TerminalEvent>) -> Result<()> {
let mut interval = tokio::time::interval(TICK_RATE);
loop {
interval.tick().await;
if let Err(_err) = ui_event_tx.send(TerminalEvent::Tick) {
break;
}
}
Ok(())
}
fn read_keyboard_events(
ui_event_tx: mpsc::UnboundedSender<TerminalEvent>,
) -> JoinHandle<Result<()>> {
thread::spawn(move || loop {
if event::poll(Duration::from_secs(1))? {
let event = event::read()?;
if let Err(_err) = ui_event_tx.send(TerminalEvent::Input(event)) {
return Ok(());
}
}
})
}
async fn read_commands(
ui_event_tx: mpsc::UnboundedSender<TerminalEvent>,
mut external_event_rx: mpsc::UnboundedReceiver<UiEvent>,
) -> Result<()> {
while let Some(UiEvent::FileCount { dir, count }) = external_event_rx.recv().await {
if ui_event_tx
.send(TerminalEvent::FileCount { dir, count })
.is_err()
{
break;
}
}
Ok(())
}
fn take_available_logs<T>(
receiver: &mut UnboundedReceiver<T>,
size: usize,
buffer: &mut ArrayDeque<[T; LOGS_BUFFER_SIZE], Wrapping>,
) {
let mut count = 0;
while let Ok(v) = receiver.try_recv() {
count += 1;
buffer.push_front(v);
if count >= size {
break;
}
}
}
async fn refresh_ui(ui_state: UiLoopState) -> Result<UiLoopState, UiLoopError> {
let mut logs = ui_state.logs;
let mut file_count_state = ui_state.file_count_state;
let file_count = ui_state.file_count;
let mut log_event_receiver = ui_state.log_event_receiver;
let mut terminal = ui_state.terminal;
Self::take_available_logs(&mut log_event_receiver, 10, &mut logs);
terminal.draw(|f| {
let chunks = Layout::default()
.direction(Direction::Vertical)
.constraints([Constraint::Percentage(25), Constraint::Percentage(75)].as_ref())
.split(f.size());
let mut sorted_file_count = file_count.iter().collect::<Vec<_>>();
sorted_file_count.sort_by(|(p1, _), (p2, _)| p1.cmp(p2));
let files = sorted_file_count
.iter()
.map(|(path, count)| {
ListItem::new(Spans::from(vec![
Span::raw(
path.file_name()
.map(|f| f.to_string_lossy())
.unwrap_or_default(),
),
Span::raw(": "),
Span::raw(format!("{}", count)),
]))
})
.collect::<Vec<_>>();
let log_list = List::new(files)
.block(Block::default().borders(Borders::ALL).title("files"))
.highlight_style(Style::default().add_modifier(Modifier::BOLD))
.start_corner(Corner::TopLeft);
f.render_stateful_widget(log_list, chunks[0], &mut file_count_state);
let log_items = logs
.iter()
.map(|(level, log)| {
let style = match level {
Level::Debug => Style::default().fg(Color::Magenta),
Level::Error => Style::default().fg(Color::Red),
Level::Warn => Style::default().fg(Color::Yellow),
Level::Info => Style::default().fg(Color::Blue),
Level::Trace => Style::default(),
};
ListItem::new(Spans::from(vec![
Span::styled(format!("{:<9}", level), style),
Span::raw(" "),
Span::raw(log),
]))
})
.collect::<Vec<_>>();
let log_list = List::new(log_items)
.block(Block::default().borders(Borders::ALL).title("Logs"))
.start_corner(Corner::BottomLeft);
f.render_widget(log_list, chunks[1]);
})?;
Ok(UiLoopState {
logs,
file_count_state,
file_count,
terminal,
log_event_receiver,
..ui_state
})
}
async fn on_key_down(ui_state: UiLoopState) -> Result<UiLoopState, UiLoopError> {
let mut file_count_state = ui_state.file_count_state;
let count = ui_state.file_count.len();
let i = file_count_state
.selected()
.map(|i| {
if count == 0 {
0
} else {
(i + count + 1) % count
}
})
.unwrap_or_default();
file_count_state.select(Some(i));
Ok(UiLoopState {
file_count_state,
..ui_state
})
}
async fn on_key_up(ui_state: UiLoopState) -> Result<UiLoopState, UiLoopError> {
let mut file_count_state = ui_state.file_count_state;
let count = ui_state.file_count.len();
let i = file_count_state
.selected()
.map(|i| {
if count == 0 {
0
} else {
(i + count - 1) % count
}
})
.unwrap_or_default();
file_count_state.select(Some(i));
Ok(UiLoopState {
file_count_state,
..ui_state
})
}
async fn on_quit(ui_state: UiLoopState) -> Result<UiLoopState, UiLoopError> {
let mut terminal = ui_state.terminal;
disable_raw_mode().map_err(|e| anyhow!("{:?}", e))?;
execute!(terminal.backend_mut(), LeaveAlternateScreen).map_err(|e| anyhow!("{:?}", e))?;
terminal.show_cursor()?;
Err(UiLoopError::Exit)
}
async fn on_file_count(
ui_state: UiLoopState,
dir: PathBuf,
count: usize,
) -> Result<UiLoopState, UiLoopError> {
let mut file_count = ui_state.file_count;
file_count.insert(dir, count);
Ok(UiLoopState {
file_count,
..ui_state
})
}
async fn ui_loop(
initial_state: UiLoopState,
ui_event_rx: mpsc::UnboundedReceiver<TerminalEvent>,
) -> Result<()> {
let loop_result = ui_event_rx
.map(Ok)
.try_fold(initial_state, |ui_state, event| async {
match event {
TerminalEvent::Tick => Self::refresh_ui(ui_state).await,
TerminalEvent::Input(Event::Key(k)) => match k.code {
KeyCode::Char('q') => Self::on_quit(ui_state).await,
KeyCode::Down => Self::on_key_down(ui_state).await,
KeyCode::Up => Self::on_key_up(ui_state).await,
_ => Ok(ui_state),
},
TerminalEvent::FileCount { dir, count } => {
Self::on_file_count(ui_state, dir, count).await
}
TerminalEvent::Quit => Self::on_quit(ui_state).await,
_ => Ok(ui_state),
}
})
.await;
match loop_result {
Err(UiLoopError::Exit) | Ok(_) => Ok(()),
Err(UiLoopError::Anyhow(e)) => Err(e),
}
}
}

View File

@ -22,8 +22,6 @@ const LOCAL_CMD: &str = "local";
const MANAGED_CMD: &str = "managed"; const MANAGED_CMD: &str = "managed";
fn main() -> Result<()> { fn main() -> Result<()> {
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
let built_version = format!( let built_version = format!(
"{} onefuzz:{} git:{}", "{} onefuzz:{} git:{}",
crate_version!(), crate_version!(),
@ -45,10 +43,10 @@ fn main() -> Result<()> {
result result
} }
async fn run(args: ArgMatches<'_>) -> Result<()> { async fn run(args: ArgMatches<'static>) -> Result<()> {
match args.subcommand() { match args.subcommand() {
(LICENSE_CMD, Some(_)) => licenses(), (LICENSE_CMD, Some(_)) => licenses(),
(LOCAL_CMD, Some(sub)) => local::cmd::run(sub).await, (LOCAL_CMD, Some(sub)) => local::cmd::run(sub.to_owned()).await,
(MANAGED_CMD, Some(sub)) => managed::cmd::run(sub).await, (MANAGED_CMD, Some(sub)) => managed::cmd::run(sub).await,
_ => { _ => {
anyhow::bail!("missing subcommand\nUSAGE: {}", args.usage()); anyhow::bail!("missing subcommand\nUSAGE: {}", args.usage());

View File

@ -7,6 +7,7 @@ use clap::{App, Arg, SubCommand};
use std::path::PathBuf; use std::path::PathBuf;
pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
let config_path = value_t!(args, "config", PathBuf)?; let config_path = value_t!(args, "config", PathBuf)?;
let setup_dir = value_t!(args, "setup_dir", PathBuf)?; let setup_dir = value_t!(args, "setup_dir", PathBuf)?;
let config = Config::from_file(config_path, setup_dir)?; let config = Config::from_file(config_path, setup_dir)?;

View File

@ -123,6 +123,7 @@ impl CoverageRecorder {
test_input.to_string_lossy(), test_input.to_string_lossy(),
output.to_string_lossy(), output.to_string_lossy(),
)) ))
.stdin(Stdio::null())
.stdout(Stdio::piped()) .stdout(Stdio::piped())
.stderr(Stdio::piped()) .stderr(Stdio::piped())
.kill_on_drop(true); .kill_on_drop(true);
@ -160,6 +161,7 @@ impl CoverageRecorder {
.arg(cdb_cmd) .arg(cdb_cmd)
.arg(&self.config.target_exe) .arg(&self.config.target_exe)
.arg(test_input) .arg(test_input)
.stdin(Stdio::null())
.stdout(Stdio::piped()) .stdout(Stdio::piped())
.stderr(Stdio::piped()) .stderr(Stdio::piped())
.kill_on_drop(true); .kill_on_drop(true);

View File

@ -196,8 +196,9 @@ impl LibFuzzerFuzzTask {
&self.config.common.setup_dir, &self.config.common.setup_dir,
); );
let mut running = fuzzer.fuzz(crash_dir.path(), local_inputs, &inputs)?; let mut running = fuzzer.fuzz(crash_dir.path(), local_inputs, &inputs)?;
let running_id = running.id();
let sys_info = task::spawn(report_fuzzer_sys_info(worker_id, run_id, running.id())); let sys_info = task::spawn(report_fuzzer_sys_info(worker_id, run_id, running_id));
// Splitting borrow. // Splitting borrow.
let stderr = running let stderr = running

View File

@ -211,6 +211,7 @@ async fn start_supervisor(
let cmd = cmd let cmd = cmd
.kill_on_drop(true) .kill_on_drop(true)
.env_remove("RUST_LOG") .env_remove("RUST_LOG")
.stdin(Stdio::null())
.stdout(Stdio::piped()) .stdout(Stdio::piped())
.stderr(Stdio::piped()); .stderr(Stdio::piped());

View File

@ -155,6 +155,7 @@ async fn merge(config: &Config, output_dir: impl AsRef<Path>) -> Result<()> {
cmd.kill_on_drop(true) cmd.kill_on_drop(true)
.env_remove("RUST_LOG") .env_remove("RUST_LOG")
.stdin(Stdio::null())
.stdout(Stdio::piped()) .stdout(Stdio::piped())
.stderr(Stdio::piped()); .stderr(Stdio::piped());

View File

@ -1,14 +1,17 @@
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT License. // Licensed under the MIT License.
use std::env;
use std::error::Error; use std::error::Error;
use std::fs::File; use std::fs::File;
use std::io::prelude::*; use std::io::prelude::*;
use std::process::Command; use std::process::Command;
use std::{env, process::Stdio};
fn run_cmd(args: &[&str]) -> Result<String, Box<dyn Error>> { fn run_cmd(args: &[&str]) -> Result<String, Box<dyn Error>> {
let cmd = Command::new(args[0]).args(&args[1..]).output()?; let cmd = Command::new(args[0])
.stdin(Stdio::null())
.args(&args[1..])
.output()?;
if cmd.status.success() { if cmd.status.success() {
Ok(String::from_utf8_lossy(&cmd.stdout).trim().to_string()) Ok(String::from_utf8_lossy(&cmd.stdout).trim().to_string())
} else { } else {

View File

@ -11,6 +11,7 @@ pub async fn sync(src: impl AsRef<OsStr>, dst: impl AsRef<OsStr>, delete_dst: bo
let mut cmd = Command::new("azcopy"); let mut cmd = Command::new("azcopy");
cmd.kill_on_drop(true) cmd.kill_on_drop(true)
.stdin(Stdio::null())
.stdout(Stdio::piped()) .stdout(Stdio::piped())
.stderr(Stdio::piped()) .stderr(Stdio::piped())
.arg("sync") .arg("sync")
@ -48,6 +49,7 @@ pub async fn copy(src: impl AsRef<OsStr>, dst: impl AsRef<OsStr>, recursive: boo
let mut cmd = Command::new("azcopy"); let mut cmd = Command::new("azcopy");
cmd.kill_on_drop(true) cmd.kill_on_drop(true)
.stdin(Stdio::null())
.stdout(Stdio::piped()) .stdout(Stdio::piped())
.stderr(Stdio::piped()) .stderr(Stdio::piped())
.arg("copy") .arg("copy")

View File

@ -179,6 +179,12 @@ impl BlobContainerUrl {
} }
} }
impl AsRef<Url> for BlobContainerUrl {
fn as_ref(&self) -> &Url {
self.url()
}
}
impl fmt::Debug for BlobContainerUrl { impl fmt::Debug for BlobContainerUrl {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{}", redact_query_sas_sig(self.url())) write!(f, "{}", redact_query_sas_sig(self.url()))

View File

@ -181,6 +181,7 @@ pub async fn sync_impl(
) -> Result<()> { ) -> Result<()> {
let mut cmd = Command::new("rsync"); let mut cmd = Command::new("rsync");
cmd.kill_on_drop(true) cmd.kill_on_drop(true)
.stdin(Stdio::null())
.stdout(Stdio::piped()) .stdout(Stdio::piped())
.stderr(Stdio::piped()) .stderr(Stdio::piped())
.arg(if recursive { "-zhr" } else { "-zh" }); .arg(if recursive { "-zhr" } else { "-zh" });
@ -221,6 +222,7 @@ pub async fn sync_impl(
) -> Result<()> { ) -> Result<()> {
let mut cmd = Command::new("robocopy"); let mut cmd = Command::new("robocopy");
cmd.kill_on_drop(true) cmd.kill_on_drop(true)
.stdin(Stdio::null())
.stdout(Stdio::piped()) .stdout(Stdio::piped())
.stderr(Stdio::piped()) .stderr(Stdio::piped())
.arg(&src) .arg(&src)

View File

@ -11,6 +11,8 @@ use crate::{
}; };
use anyhow::{Error, Result}; use anyhow::{Error, Result};
use stacktrace_parser::{CrashLog, StackEntry}; use stacktrace_parser::{CrashLog, StackEntry};
#[cfg(target_os = "linux")]
use std::process::Stdio;
use std::{collections::HashMap, path::Path, time::Duration}; use std::{collections::HashMap, path::Path, time::Duration};
use tempfile::tempdir; use tempfile::tempdir;
@ -197,7 +199,7 @@ impl<'a> Tester<'a> {
env: HashMap<String, String>, env: HashMap<String, String>,
) -> Result<Option<CrashLog>> { ) -> Result<Option<CrashLog>> {
let mut cmd = std::process::Command::new(self.exe_path); let mut cmd = std::process::Command::new(self.exe_path);
cmd.args(args); cmd.args(args).stdin(Stdio::null());
cmd.envs(&env); cmd.envs(&env);
let (sender, receiver) = std::sync::mpsc::channel(); let (sender, receiver) = std::sync::mpsc::channel();

View File

@ -114,6 +114,7 @@ pub async fn run_cmd<S: ::std::hash::BuildHasher>(
let mut cmd = Command::new(program); let mut cmd = Command::new(program);
cmd.env_remove("RUST_LOG") cmd.env_remove("RUST_LOG")
.stdin(Stdio::null())
.stdout(Stdio::piped()) .stdout(Stdio::piped())
.stderr(Stdio::piped()) .stderr(Stdio::piped())
.args(argv) .args(argv)